Compare commits

..

3 Commits

Author SHA1 Message Date
ringabout
af5bf7f87f Merge branch 'devel' into pr_disable_sink 2026-05-29 20:26:03 +08:00
ringabout
db00f2d2d8 Merge branch 'devel' into pr_disable_sink 2026-05-23 12:20:45 +08:00
ringabout
7b03b8a618 disable sink openarray 2025-03-14 20:21:34 +08:00
273 changed files with 2573 additions and 16977 deletions

View File

@@ -15,7 +15,7 @@ jobs:
name: ${{ matrix.platform }}-bisects
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
- name: Install OpenSSL (Windows)
if: |

View File

@@ -53,7 +53,7 @@ jobs:
steps:
- name: 'Checkout'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 2

View File

@@ -18,12 +18,12 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
batch: ["0_3", "1_3", "2_3"] # list of `index_num`
os: [ubuntu-latest, macos-14]
batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num`
include:
- os: ubuntu-latest
cpu: amd64
- os: macos-latest
- os: macos-14
cpu: arm64
name: '${{ matrix.os }} (batch: ${{ matrix.batch }})'
runs-on: ${{ matrix.os }}
@@ -33,12 +33,12 @@ jobs:
NIM_TESTAMENT_BATCH: ${{ matrix.batch }}
steps:
- name: 'Checkout'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 2
- name: 'Install node.js'
uses: actions/setup-node@v7
uses: actions/setup-node@v6
with:
node-version: 24

View File

@@ -17,12 +17,12 @@ jobs:
runs-on: ${{ matrix.os }}
steps:
- name: 'Checkout'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
fetch-depth: 2
- name: 'Install node.js'
uses: actions/setup-node@v7
uses: actions/setup-node@v6
with:
node-version: 24

1
.gitignore vendored
View File

@@ -87,7 +87,6 @@ tweeter_test.db
/tests/megatest.nim
/tests/ic/*_temp.nim
/tests/ic/*_mm/
/tests/navigator/*_temp.nim

View File

@@ -43,15 +43,6 @@ parameter and result types, not just their source-level shape. Use
[//]: # "Additions:"
- Added `system.readRawDataStable`, a companion to `readRawData` that returns a
raw `ptr UncheckedArray[char]` into a string's character data which stays valid
across moves and copies of the string value. It is available under every string
implementation (refc, ARC/ORC and `--strings:sso`) with the same signature, so
code can pin an interior buffer pointer today and be ready for `--strings:sso`
without `when declared` guards. Under `--strings:sso` it promotes a small inline
string to its heap representation first; under the other implementations the data
is already heap-resident, so it is equivalent to `readRawData`.
- `setutils.symmetricDifference` along with its operator version
`` setutils.`-+-` `` and in-place version `setutils.toggle` have been added
to more efficiently calculate the symmetric difference of bitsets.
@@ -79,12 +70,8 @@ parameter and result types, not just their source-level shape. Use
Modes include `Nim` (default, fully compatible) and two new experimental modes:
`Lax` and `Gnu` for different option parsing behaviors.
- `std/symlinks.expandSymlink` now supports Windows symlinks and junctions with
POSIX-like single-hop `readlink` semantics.
- `std/nre2` is added to replace deprecated NRE.
- `system.typeof` adds a new parameter `modifierMode` to specify how type modifiers are handled.
[//]: # "Changes:"
- `std/math` The `^` symbol now supports floating-point as exponent in addition to the Natural type.
@@ -97,8 +84,6 @@ parameter and result types, not just their source-level shape. Use
- `std/pegs` now correctly lexes UTF-8 bytes inside bare identifier-style
terminals, so case-insensitive matching of non-ASCII terms (e.g. ``\i café``)
works without single-quoting.
- `std/uri`: The `?` operator now appends query parameters to an existing query
string instead of replacing it. Fixes [#19782](https://github.com/nim-lang/Nim/issues/19782).
## Language changes

View File

@@ -36,13 +36,6 @@ proc setupProgram*(config: ConfigRef; cache: IdentCache) =
when not defined(nimKochBootstrap):
program = createDecodeContext(config, cache)
proc setIcMainModule*(fileIdx: FileIndex) =
## Tells the IC loader which module is being compiled fresh, so that
## re-exports of that module's symbols by dependencies are not loaded as
## duplicate stubs.
when not defined(nimKochBootstrap):
ast2nif.setMainModule(program, fileIdx)
template loadSym(s: PSym) =
## Loads a symbol from NIF file if it's in Partial state.
when not defined(nimKochBootstrap):
@@ -77,16 +70,6 @@ proc backendEnsureMutable*(t: PType) {.inline.} =
# ^ IC review this later
if t.state == Partial: loadType(t)
proc unsealForTransform*(t: PType) {.inline.} =
## The transformer/lambda lifting also run inside `nim m` when the VM
## compiles a LOADED routine (macro evaluation, `getImpl`). Their mutations
## are process-local — transformed bodies are never written back to a NIF —
## so downgrade the loaded type to mutable, mirroring the `cmdNifC` loader
## which loads everything `Complete` for exactly this reason (see
## `ast2nif.loadedState`).
if t.state == Partial: loadType(t)
if t.state == Sealed: t.state = Complete
proc owner*(s: PSym): PSym {.inline.} =
if s.state == Partial: loadSym(s)
result = s.ownerFieldImpl
@@ -238,10 +221,7 @@ proc position*(s: PSym): int {.inline.} =
result = s.positionImpl
proc `position=`*(s: PSym, val: int) {.inline.} =
# No `Sealed` guard: the VM reuses `position` as a register slot while compiling
# a macro for execution (see `vmgen.genGenericParams`), which under IC may be a
# macro loaded from a NIF file. The macro is run, not code-generated, so this
# scratch mutation is harmless.
assert s.state != Sealed
if s.state == Partial: loadSym(s)
s.positionImpl = val
@@ -332,10 +312,7 @@ when defined(nimsuggest):
result = s.allUsagesImpl
proc `allUsages=`*(s: PSym, val: sink seq[TLineInfo]) {.inline.} =
# No `assert s.state != Sealed`: `allUsagesImpl` is nimsuggest-only usage
# tracking, NOT part of the NIF-serialized symbol. nimsuggest loads symbols
# as `Sealed` (ast2nif.loadedState under cmdM) yet `suggestSym` legitimately
# records usages on them; the getter likewise doesn't assert.
assert s.state != Sealed
if s.state == Partial: loadSym(s)
s.allUsagesImpl = val
@@ -468,18 +445,12 @@ var gconfig {.threadvar.}: Gconfig
proc setUseIc*(useIc: bool) = gconfig.useIc = useIc
proc comment*(n: PNode): string =
if nfHasComment in n.flags:
# NIF-based IC doesn't serialize comments, but the comment table is keyed by
# the node's address (`nodeId`), which is unique among live nodes; a loaded
# node that carries `nfHasComment` simply has no entry here (its comment was
# set in another process), so `getOrDefault` safely returns "" for it while
# in-process VM macro nodes (e.g. newCommentStmtNode) still round-trip.
result = gconfig.comments.getOrDefault(n.nodeId)
if nfHasComment in n.flags and not gconfig.useIc:
# IC doesn't track comments, see `packed_ast`, so this could fail
result = gconfig.comments[n.nodeId]
else:
result = ""
nodeCommentReader = proc(n: PNode): string {.nimcall.} = comment(n)
proc `comment=`*(n: PNode, a: string) =
let id = n.nodeId
if a.len > 0:
@@ -495,8 +466,6 @@ proc `comment=`*(n: PNode, a: string) =
n.flags.excl nfHasComment
gconfig.comments.del(id)
nodeCommentWriter = proc(n: PNode; s: string) {.nimcall.} = n.comment = s
# BUGFIX: a module is overloadable so that a proc can have the
# same name as an imported module. This is necessary because of
# the poor naming choices in the standard library.
@@ -509,6 +478,13 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym: a.sons[0].sym.name
else: nil
const
moduleShift = when defined(cpu32): 20 else: 24
template toId*(a: ItemId): int =
let x = a
(x.module.int shl moduleShift) + x.item.int
template id*(a: PType | PSym): int = toId(a.itemId)
type
@@ -517,62 +493,28 @@ type
symId*: int32
typeId*: int32
sealed*: bool
backendMinted*: bool
disambTable*: CountTable[PIdent]
const
PackageModuleId* = -3'i32
proc idGeneratorFromModule*(m: PSym): IdGenerator =
assert m.kind == skModule
result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0, disambTable: initCountTable[PIdent]())
result.disambTable.inc m.name
proc idGeneratorForBackend*(m: PSym): IdGenerator =
## Like `idGeneratorFromModule`, but for IC codegen (`nim nifc`): symbols and
## types minted fresh during codegen (transf labels/temps, lifted hooks, type
## copies) must not collide with the itemIds the NIF loader synthesizes for
## lazily-loaded symbols/types of the same module — those come from a
## per-module load-order counter that keeps running while codegen mints its
## own ids. A collision corrupts itemId-keyed tables, e.g. `transf`'s inline
## iterator mapping then substitutes a random loaded sym (a call's callee)
## with a `:tmp` block label. Backend-minted ids carry a marker bit in the
## module half (see `itemids.backendItemId`), so the two id spaces are
## disjoint by construction.
assert m.kind == skModule
result = IdGenerator(module: m.itemId.module, symId: 0, typeId: 0,
backendMinted: true, disambTable: initCountTable[PIdent]())
result.disambTable.inc m.name
proc idGeneratorForPackage*(nextIdWillBe: int32): IdGenerator =
result = IdGenerator(module: PackageModuleId, symId: nextIdWillBe - 1'i32, typeId: 0, disambTable: initCountTable[PIdent]())
proc nextSymId(x: IdGenerator): ItemId {.inline.} =
assert(not x.sealed)
when not defined(nimKochBootstrap):
if x.backendMinted:
# Share the loader's per-module backend counter so a freshly-minted
# backend sym never collides with an `@bk` sym loaded from the module's
# `.t.bif` (see ast2nif.nextBackendSymItem).
let it = nextBackendSymItem(program, x.module)
if it >= 0'i32:
return backendItemId(x.module, it)
inc x.symId
result = if x.backendMinted: backendItemId(x.module, x.symId)
else: itemId(x.module, x.symId)
result = ItemId(module: x.module, item: x.symId)
proc nextTypeId*(x: IdGenerator): ItemId {.inline.} =
assert(not x.sealed)
when not defined(nimKochBootstrap):
if x.backendMinted:
# Share the loader's per-module backend TYPE counter (seeded from the
# module's `(unusedid)`) so a freshly-minted backend type sits ABOVE every
# loaded type — never colliding with a frontend type's `toId` (the bug that
# crashed cgen's `getTypeDescAux` cycle check on `AsyncBufferRef`). Mirrors
# `nextSymId` (see ast2nif.nextBackendTypeItem).
let it = nextBackendTypeItem(program, x.module)
if it >= 0'i32:
return backendItemId(x.module, it)
inc x.typeId
result = if x.backendMinted: backendItemId(x.module, x.typeId)
else: itemId(x.module, x.typeId)
result = ItemId(module: x.module, item: x.typeId)
when false:
proc nextId*(x: IdGenerator): ItemId {.inline.} =
@@ -849,10 +791,6 @@ proc newSymNode*(sym: PSym): PNode =
result = newNode(nkSym)
result.sym = sym
result.typField = sym.typ
if result.typField == nil and nifcBackendActive:
# See the two-arg overload in astdef: in the NIF backend cg stage a sym node
# built from a not-yet-typed stub must track the symbol's type lazily.
result.flags.incl nfLazyType
result.info = sym.info
proc newOpenSym*(n: PNode): PNode {.inline.} =
@@ -1105,11 +1043,6 @@ proc newType*(kind: TTypeKind; idgen: IdGenerator; owner: PSym; son: sink PType
if result.itemId.module == 55 and result.itemId.item == 2:
echo "KNID ", kind
writeStackTrace()
when defined(icDbg):
if kind == tyOpenArray:
echo "NEWTYPE openArray id=", id.module, ".", id.item,
" owner=", (if owner != nil: owner.name.s else: "nil")
echo getStackTrace()
proc setSons*(dest: PType; sons: sink seq[PType]) {.inline.} =
assert dest.kind != tyProc or sons.len <= 1
@@ -1172,19 +1105,10 @@ proc copyType*(t: PType, idgen: IdGenerator, owner: PSym): PType =
assignType(result, t)
result.symImpl = t.sym # backend-info should not be copied
proc exactReplica*(t: PType; idgen: IdGenerator): PType =
## Replica that KEEPS `itemId` — the generic-param binding tables
## (`LayeredIdTable`) key on it, so the copy must keep matching its
## original — but mints a FRESH `uniqueId`: uniqueId is the SERIALIZATION
## identity (NIF type names key on it) and must be unique per instance.
## Replicas sharing the original's uniqueId serialized as duplicate defs
## under one NIF name; the loader collapsed them into a single type,
## losing their flag differences (use-site `tfUnresolved` typedescs) or
## their structure (meta instance bodies shadowing a generic's canonical
## body).
proc exactReplica*(t: PType): PType =
result = PType(kind: t.kind, ownerFieldImpl: t.owner, sizeImpl: defaultSize,
alignImpl: defaultAlignment, itemId: t.itemId,
uniqueId: nextTypeId(idgen))
uniqueId: t.uniqueId)
assignType(result, t)
result.symImpl = t.sym # backend-info should not be copied
@@ -1347,9 +1271,6 @@ proc transitionNoneToSym*(n: PNode) =
transitionNodeKindCommon(nkSym)
template transitionSymKindCommon*(k: TSymKind) =
# Under IC the symbol may still be an unloaded stub (`skStub`); materialise it
# first so its kind-specific fields (read below as `obj.*`) actually exist.
if s.state == Partial: loadSym(s)
let obj {.inject.} = s[]
s[] = TSym(kindImpl: k, itemId: obj.itemId, magicImpl: obj.magicImpl, typImpl: obj.typImpl, name: obj.name,
infoImpl: obj.infoImpl, ownerFieldImpl: obj.ownerFieldImpl, flagsImpl: obj.flagsImpl, astImpl: obj.astImpl,
@@ -1726,13 +1647,9 @@ proc canRaise*(fn: PNode): bool =
if fn.typ.n[0].kind == nkSym:
result = false
else:
# A proc-typed value with no explicit raises slot still has
# unspecified effects, which sempass2 treats conservatively.
# Codegen needs to do the same in order to keep goto-exception
# checks after indirect/closure calls.
result = ((fn.typ.n[0].len < effectListLen) or
fn.typ.n[0][exceptionEffects] == nil or
fn.typ.n[0][exceptionEffects].safeLen > 0)
(fn.typ.n[0][exceptionEffects] != nil and
fn.typ.n[0][exceptionEffects].safeLen > 0))
else:
result = false

File diff suppressed because it is too large Load Diff

View File

@@ -17,21 +17,9 @@ when defined(nimPreviewSlimSystem):
export int128
var nifcBackendActive* = false
## Set only while the per-module NIF backend codegen stage runs
## (`nifbackend.generateCgStage`, `cmd == cmdNifC`). It gates `newSymNode`'s
## lazy-type marking so it applies ONLY in the backend — where syms are loaded
## from NIF and a cg-stage transform can build a sym node from a not-yet-typed
## stub — and never during frontend sem, where the same marking would perturb
## effect/exception inference (it diverges from a non-IC build, e.g.
## `times.toDateTimeByWeek` gaining a spurious unlisted `Exception`).
import nodekinds
export nodekinds
import itemids
export itemids
type
TCallingConvention* = enum
ccNimCall = "nimcall" # nimcall, also the default
@@ -339,14 +327,6 @@ type
# because openSym experimental switch is disabled
# gives warning instead
nfLazyType # node has a lazy type
nfLazyBody # IC: this node is a placeholder for a routine body (bodyPos son)
# not yet materialized. Reading its children (via `len`/`safeLen`)
# triggers `forceLazyBodyHook`. Process-local, stripped on serialize.
nfBroadcast # this `nkBracket` is a *broadcast* default array: a single son
# standing for `lengthOrd` identical zero copies (see
# `broadcastArrayThreshold`). The flag disambiguates it from an
# ordinary 1-element collection (e.g. a seq value that happens to
# carry an array type), so it must survive copies + serialization.
TNodeFlags* = set[TNodeFlag]
TTypeFlag* = enum # keep below 32 for efficiency reasons (now: 47)
@@ -591,6 +571,23 @@ const
generatedMagics* = {mNone, mIsolate, mFinished, mOpenArrayToSeq}
## magics that are generated as normal procs in the backend
type
ItemId* = object
module*: int32
item*: int32
proc `$`*(x: ItemId): string =
"(module: " & $x.module & ", item: " & $x.item & ")"
proc `==`*(a, b: ItemId): bool {.inline.} =
a.item == b.item and a.module == b.module
proc hash*(x: ItemId): Hash =
var h: Hash = hash(x.module)
h = h !& hash(x.item)
result = !$h
type
PNode* = ref TNode
TNodeSeq* = seq[PNode]
@@ -874,8 +871,7 @@ const
nfFromTemplate, nfDefaultRefsParam,
nfExecuteOnReload, nfLastRead,
nfFirstWrite, nfSkipFieldChecking,
nfDisabledOpenSym, nfLazyType,
nfBroadcast}
nfDisabledOpenSym, nfLazyType}
namePos* = 0
patternPos* = 1 # empty except for term rewriting macros
genericParamsPos* = 2
@@ -912,24 +908,7 @@ const
defaultOffset* = -1
var forceLazyBodyHook*: proc (n: PNode) {.nimcall, raises: [], tags: [], gcsafe.}
## Set by the IC loader (ast2nif). When a node carries `nfLazyBody`, any access
## to its children through `len` materializes the deferred routine body in place.
## `safeLen` delegates to `len`, so it is covered transitively; a lazy body is
## never a leaf kind, so the `{nkNone..nkNilLit}` short-circuit never hides it.
##
## The type MUST be effect-free (`raises: []`/`tags: []`): `len` is a fundamental
## `PNode` accessor that the whole compiler — and every compiler-as-library
## consumer (nimble, nimsuggest, ...) — assumes cannot raise. An unannotated
## `proc` var defaults to `raises: [Exception]`, so the indirect call tainted
## `len`/`safeLen`/`items` with `Exception`, breaking any iterator/`{.raises.}`
## over a `PNode` (e.g. nimble's `extract {.raises: [CatchableError].}`).
## Materialization is a pure in-memory buffer transform; a corrupt buffer is a
## `Defect` (`raiseAssert`), which is outside exception tracking.
proc len*(n: PNode): int {.inline.} =
if nfLazyBody in n.flags and forceLazyBodyHook != nil:
forceLazyBodyHook(n)
result = n.sons.len
proc safeLen*(n: PNode): int {.inline.} =
@@ -1005,14 +984,6 @@ proc newSymNode*(sym: PSym, info: TLineInfo): PNode =
result = newNode(nkSym)
result.sym = sym
result.typField = sym.typImpl
if result.typField == nil and nifcBackendActive:
# In the per-module NIF backend cg stage a transform (chronos async
# closure-iterator lowering) builds `result = …` sym nodes from a not-yet-typed
# NIF stub; snapshotting the nil here would leave the node permanently typeless
# and the backend later reads `t.flags` off it and SIGSEGVs (injectdestructors
# hasDestructor). Mark it lazy so `typ` re-reads `sym.typ` once resolved. Gated
# on `nifcBackendActive` so frontend sem is untouched (see the flag's doc).
result.flags.incl nfLazyType
result.info = info
proc newStrNode*(kind: TNodeKind, strVal: string): PNode =
@@ -1029,8 +1000,7 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode =
type
LogEntryKind* = enum
HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry,
PureEnumEntry
HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry
LogEntry* = object
kind*: LogEntryKind
op*: TTypeAttachedOp
@@ -1193,11 +1163,3 @@ proc strTableGet*(t: TStrTable, name: PIdent): PSym =
if result == nil: break
if result.name.id == name.id: break
h = nextTry(h, high(t.data))
# --- doc-comment bridge for the NIF serializer -------------------------------
# `ast2nif` (the NIF reader/writer) cannot import `ast` (where the comment
# accessor and its `gconfig.comments` side table live) because `ast` imports
# `ast2nif`. These hooks are assigned by `ast` and let the serializer carry a
# decl's `##` doc comment across a NIF round-trip.
var nodeCommentReader*: proc(n: PNode): string {.nimcall.}
var nodeCommentWriter*: proc(n: PNode; s: string) {.nimcall.}

View File

@@ -43,13 +43,13 @@ proc flagsToStr[T](flags: set[T]): string =
proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): string =
result = "["
result.addYamlString(toFilename(conf, info))
result.addf ", $1, $2]", toLinenumber(info), toColumn(info)
result.addf ", $1, $2]", [toLinenumber(info), toColumn(info)]
proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; nl: bool, indent, maxRecDepth: int)
proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; nl: bool, indent, maxRecDepth: int)
proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; nl: bool, indent, maxRecDepth: int)
proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; indent, maxRecDepth: int)
proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; indent, maxRecDepth: int)
proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; indent, maxRecDepth: int)
proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; nl: bool, indent: int; maxRecDepth: int) =
proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; indent: int; maxRecDepth: int) =
if n == nil:
res.add("null")
elif containsOrIncl(marker, n.id):
@@ -57,12 +57,10 @@ proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet;
else:
let istr = spaces(indent * 4)
if nl:
res.addf("\n$1", istr)
res.addf("kind: $1", [makeYamlString($n.kind)])
res.addf("\n$1name: $2", [istr, makeYamlString(n.name.s)])
res.addf("\n$1typ: ", [istr])
res.typeToYamlAux(conf, n.typ, marker, true, indent + 1, maxRecDepth - 1)
res.typeToYamlAux(conf, n.typ, marker, indent + 1, maxRecDepth - 1)
if conf != nil:
# if we don't pass the config, we probably don't care about the line info
res.addf("\n$1info: $2", [istr, lineInfoToStr(conf, n.info)])
@@ -70,7 +68,7 @@ proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet;
res.addf("\n$1flags: $2", [istr, flagsToStr(n.flags)])
res.addf("\n$1magic: $2", [istr, makeYamlString($n.magic)])
res.addf("\n$1ast: ", [istr])
res.treeToYamlAux(conf, n.ast, marker, true, indent + 1, maxRecDepth - 1)
res.treeToYamlAux(conf, n.ast, marker, indent + 1, maxRecDepth - 1)
res.addf("\n$1options: $2", [istr, flagsToStr(n.options)])
res.addf("\n$1position: $2", [istr, $n.position])
res.addf("\n$1k: $2", [istr, makeYamlString($n.loc.k)])
@@ -78,57 +76,53 @@ proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet;
if card(n.loc.flags) > 0:
res.addf("\n$1flags: $2", [istr, makeYamlString($n.loc.flags)])
res.addf("\n$1snippet: $2", [istr, n.loc.snippet])
res.addf("\n$1lode: ", [istr])
res.treeToYamlAux(conf, n.loc.lode, marker, true, indent + 1, maxRecDepth - 1)
res.addf("\n$1lode: $2", [istr])
res.treeToYamlAux(conf, n.loc.lode, marker, indent + 1, maxRecDepth - 1)
proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; nl: bool, indent: int; maxRecDepth: int) =
proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; indent: int; maxRecDepth: int) =
if n == nil:
res.add("null")
elif containsOrIncl(marker, n.id):
res.addf "\"$1 @$2\"" % [$n.kind, strutils.toHex(cast[uint](n), sizeof(n) * 2)]
else:
let istr = spaces(indent * 4)
if nl:
res.addf("\n$1", istr)
res.addf("kind: $2", [istr, makeYamlString($n.kind)])
res.addf("\n$1sym: ", istr)
res.symToYamlAux(conf, n.sym, marker, true, indent + 1, maxRecDepth - 1)
res.addf("\n$1n: ", istr)
res.treeToYamlAux(conf, n.n, marker, true, indent + 1, maxRecDepth - 1)
res.addf("\n$1sym: ")
res.symToYamlAux(conf, n.sym, marker, indent + 1, maxRecDepth - 1)
res.addf("\n$1n: ")
res.treeToYamlAux(conf, n.n, marker, indent + 1, maxRecDepth - 1)
if card(n.flags) > 0:
res.addf("\n$1flags: $2", [istr, flagsToStr(n.flags)])
res.addf("\n$1callconv: $2", [istr, makeYamlString($n.callConv)])
res.addf("\n$1size: $2", [istr, $(n.size)])
res.addf("\n$1align: $2", [istr, $(n.align)])
if n.hasElementType:
res.addf("\n$1sons:", istr)
res.addf("\n$1sons:")
for a in n.kids:
res.addf("\n$1 - ", istr)
res.typeToYamlAux(conf, a, marker, false, indent + 1, maxRecDepth - 1)
res.addf("\n - ")
res.typeToYamlAux(conf, a, marker, indent + 1, maxRecDepth - 1)
proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; nl: bool, indent: int;
proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; indent: int;
maxRecDepth: int) =
if n == nil:
res.add("null")
else:
var istr = spaces(indent * 4)
if nl:
res.addf("\n$1", istr)
res.addf("kind: $1" % [makeYamlString($n.kind)])
if maxRecDepth != 0:
if conf != nil:
res.addf("\n$1info: $2", [istr, lineInfoToStr(conf, n.info)])
case n.kind
of nkCharLit .. nkUInt64Lit:
of nkCharLit .. nkInt64Lit:
res.addf("\n$1intVal: $2", [istr, $(n.intVal)])
of nkFloatLit .. nkFloat128Lit:
of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
res.addf("\n$1floatVal: $2", [istr, n.floatVal.toStrMaxPrecision])
of nkStrLit .. nkTripleStrLit:
res.addf("\n$1strVal: $2", [istr, makeYamlString(n.strVal)])
of nkSym:
res.addf("\n$1sym: ", [istr])
res.symToYamlAux(conf, n.sym, marker, true, indent + 1, maxRecDepth)
res.symToYamlAux(conf, n.sym, marker, indent + 1, maxRecDepth)
of nkIdent:
if n.ident != nil:
res.addf("\n$1ident: $2", [istr, makeYamlString(n.ident.s)])
@@ -139,22 +133,22 @@ proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSe
res.addf("\n$1sons: ", [istr])
for i in 0 ..< n.len:
res.addf("\n$1 - ", [istr])
res.treeToYamlAux(conf, n[i], marker, false, indent + 1, maxRecDepth - 1)
res.treeToYamlAux(conf, n[i], marker, indent + 1, maxRecDepth - 1)
if n.typ != nil:
res.addf("\n$1typ: ", [istr])
res.typeToYamlAux(conf, n.typ, marker, true, indent + 1, maxRecDepth)
res.typeToYamlAux(conf, n.typ, marker, indent + 1, maxRecDepth)
proc treeToYaml*(conf: ConfigRef; n: PNode; indent: int = 0; maxRecDepth: int = -1): string =
var marker = initIntSet()
result = newStringOfCap(1024)
result.treeToYamlAux(conf, n, marker, false, indent, maxRecDepth)
result.treeToYamlAux(conf, n, marker, indent, maxRecDepth)
proc typeToYaml*(conf: ConfigRef; n: PType; indent: int = 0; maxRecDepth: int = -1): string =
var marker = initIntSet()
result = newStringOfCap(1024)
result.typeToYamlAux(conf, n, marker, false, indent, maxRecDepth)
result.typeToYamlAux(conf, n, marker, indent, maxRecDepth)
proc symToYaml*(conf: ConfigRef; n: PSym; indent: int = 0; maxRecDepth: int = -1): string =
var marker = initIntSet()
result = newStringOfCap(1024)
result.symToYamlAux(conf, n, marker, false, indent, maxRecDepth)
result.symToYamlAux(conf, n, marker, indent, maxRecDepth)

View File

@@ -394,7 +394,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; n
# variable. Thus, we create a temporary pointer variable instead.
let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray
if needsIndirect:
n.typ = n.typ.exactReplica(p.module.idgen)
n.typ = n.typ.exactReplica
n.typ.incl tfVarIsPtr
a = initLocExprSingleUse(p, n)
a = withTmpIfNeeded(p, a, needsTmp)
@@ -909,16 +909,6 @@ proc isInactiveDestructorCall(p: BProc, e: PNode): bool =
proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri):
return
when defined(icDbgHash):
if ri[0].typ == nil:
echo "NILCALLEE kind=", ri[0].kind,
" sym=", (if ri[0].kind == nkSym: ri[0].sym.name.s else: "-"),
" symKind=", (if ri[0].kind == nkSym: $ri[0].sym.kind else: "-"),
" flags=", (if ri[0].kind == nkSym: $ri[0].sym.flags else: "-"),
" lazy=", nfLazyType in ri[0].flags,
" inProc=", (if p.prc != nil: p.prc.name.s else: "NIL"),
" module=", p.module.module.name.s
raiseAssert "nil callee type, see NILCALLEE above"
if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
genClosureCall(p, le, ri, d)
elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags:

View File

@@ -1071,7 +1071,7 @@ proc genRecordField(p: BProc, e: PNode, d: var TLoc) =
proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc)
proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym, ty: PType) =
proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
var test, u, v: TLoc
for i in 1..<e.len:
var it = e[i]
@@ -1081,17 +1081,10 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym, ty: PType) =
if op.magic == mNot: it = it[1]
let disc = it[2].skipConv
assert(disc.kind == nkSym)
# Re-navigate the discriminant in the object type: under `nim ic` `disc.sym` is
# a field-use stub whose `loc.snippet` is empty (the backend fills it on the
# canonical reclist field, not on per-use leaves). Look up the canonical field
# for the C member name; `disc`'s own node still supplies its type (TLoc.t).
# Byte-neutral for non-IC, where re-navigation returns the same field.
var rr = obj
let dfield = lookupFieldAgain(p, ty, disc.sym, rr)
test = initLoc(locNone, it, OnStack)
u = initLocExpr(p, it[1])
v = initLoc(locExpr, disc, OnUnknown)
v.snippet = dotField(obj, dfield.loc.snippet)
v.snippet = dotField(obj, disc.sym.loc.snippet)
genInExprAux(p, it, u, v, test)
var msg = ""
if optDeclaredLocs in p.config.globalOptions:
@@ -1101,7 +1094,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym, ty: PType) =
# by encoding the file names separately from `file(line:col)`, essentially
# passing around `TLineInfo` + the set of files in the project.
msg.add toFileLineCol(p.config, e.info) & " "
msg.add genFieldDefect(p.config, field.name.s, dfield)
msg.add genFieldDefect(p.config, field.name.s, disc.sym)
var strLitBuilder = newBuilder("")
genStringLiteral(p.module, newStrNode(nkStrLit, msg), strLitBuilder)
let strLit = extract(strLitBuilder)
@@ -1165,7 +1158,7 @@ proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) =
if field.loc.snippet == "": fillObjectFields(p.module, ty)
if field.loc.snippet == "":
internalError(p.config, e.info, "genCheckedRecordField") # generate the checks:
genFieldCheck(p, e, r, field, ty)
genFieldCheck(p, e, r, field)
r = dotField(r, field.loc.snippet)
putIntoDest(p, d, e[0], r, a.storage)
r.freeze
@@ -1865,7 +1858,7 @@ proc genFieldObjConstr(p: BProc; ty: PType; useTemp, isRef: bool; nField, val, c
if field.loc.snippet == "": fillObjectFields(p.module, ty)
if field.loc.snippet == "": internalError(p.config, info, "genFieldObjConstr")
if check != nil and optFieldCheck in p.options:
genFieldCheck(p, check, r, field, ty)
genFieldCheck(p, check, r, field)
tmp2.snippet = dotField(tmp2.snippet, field.loc.snippet)
if useTemp:
tmp2.k = locTemp
@@ -1911,9 +1904,7 @@ proc genObjConstr(p: BProc, e: PNode, d: var TLoc) =
var tmp: TLoc = default(TLoc)
var r: Rope
let needsZeroMem =
nfAllFieldsSet notin e.flags or
(optSeqDestructors notin p.config.globalOptions and containsGarbageCollectedRef(t))
let needsZeroMem = p.config.selectedGC notin {gcArc, gcAtomicArc, gcOrc, gcYrc} or nfAllFieldsSet notin e.flags
if useTemp:
tmp = getTemp(p, t)
r = rdLoc(tmp)
@@ -2940,13 +2931,6 @@ proc genEnumToStr(p: BProc, e: PNode, d: var TLoc) =
proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
case op
of mAsgn:
let kind = if e[0].sym.name.s == "=sink": nkSinkAsgn else: nkAsgn
let lhs = e[1].skipHiddenAddr
let n = newTreeI(kind, e.info, lhs, e[2])
n.typ = e.typ
cow(p, e[2])
genAsgn(p, n, fastAsgn = kind != nkAsgn)
of mOr, mAnd: genAndOr(p, e, d, op)
of mNot..mUnaryMinusF64: unaryArith(p, e, d, op)
of mUnaryMinusI..mAbsI: unaryArithOverflow(p, e, d, op)
@@ -3143,12 +3127,6 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
localError(p.config, e.info,
"for --mm:arc|atomicArc|orc 'deepcopy' support has to be enabled with --deepcopy:on")
let typ = e[1].typ.skipTypes({tyVar, tyRef, tyGenericInst, tyTypeDesc,
tyAlias, tyInferred, tySink, tyLent, tyOwned})
if hasDisabledAsgn(p.module.g.graph, typ):
localError(p.config, e.info,
"'deepCopy' is not available for type <" & typeToString(typ) & ">")
let x = if e[1].kind in {nkAddr, nkHiddenAddr}: e[1][0] else: e[1]
var a = initLocExpr(p, x)
var b = initLocExpr(p, e[2])
@@ -3511,23 +3489,7 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) =
data.addDeclWithVisibility(Private):
data.addVarWithInitializer(Local, actualConstName, typ = td):
genBracedInit(q.initProc, sym.astdef, isConst = true, sym.typ, data)
if q.config.cmd == cmdNifC:
# Each `cg` process that demands this const emits its definition
# (emit-everywhere). Always declare it first (the data analogue of a proc
# prototype) so a TU whose copy the merge stage drops still has a valid
# declaration; wrap the definition as a droppable `'d'` unit the merge
# stage assigns to a single owner.
let cname = stripCnifMarks(actualConstName)
var decl = newBuilder("")
decl.addDeclWithVisibility(Extern):
decl.addVar(kind = Local, name = actualConstName, typ = td)
q.s[cfsData].add(extract(decl))
q.s[cfsData].add(cnifDefDirective(cname, "d", icNifName(q, sym)))
q.s[cfsData].add(extract(data))
q.s[cfsData].add(cnifEndDefs())
q.icDataDefs.add (cname, icNifName(q, sym))
else:
q.s[cfsData].add(extract(data))
q.s[cfsData].add(extract(data))
if q.hcrOn:
# generate the global pointer with the real name
q.s[cfsVars].addVar(kind = Global, name = sym.loc.snippet,
@@ -3591,17 +3553,6 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
of skProc, skConverter, skIterator, skFunc:
#if sym.kind == skIterator:
# echo renderTree(sym.getBody, {renderIds})
if p.config.cmd == cmdNifC and
(isGenericRoutineStrict(sym) or sfCompileTime in sym.flags or
(sym.kind == skIterator and sym.typ.callConv == ccInline)):
# Under IC a module's top-level routine definitions are serialized as bare
# symbol references that reappear in the loaded statement list. Uninstantiated
# generic routines (incl. those with type-class params like `tuple`) and
# `.compileTime` routines have no run-time code, so skip them here.
# Inline iterators likewise have no standalone code — they are always inlined
# at their for-loop call sites by the transformer (only closure iterators get
# a standalone C function), so a bare serialized def reference is a no-op.
return
if sfCompileTime in sym.flags:
localError(p.config, n.info, "request to generate code for .compileTime proc: " &
sym.name.s)
@@ -3676,11 +3627,6 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
# echo renderTree(p.prc.ast, {renderIds})
internalError(p.config, n.info, "expr: param not init " & sym.name.s & "_" & $sym.id)
putLocIntoDest(p, d, sym.loc)
of skTemplate, skMacro:
# Under IC a module's top-level template/macro definitions are serialized as
# bare symbol references (only their interface matters), so they reappear in
# the loaded statement list. They are compile-time only and produce no code.
discard
else: internalError(p.config, n.info, "expr(" & $sym.kind & "); unknown symbol")
of nkNilLit:
if not isEmptyType(n.typ):
@@ -3970,13 +3916,6 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Builder)
let elemTyp = skipTypes(t.elementType, abstractRange+{tyOwned}-{tyTypeDesc})
if isOpaqueImportcType(elemTyp):
result.add "{0}"
elif toInt(lengthOrd(p.config, t.indexType)) > broadcastArrayThreshold and
elemTyp.kind in {tyInt..tyUInt64, tyBool, tyChar, tyFloat..tyFloat128,
tyPtr, tyPointer, tyCstring}:
# Large array of a scalar whose default is the zero representation: a single
# C `{0}` zero-fills all `lengthOrd` slots instead of emitting that many
# initializers (keeps huge SSZ-style zero buffers compact in the C output).
result.add "{0}"
else:
var arrInit: StructInitializer
result.addStructInitializer(arrInit, kind = siArray):
@@ -4263,13 +4202,7 @@ proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType; resul
var d: TLoc = initLocExpr(p, n)
result.add rdLoc(d)
of tyArray, tyVarargs:
if isDefaultBroadcastArray(n, p.config):
# Compact zero/null-default array (see `isDefaultBroadcastArray`): the
# whole thing is the null value of every slot, so a single C `{0}`
# zero-fills all `lengthOrd` elements — no need to materialise them.
result.add "{0}"
else:
genConstSimpleList(p, n, isConst, result)
genConstSimpleList(p, n, isConst, result)
of tyTuple:
genConstTuple(p, n, isConst, typ, result)
of tyOpenArray:

View File

@@ -1986,9 +1986,4 @@ proc genStmts(p: BProc, t: PNode) =
if isPush: pushInfoContext(p.config, t.info)
expr(p, t, a)
if isPush: popInfoContext(p.config)
# A bare `nkSym` statement is how IC serializes a definition that lives inside a
# top-level block (e.g. a nested `proc`/`var`): codegen emits the definition and
# leaves the symbol's own location in `a` (e.g. `locProc`), which is discarded
# here, so the value-sanity check below does not apply to it.
internalAssert p.config, t.kind == nkSym or
a.k in {locNone, locTemp, locLocalVar, locExpr}
internalAssert p.config, a.k in {locNone, locTemp, locLocalVar, locExpr}

View File

@@ -39,30 +39,11 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
if isExtern: Extern
elif lfExportLib in s.loc.flags: ExportLibVar
else: Private
if m.config.cmd == cmdNifC and vis == Private and not isExtern:
# A `{.threadvar.}`/`{.global.}` thread-local declared inside a routine is
# emitted by every module that emit-everywhere's its enclosing routine
# (e.g. libp2p's `var keys {.global.}: HashSet`), so its content-addressed
# name collides at link. Same fix as a plain global (genGlobalVarDecl):
# `extern` declaration + a droppable `'d'` definition unit the merge stage
# assigns one owner. The thread-local storage class rides on both.
let cname = stripCnifMarks(s.loc.snippet)
let td = getTypeDesc(m, s.loc.t)
# `extern` declaration via the full `addVar` overload — it knows the
# thread-local storage class (`NIM_THREADVAR`); the simple `addVar`'s
# `addVarHeader` does not implement `Threadvar`.
m.s[cfsVars].addVar(m, s, name = s.loc.snippet, typ = td,
kind = Threadvar, visibility = Extern)
m.s[cfsVars].add(cnifDefDirective(cname, "d", icNifName(m, s)))
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet, typ = td, kind = Threadvar, visibility = vis)
m.s[cfsVars].add(cnifEndDefs())
else:
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet,
typ = getTypeDesc(m, s.loc.t),
kind = Threadvar,
visibility = vis)
m.s[cfsVars].addVar(m, s,
name = s.loc.snippet,
typ = getTypeDesc(m, s.loc.t),
kind = Threadvar,
visibility = vis)
proc generateThreadLocalStorage(m: BModule) =
if m.g.nimtv.buf.len != 0 and (usesThreadVars in m.flags or sfMainModule in m.module.flags):

View File

@@ -72,67 +72,20 @@ proc mangleProc(m: BModule; s: PSym; makeUnique: bool): string =
else:
m.g.mangledPrcs.incl(result)
proc sharedInstanceCName(m: BModule; s: PSym): string =
## The module-free canonical C name for a content-keyed generic instance,
## or "" when the symbol must keep its module-suffixed name. With a shared
## name, every TU that instantiated the same generic with the same type
## arguments calls one extern definition (first claimant's TU embeds it,
## see `genProcLvl3`) instead of compiling its own static copy.
##
## The name is program-unique only if the 30-bit content hash does not
## collide for same-named instances of *different* instantiations across
## modules — the per-module probe in `setInstanceDisamb` cannot see that.
## Claimants therefore must present the same signature; on mismatch the
## later one keeps its module-suffixed name (no merge, still correct).
## Residual risk: same name and signature, different generic args, AND a
## 30-bit collision — vanishingly unlikely; a full-typeKey verification
## channel can close it later.
result = ""
if m.config.cmd == cmdNifC and s.kind in routineKinds and
(s.disamb and InstanceDisambBit) != 0'i32 and
s.typ != nil and s.typ.callConv != ccInline and not m.hcrOn and
{sfImportc, sfExportc, sfCodegenDecl} * s.flags == {}:
# The content-derived `disamb` is unique per process (collision-probed in
# `setInstanceDisamb`), so the mint-site-independent `_i<disamb>` name is
# safe to use directly; identical instances across modules collide on it
# exactly and the merge stage keeps one.
result = s.name.s.mangle & "_i" & $s.disamb
proc isSharedInstanceCName(m: BModule; s: PSym): bool =
m.config.cmd == cmdNifC and s.kind in routineKinds and
(s.disamb and InstanceDisambBit) != 0'i32 and
stripCnifMarks(s.loc.snippet) == s.name.s.mangle & "_i" & $s.disamb
proc fillBackendName(m: BModule; s: PSym) =
if s.loc.snippet == "":
var result: Rope
if s.kind in routineKinds and {optCDebug, optItaniumMangle} * m.g.config.globalOptions == {optCDebug, optItaniumMangle} and
m.g.config.symbolFiles == disabledSf:
# Under the per-module IC backend the bare-name uniqueness probe
# (`m.g.mangledPrcs`) only sees the routines of the CURRENT module, so the
# clean-vs-`makeUnique` decision is made independently per process: a
# method base mangles clean at its owner but loses the in-module race to
# its same-signature dispatcher elsewhere (clean `speak` defined twice ->
# "multiple definition"; demanders call `speak_u<n>` that nobody defines).
# Force the stable, disamb-based unique name so every process agrees.
result = mangleProc(m, s, makeUnique = m.config.cmd == cmdNifC).rope
result = mangleProc(m, s, false).rope
else:
let shared = sharedInstanceCName(m, s)
if shared.len > 0:
result = shared.rope
else:
result = s.name.s.mangle.rope
result.add mangleProcNameExt(m.g.graph, s)
result = s.name.s.mangle.rope
result.add mangleProcNameExt(m.g.graph, s)
if m.hcrOn:
result.add '_'
result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config))
backendEnsureMutable s
if m.config.cmd == cmdNifC:
# mark the name so the cnif artifact writer can turn every occurrence
# into a Symbol token; stripped from the actual C output in genModule
s.locImpl.snippet = markCName(result)
else:
s.locImpl.snippet = result
s.locImpl.snippet = result
proc fillParamName(m: BModule; s: PSym) =
if s.loc.snippet == "":
@@ -420,12 +373,6 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope =
m.typeCache[sig] = result
proc pushType(m: BModule; typ: PType) =
when defined(icDbgRefc):
if typ.kind == tySequence and
typ.elementType.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyGenericParam:
echo "[icRefc] pushType seq-of-genericparam t=", typeToString(typ),
" itemId=", typ.itemId.module, ".", typ.itemId.item, " mod=", m.module.name.s
echo getStackTrace()
for i in 0..high(m.typeStack):
# pointer equality is good enough here:
if m.typeStack[i] == typ: return
@@ -671,18 +618,6 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
for i in 1..<t.n.len:
if t.n[i].kind != nkSym: internalError(m.config, t.n.info, "genProcParams")
var param = t.n[i].sym
# The hidden closure environment param (`:envP`) is not a real C parameter:
# the environment is passed via the trailing `ClE_0` (added below) and
# `closureSetup` materialises `:envP` as a local cast of it. In a from-source
# build `:envP` only lives in the routine's AST params, never in the proc
# *type's* `n`, so it never reaches here. Under IC `closureParams` re-shares
# the AST param node with `typ.n`, so the lifted `:envP` leaks into `t.n`;
# emitting it would produce a bogus extra parameter that collides with the
# `closureSetup` local (the "redeclared as different kind of symbol" / env
# pointer-type mismatch). We still must fill its name/loc (later passes such
# as `assignParam` and `closureSetup` reference it), but it is omitted from
# the C signature to match the from-source ABI.
let isClosureEnv = t.callConv == ccClosure and param.name.s == ":envP"
var descKind = dkParam
if m.config.backend == backendCpp and optByRef in param.options:
if param.typ.kind == tyGenericInst:
@@ -694,7 +629,6 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder,
fillParamName(m, param)
fillLoc(param.locImpl, locParam, t.n[i],
param.paramStorageLoc)
if isClosureEnv: continue # name/loc filled, but not part of the C signature
var typ: Rope
if ccgIntroducedPtr(m.config, param, t.returnType) and descKind == dkParam:
typ = ptrType(getTypeDescWeak(m, param.typ, check, descKind))
@@ -819,11 +753,7 @@ proc genRecordFieldsAux(m: BModule; n: PNode,
# don't use fieldType here because we need the
# tyGenericInst for C++ template support
let noInit = sfNoInit in field.flags or (field.typ.sym != nil and sfNoInit in field.typ.sym.flags)
# Under `nim ic`, object fields are local NIF syms restored without an
# `owner`; `rectype` is the owning record type, so fall back to it rather
# than deref a nil `field.owner`.
let ownerTyp = if field.owner != nil: field.owner.typ else: rectype
if not noInit and (fieldType.isOrHasImportedCppType() or hasCppCtor(m, ownerTyp)):
if not noInit and (fieldType.isOrHasImportedCppType() or hasCppCtor(m, field.owner.typ)):
var didGenTemp = false
initializer = genCppInitializer(m, nil, fieldType, didGenTemp)
result.addField(field, sname, typ, isFlexArray, initializer)
@@ -1178,11 +1108,6 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes
tyUserTypeClass, tyUserTypeClassInst, tyInferred:
result = getTypeDescAux(m, skipModifier(t), check, kind)
else:
when defined(icDbgRefc):
echo "[icRefc] getTypeDescAux ", t.kind, " t=", typeToString(t),
" origTyp=", typeToString(origTyp), " t.itemId=", t.itemId.module, ".", t.itemId.item,
" sym=", (if t.sym != nil: t.sym.name.s else: "nil"),
" owner=", (if t.owner != nil: t.owner.name.s else: "nil")
internalError(m.config, "getTypeDescAux(" & $t.kind & ')')
result = ""
# fixes bug #145:
@@ -1221,10 +1146,6 @@ proc finishTypeDescriptions(m: BModule) =
var check = initIntSet()
while i < m.typeStack.len:
let t = m.typeStack[i]
when defined(icDbgRefc):
echo "[icRefc] finishTypeDescriptions[", i, "] mod=", m.module.name.s,
" t=", typeToString(t), " kind=", t.kind,
" itemId=", t.itemId.module, ".", t.itemId.item
if optSeqDestructors in m.config.globalOptions and t.skipTypes(abstractInst).kind == tySequence:
seqV2ContentType(m, t, check)
else:
@@ -1339,9 +1260,7 @@ proc genProcHeader(m: BModule; prc: PSym; result: var Builder; visibility: var D
elif prc.typ.callConv == ccInline or isNonReloadable(m, prc):
visibility = StaticProc
elif sfImportc notin prc.flags:
if not isSharedInstanceCName(m, prc):
visibility = Private
# else: plain extern — the definition is shared across TUs
visibility = Private
if asPtr:
result.addProcVar(m, prc, name, params, rettype, isStatic = isStaticVar, ignoreAttributes = true)
else:
@@ -1419,24 +1338,8 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
m.hcrCreateTypeInfosProc.addCast(typ = ptrType(CPointer)):
m.hcrCreateTypeInfosProc.add(cAddr(name))
else:
if m.config.cmd == cmdNifC:
# Emit-everywhere (see genTypeInfoV1's perModuleCg gate): every demanding
# `cg` process emits this type info's tentative definition. Declare it
# `extern` first (the data analogue of a proc prototype) so a TU whose copy
# the merge stage drops still has a valid declaration; wrap the definition
# as a droppable `'d'` unit the merge stage assigns to a single owner so
# exactly one external-linkage tentative definition survives (preserving
# the RTTI pointer identity refc relies on).
m.s[cfsStrData].addDeclWithVisibility(Extern):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
m.s[cfsStrData].add(cnifDefDirective(name, "d", icNifName(m, origType)))
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
m.s[cfsStrData].add(cnifEndDefs())
m.icDataDefs.add (name, icNifName(m, origType))
else:
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimType")
proc genTypeInfoAux(m: BModule; typ, origType: PType, name: Rope;
info: TLineInfo) =
@@ -1529,25 +1432,8 @@ proc genObjectFields(m: BModule; typ, origType: PType, n: PNode, expr: Rope;
m.s[cfsTypeInit3].addFieldAssignment(expr, "name", makeCString(field.name.s))
m.s[cfsTypeInit3].addFieldAssignment(expr, "sons", cAddr(subscript(tmp, cIntValue(0))))
m.s[cfsTypeInit3].addFieldAssignment(expr, "len", L)
if m.config.cmd == cmdNifC:
# The discriminator table has a content-addressed name
# (`NimDT_<hashType>_<field>`) and is emitted by every module that demands
# this variant type's RTTI (emit-everywhere; RTTI has no single owner —
# emission is lazy and often skipped). Declare it `extern` + wrap the
# tentative definition as a droppable `'d'` unit so the merge stage keeps
# exactly one external-linkage definition (mirrors the `TNimType` var and
# consts); otherwise the identical name collides across modules at link.
m.s[cfsData].addDeclWithVisibility(Extern):
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].add(cnifDefDirective(tmp, "d", ""))
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].add(cnifEndDefs())
m.icDataDefs.add (tmp, "")
else:
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
m.s[cfsData].addArrayVar(kind = Local, name = tmp,
elementType = ptrType("TNimNode"), len = toInt(L)+1)
for i in 1..<n.len:
var b = n[i] # branch
var tmp2 = getNimNode(m)
@@ -1741,13 +1627,8 @@ proc declareNimType(m: BModule; name: string; str: Rope, module: int) =
m.s[cfsTypeInit1].addArgument(hcrGlobal):
m.s[cfsTypeInit1].add("\"" & str & "\"")
else:
# cnif-mark the name: this extern declaration is the reference the
# def-retention check consults when the defining TU regenerates and
# the typeinfo cannot be re-demanded (type vanished) — the referencing
# TU must lose its reuse then instead of producing a link error
let declName = if m.config.cmd == cmdNifC: markCName(str) else: str
m.s[cfsStrData].addDeclWithVisibility(Extern):
m.s[cfsStrData].addVar(kind = Local, name = declName, typ = nr)
m.s[cfsStrData].addVar(kind = Local, name = str, typ = nr)
proc genTypeInfo2Name(m: BModule; t: PType): Rope =
var it = t
@@ -1811,16 +1692,6 @@ proc generateRttiDestructor(g: ModuleGraph; typ: PType; owner: PSym; kind: TType
incl result.flagsImpl, sfFromGeneric
incl result.flagsImpl, sfGeneratedOp
# Under IC the `rttiDestroy` wrapper is generated independently in every cg
# process that emits `typ`'s RTTI (the type-info is emit-everywhere). A plain
# counter `disamb` renumbers per process, so the RTTI table baked in module A
# references `rttiDestroy_c<n>` while module B (the =destroy owner) defines a
# different number → undefined at link. Give it a content-derived `disamb`
# (stable across processes) + `HookDisambBit`, exactly like `symPrototype` does
# for the hook itself: same `typ` ⇒ same C name everywhere, and the bit makes
# `emitsBodyInThisModule` emit the body in every demander (merge dedups). The
# `"rttiDestroy"` op-name keeps its key disjoint from the real `=destroy` hook's.
setHookDisamb(g, result, "rttiDestroy", typ)
proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp; result: var Builder) =
let theProc = getAttachedOp(m.g.graph, t, op)
@@ -1896,8 +1767,6 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin
cgsym(m, "TNimTypeV2")
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, icNifName(m, origType))
var flags = 0
if not canFormAcycle(m.g.graph, t): flags = flags or 1
@@ -1960,15 +1829,8 @@ proc genTypeInfoV2OldImpl(m: BModule; t, origType: PType, name: Rope; info: TLin
proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineInfo) =
cgsym(m, "TNimTypeV2")
# Under `nim nifc` every `cg` process that demands this type's RTTI emits its
# definition (emit-everywhere). The forward declaration must therefore be a
# real `extern` (not a tentative definition) so a TU whose copy the merge
# stage drops still only *declares* it; the definition itself is wrapped as a
# droppable `'d'` unit below and assigned to a single owner.
m.s[cfsStrData].addDeclWithVisibility(if m.config.cmd == cmdNifC: Extern else: Private):
m.s[cfsStrData].addDeclWithVisibility(Private):
m.s[cfsStrData].addVar(kind = Local, name = name, typ = "TNimTypeV2")
if m.config.cmd == cmdNifC:
m.icDataDefs.add (name, icNifName(m, origType))
var flags = 0
if not canFormAcycle(m.g.graph, t): flags = flags or 1
@@ -2029,12 +1891,7 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn
else:
typeEntry.addField(typeInit, name = "flags"):
typeEntry.addIntValue(flags)
if m.config.cmd == cmdNifC:
m.s[cfsVars].add(cnifDefDirective(name, "d", icNifName(m, origType)))
m.s[cfsVars].add extract(typeEntry)
m.s[cfsVars].add(cnifEndDefs())
else:
m.s[cfsVars].add extract(typeEntry)
m.s[cfsVars].add extract(typeEntry)
if t.kind == tyObject and t.baseClass != nil and optEnableDeepCopy in m.config.globalOptions:
discard genTypeInfoV1(m, t, info)
@@ -2073,13 +1930,7 @@ proc genTypeInfoV2(m: BModule; t: PType; info: TLineInfo): Rope =
m.typeInfoMarkerV2[sig] = result
let owner = t.skipTypes(typedescPtrs).itemId.module
# In the per-module backend (`cg`) RTTI is emit-everywhere like procs and
# consts: every demanding module emits the `'d'` definition (deduped to one
# owner by the merge stage). The owner-routing below would instead push the
# definition into the owner module's *unwritten* backend module (discarded in
# this process) and emit only an extern here, leaving the symbol undefined.
let perModuleCg = m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"
if not perModuleCg and owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
# make sure the type info is created in the owner module
discard genTypeInfoV2(m.g.mods[owner], origType, info)
# reference the type info as extern here
@@ -2146,10 +1997,6 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
let marker = m.g.typeInfoMarker.getOrDefault(sig)
if marker.str != "":
when defined(icDbgRefc):
if "catchableerror" in marker.str:
echo "[icNti] ", marker.str, " in mod=", m.module.name.s,
" -> extern:globalMarker owner=", marker.owner
cgsym(m, "TNimType")
cgsym(m, "TNimNode")
declareNimType(m, "TNimType", marker.str, marker.owner)
@@ -2160,32 +2007,15 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
result = "NTI$1$2_" % [rope(typeToC(t)), rope($sig)]
m.typeInfoMarker[sig] = result
when defined(icDbgRefc):
template dbgNti(branch: string) =
if "catchableerror" in result:
echo "[icNti] ", result, " in mod=", m.module.name.s, " -> ", branch
else:
template dbgNti(branch: string) = discard
let old = m.g.graph.emittedTypeInfo.getOrDefault($result)
if old != FileIndex(0):
dbgNti "extern:emittedTypeInfo"
cgsym(m, "TNimType")
cgsym(m, "TNimNode")
declareNimType(m, "TNimType", result, old.int)
return prefixTI(result)
var owner = t.skipTypes(typedescPtrs).itemId.module
# In the per-module backend (`cg`) V1 RTTI is emit-everywhere like procs,
# consts and V2 type info: every demanding module emits the `'d'` definition
# (deduped to one owner by the merge stage). The owner-routing below would
# instead push the definition into the owner module's *unwritten* backend
# module (discarded in this process) and emit only an extern here, leaving the
# symbol undefined at link — the refc `NTI*` undefined-reference bug. (V2 got
# this gate in 8e0dd4bfb; V1, only reached under `--mm:refc`, was missed.)
let perModuleCg = m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"
if not perModuleCg and owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
dbgNti "extern:ownerRouted"
if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner):
# make sure the type info is created in the owner module
discard genTypeInfoV1(m.g.mods[owner], origType, info)
# reference the type info as extern here
@@ -2196,7 +2026,6 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope =
else:
owner = m.module.position.int32
dbgNti "DEFINED-HERE"
m.g.typeInfoMarker[sig] = (str: result, owner: owner)
#rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result)
@@ -2283,21 +2112,3 @@ proc genTypeSection(m: BModule, n: PNode) =
discard getTypeDescAux(m, s.typ, intSet, descKindFromSymKind(s.kind))
if m.g.generatedHeader != nil:
discard getTypeDescAux(m.g.generatedHeader, s.typ, intSet, descKindFromSymKind(s.kind))
# Unlike genCppInitializer which returns just the braced value list (e.g. "{a, b}"),
# genCppConstructorExpr returns a full type-prefixed expression (e.g. "Foo(a, b)").
# This is used when a standalone construction expression is needed — e.g. on the
# right-hand side of an assignment — whereas genCppInitializer is used in variable
# declarations where the type is already written separately before the initializer.
proc genCppConstructorExpr(m: BModule, prc: BProc; typ: PType; didGenTemp: var bool): Snippet =
var params = ""
if typ.itemId in m.g.graph.initializersPerType:
let call = m.g.graph.initializersPerType[typ.itemId]
if call != nil:
var p = prc
if p == nil:
p = BProc(module: m)
params = genCppParamsForCtor(p, call, didGenTemp)
if prc == nil:
assert p.blocks.len == 0, "BProc belongs to a struct doesnt have blocks"
result = getTypeDesc(m, typ, dkVar) & "(" & params & ")"

View File

@@ -11,7 +11,7 @@
import
ast, types, msgs, wordrecg,
platform, trees, options, cgendata, mangleutils, renderer, modulegraphs
platform, trees, options, cgendata, mangleutils, renderer
import std/[hashes, strutils, formatfloat]
@@ -112,34 +112,10 @@ proc encodeName*(name: string): string =
proc makeUnique(m: BModule; s: PSym, name: string = ""): string =
result = if name == "": s.name.s else: name
# keep backend-minted ids out of the `_u` namespace; their item counter
# restarts at 0 and would collide with loaded symbols' ids
if s.itemId.isBackendMinted:
result.add "_c"
if (s.disamb and HookDisambBit) != 0'i32:
# A backend-minted sym whose `disamb` is content-derived (setHookDisamb gave
# it HookDisambBit) — e.g. the `rttiDestroy` wrapper. Its `itemId.item` is a
# PER-PROCESS backend counter, so using it makes the C name diverge across
# the emit-everywhere processes: the type's RTTI table (emit-everywhere,
# merge-deduped) ends up referencing one process's `_c<item>` while the
# wrapper is defined with another's -> undefined at link (`rttiDestroy_c23`).
# The content-derived disamb is stable across processes, so use it.
result.add $s.disamb
else:
result.add $s.itemId.item
else:
result.add "_u"
# Mirror `mangleProcNameExt`: use the per-(module,name) `disamb`, NOT
# `itemId.item`. Under the per-module IC backend the same symbol is loaded
# from a NIF in many processes and `itemId.item` is a fresh, load-order
# dependent counter — so a method base would mangle to `_u1` in one module,
# `_u3` in another and clean at its owner, none of which link. `disamb` is
# assigned deterministically per (module, name) and is serialized, so every
# process that touches the symbol derives the identical C name.
result.add $s.disamb
# module suffix LAST (a strippable trailing token; see `mangleProcNameExt`)
result.add "__"
result.add m.g.graph.ifaces[s.itemId.module].uniqueName
result.add "_u"
result.add $s.itemId.item
proc encodeSym*(m: BModule; s: PSym; makeUnique: bool = false; extra: string = ""): string =
#Module::Type

View File

@@ -19,10 +19,6 @@ import
mangleutils, cbuilderbase, modulegraphs
from expanddefaults import caseObjDefaultBranch
from ast2nif import globalName, toNifFilename, icNifTypeName
from typekeys import modname
from std/algorithm import sort
import cnif
import pipelineutils
@@ -55,7 +51,7 @@ when not declared(dynlib.libCandidates):
else:
dest.add(s)
when defined(tinyc): # == hasTinyCBackend; spelled out for the IC dep scanner
when options.hasTinyCBackend:
import tccgen
proc hcrOn(m: BModule): bool = m.config.hcrOn
@@ -65,18 +61,9 @@ proc addForwardedProc(m: BModule, prc: PSym) =
m.g.forwardedProcs.add(prc)
proc newModule*(g: BModuleList; module: PSym; conf: ConfigRef; idgen: IdGenerator): BModule
proc getCFile*(m: BModule): AbsoluteFile
proc findPendingModule(m: BModule, s: PSym): BModule =
# TODO fixme
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg":
# Per-module backend codegen: only module M (`m`) is emitted in this
# process, so every demanded definition — whether a normal proc owned by
# another (here unwritten) module or a minted instance/hook — is emitted
# into M's TU. Definitions owned elsewhere are emitted again by their own
# module's cg process; the merge stage keeps one per C name and turns the
# rest into prototypes (which already live in the unmarked protos section).
return m
if m.config.symbolFiles == v2Sf or optCompress in m.config.globalOptions:
let ms = s.itemId.module #getModule(s)
result = m.g.mods[ms]
@@ -84,87 +71,15 @@ proc findPendingModule(m: BModule, s: PSym): BModule =
var ms = getModule(s)
registerModule m.g.graph, ms
if ms.position >= m.g.mods.len:
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms))
else:
result = m.g.mods[ms.position]
if result == nil:
result = newModule(m.g, ms, m.config, idGeneratorForBackend(ms))
result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms))
else:
var ms = getModule(s)
result = m.g.mods[ms.position]
proc icNifName(m: BModule; s: PSym): string =
## The serialized NIF name of `s`, recorded next to its C name in the cnif
## artifact so a later run can re-demand the definition when a reused TU
## still references it (the def-retention check). Backend-minted symbols
## have no NIF name.
if m.config.cmd == cmdNifC and s != nil and not isBackendMinted(s.itemId):
result = globalName(s, m.config)
else:
result = ""
proc icNifName(m: BModule; t: PType): string =
## The type flavor: recorded next to RTTI data definitions so the
## def-retention check can re-demand the typeinfo of a regenerating TU's
## previous artifact (`genTypeInfo` is type-driven, not symbol-driven).
if m.config.cmd == cmdNifC:
result = icNifTypeName(t, m.config)
else:
result = ""
proc emitsBodyInThisModule(m: BModule, prc: PSym): bool =
## Per-module backend codegen is concerned with ONE module: it emits the
## bodies of the routines that module OWNS (its own top-level defs) and only
## *prototypes* a routine owned by another module — that routine's body is
## emitted by its own module's `cg` process, and the merge stage's DCE prunes
## whatever ends up globally dead. The funnel where the main module re-emitted
## its entire transitive closure (≈1.8 GB, a 56 MB `.c.nif`) is exactly this
## rule being absent.
##
## Generic instances and synthesized hooks (`=destroy`, `$`, …) have no single
## owning-module top-level — they are minted on demand — so each demander emits
## them and the merge stage deduplicates by their content-addressed C name.
##
## A NESTED routine is not emitted on its own: it is lambda-lifted and emitted
## as part of its ENCLOSING routine's body, into the same TU. So the decision
## must follow the OUTERMOST enclosing routine (the one directly under the
## module — `skipGenericOwner` stops at a generic *instance*, not its
## originating generic), never the nested symbol's own identity. Otherwise a
## nested proc whose enclosing is a generic instance (content-addressed,
## emitted by every demander) — e.g. nim-serialization's per-field `readField`
## inside the `makeFieldReadersTable[R,W]` instance, whose address fills the
## returned table — is gated out (its own `itemId.module` is the minting module
## and its disamb is a plain counter), so the enclosing's lift degrades it to a
## prototype and its body lands in no TU → undefined at link.
if not (m.config.cmd == cmdNifC and m.config.icBackendStage == "cg"):
return true
# The symbol may ITSELF be content-addressed (a synthesized hook or a generic
# instance carries `Hook/InstanceDisambBit` on its OWN `disamb`): then it has no
# single owning module and every demander emits it (merge dedups by C name),
# regardless of what it is nested under. This must be checked on `prc` directly,
# not on `top`: a `=destroy`/`=sink` lifted while compiling some enclosing proc
# (e.g. system's `isZeroMemory` destroying a `ptr array`) has that PROC as its
# `skipGenericOwner`, so `top` walks up to a plain routine whose own disamb has
# no bit — gating the hook to that routine's owner module, which mints it
# on demand and emits it nowhere → undefined at link.
if (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32:
return true
var top = prc
while top.skipGenericOwner != nil and top.skipGenericOwner.kind != skModule:
top = top.skipGenericOwner
result = top.itemId.module == m.module.position or
(top.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32 or
# An INLINE iterator has no standalone body — it is expanded at each
# call site — so it is materialized in every module that iterates over
# it, never in its owner. A proc nested in one (e.g. std/uri's
# `parseData` inside `iterator decodeQuery`) is lambda-lifted into each
# of those consumer TUs and must be emitted there (its stable
# owner-suffixed name + `'u'` flag let the merge stage keep one); gating
# it to the iterator's owner module leaves it in no TU → undefined.
(top.kind == skIterator and top.typ != nil and
top.typ.callConv != ccClosure)
proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc =
result = TLoc(k: k, storage: s, lode: lode,
snippet: "", flags: flags)
@@ -185,13 +100,9 @@ proc fillLoc(a: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) {.inline.} =
a.storage = s
proc t(a: TLoc): PType {.inline.} =
if a.lode.kind == nkSym and a.lode.sym.typ != nil:
if a.lode.kind == nkSym:
result = a.lode.sym.typ
else:
# Under `nim ic` an object-field reference is a typeless leaf stub (its def
# lives in another seek; see ast2nif `FieldMarker`) that carries its type on
# the NODE instead. Fall back to the node type. Byte-neutral for non-IC, where
# a real sym always has a type.
result = a.lode.typ
proc lodeTyp(t: PType): PNode =
@@ -213,6 +124,8 @@ proc useHeader(m: BModule, sym: PSym) =
proc cgsym(m: BModule, name: string)
proc cgsymValue(m: BModule, name: string): Rope
proc getCFile(m: BModule): AbsoluteFile
proc getModuleDllPath(m: BModule): Rope =
let (dir, name, ext) = splitFile(getCFile(m))
let filename = strutils.`%`(platform.OS[m.g.config.target.targetOS].dllFrmt, [name & ext])
@@ -613,7 +526,7 @@ proc resetLoc(p: BProc, loc: var TLoc) =
if isImportedCppType(typ):
var didGenTemp = false
let rl = rdLoc(loc)
let init = genCppConstructorExpr(p.module, p, typ, didGenTemp)
let init = genCppInitializer(p.module, p, typ, didGenTemp)
p.s(cpsStmts).addAssignment(rl, init)
return
if optSeqDestructors in p.config.globalOptions and typ.kind in {tyString, tySequence}:
@@ -815,31 +728,12 @@ proc genGlobalVarDecl(res: var Builder, p: BProc, n: PNode; td: Snippet;
typ = constType(typ)
if p.hcrOn:
typ = ptrType(typ)
if p.config.cmd == cmdNifC and vis == Private and sfImportc notin s.flags:
# A `{.global.}` var (e.g. chronos's per-call-site `var loc {.global.} =
# SrcLoc(...)`, or a gensym'd `var dummy`/`var topic` with no initializer)
# declared inside a routine is emitted by every module that emit-everywhere's
# its enclosing routine; its content-addressed name then collides at link.
# Declare it `extern` + wrap the definition as a droppable `'d'` unit so the
# merge stage keeps exactly one (like consts / TNimType / the NimDT
# discriminator tables / the threadvar path). This covers no-initializer
# globals too — they collide just the same. A module-level global has a
# single claimant → its sole emitter is the owner merge keeps.
let cname = stripCnifMarks(s.loc.snippet)
res.addDeclWithVisibility(Extern):
res.addVar(kind = Local, name = s.loc.snippet, typ = typ)
res.add(cnifDefDirective(cname, "d", icNifName(p.module, s)))
res.addVar(p.module, s,
name = s.loc.snippet, typ = typ, visibility = vis,
initializer = initializer, initializerKind = initializerKind)
res.add(cnifEndDefs())
else:
res.addVar(p.module, s,
name = s.loc.snippet,
typ = typ,
visibility = vis,
initializer = initializer,
initializerKind = initializerKind)
res.addVar(p.module, s,
name = s.loc.snippet,
typ = typ,
visibility = vis,
initializer = initializer,
initializerKind = initializerKind)
proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
let s = n.sym
@@ -862,9 +756,6 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
useHeader(p.module, s)
if lfNoDecl in s.loc.flags: return
if not containsOrIncl(p.module.declaredThings, s.id):
if p.config.cmd == cmdNifC and sfImportc notin s.flags:
p.module.icDataDefs.add (stripCnifMarks(s.loc.snippet),
icNifName(p.module, s))
if sfThread in s.flags:
declareThreadVar(p.module, s, sfImportc in s.flags)
if value != "":
@@ -896,12 +787,8 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
else:
initializer = value
genGlobalVarDecl(p.module.s[cfsVars], p, n, td, initializer = initializer)
if p.withinLoop > 0 and value == "" and
s.loc.t.skipTypes(abstractInst).kind notin {tyVar, tyLent}:
if p.withinLoop > 0 and value == "":
# fixes tests/run/tzeroarray:
# Don't reset borrowed references (var/lent): the pointer itself is still
# uninitialized here, so resetLoc would dereference garbage. Such variables
# (e.g. the loop var of `mitems`) are always assigned before use anyway.
backendEnsureMutable s
resetLoc(p, s.locImpl)
@@ -1222,17 +1109,8 @@ proc closeNamespaceNim(result: var Builder) =
proc closureSetup(p: BProc, prc: PSym) =
if tfCapturesEnv notin prc.typ.flags: return
# prc.ast[paramsPos].last contains the type we're after — BUT a closure loaded
# from a `.t.bif` (a lambda-lifted nested proc / generic instance the `lower`
# stage transformed) can arrive with an EMPTY AST param node: the lifted hidden
# `:env` param lives in `typ.n`, the authoritative signature (`genProc` already
# reads `typ.n`, not the AST). The two param nodes diverge across the NIF
# boundary; fall back to `typ.n` so the env param resolves instead of indexing
# an empty container.
var params = prc.ast[paramsPos]
if params.safeLen == 0 and prc.typ.n != nil and prc.typ.n.kind == nkFormalParams:
params = prc.typ.n
var ls = lastSon(params)
# prc.ast[paramsPos].last contains the type we're after:
var ls = lastSon(prc.ast[paramsPos])
if ls.kind != nkSym:
internalError(p.config, prc.info, "closure generation failed")
var env = ls.sym
@@ -1438,34 +1316,6 @@ proc genProcBody(p: BProc; procBody: PNode) =
p.blocks[0].sections[cpsInit].addCall(cgsymValue(p.module, "nimErrorFlag"))
proc genProcLvl3*(m: BModule, prc: PSym) =
if m.config.cmd == cmdNifC:
fillBackendName(m, prc)
if (prc.disamb and (InstanceDisambBit or HookDisambBit)) != 0'i32 and
containsOrIncl(m.emittedContentDefs, stripCnifMarks(prc.loc.snippet)):
# A different symbol already emitted a body under this content-addressed
# C name in this TU (same generic instance / hook minted in two source
# modules, both loaded here). Emitting a second body is a C redefinition;
# a prototype was already produced for it, so just stop.
return
if sfDispatcher in prc.flags and sfMainModule notin m.module.flags:
# A method dispatcher enumerates the whole program's method set: its
# body is synthesized by `generateIfMethodDispatchers` only after all
# modules have been generated, and its single definition is emitted
# into the main TU by `finishModule` (main is finished last and never
# reused, so the definition can never go stale inside a cached TU).
# Any demand before that point yields a prototype.
genProcPrototype(m, prc)
return
if prc.itemId.module != m.module.position and
not isBackendMinted(prc.itemId) and
(prc.typ == nil or prc.typ.callConv != ccInline) and
sfDispatcher notin prc.flags:
# this TU embeds a definition whose body lives in another module's
# NIF: record the impl dependency (the artifact's cdeps head) so the
# reuse gate re-checks that module's impl cookie. Inline bodies are
# already part of the iface cookie; dispatcher bodies are synthesized
# from the whole program and live in main, which never reuses.
m.icImplMods.incl prc.itemId.module
var p = newProc(prc, m)
var header = newBuilder("")
let isCppMember = m.config.backend == backendCpp and sfCppMember * prc.flags != {}
@@ -1477,21 +1327,8 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
var returnStmt: Snippet = ""
assert(prc.ast != nil)
# A body LOADED from `.t.bif` was already FULLY lowered by the `lower` stage —
# transformed AND destructor-injected (see nifbackend.generateLowerStage). The
# `.t.bif` is the authoritative backend artifact; re-injecting here would lower
# it twice (double `=destroy` calls) and, worse, re-lift the env hooks per cg
# process (owned by nobody → undefined at link). So inject ONLY when the body
# was re-derived in this process (`wasLoaded == false`). Capture before
# `transformBody`, which returns the cached body (non-nil) when it was loaded.
# ONLY under IC: in a normal `nim c` build `transformedBody` is the ordinary
# transform cache (set whenever `transformBody` already ran for `prc`, e.g. a
# CT-evaluated or earlier-referenced routine), NOT a `.t.bif` load — gating on
# it there would WRONGLY skip destructor injection and miscompile (orc
# decref-on-freed). The `.t.bif`-loaded-body concept exists only under cmdNifC.
let wasLoaded = m.config.cmd == cmdNifC and prc.transformedBody != nil
var procBody = transformBody(m.g.graph, m.idgen, prc, {})
if sfInjectDestructors in prc.flags and not wasLoaded:
if sfInjectDestructors in prc.flags:
procBody = injectDestructorCalls(m.g.graph, m.idgen, prc, procBody)
let tmpInfo = prc.info
@@ -1551,17 +1388,6 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
for i in 1..<prc.typ.n.len:
let param = prc.typ.n[i].sym
if param.typ.isCompileTimeOnly: continue
if prc.typ.callConv == ccClosure and param.name.s == ":envP":
# The hidden closure-env param is materialised by `closureSetup`, never a
# normal C parameter (`genProcParams` omits it from the signature). In a
# from-source build it lives only in the routine's AST params and never in
# `typ.n`, so this loop never reaches it. Under IC `closureParams` leaks it
# into `typ.n`; for a LOADED closure it is already present at header time
# (`genProcParams` fills its loc), but for a RE-DERIVED closure
# (`wasLoaded == false`) `transformBody` appends it only AFTER
# `genProcHeader` ran, so its `loc.snippet` is still empty here. Skip it to
# match the from-source invariant — `closureSetup` assigns its local below.
continue
assignParam(p, param, prc.typ.returnType)
closureSetup(p, prc)
genProcBody(p, procBody)
@@ -1610,37 +1436,7 @@ proc genProcLvl3*(m: BModule, prc: PSym) =
generatedProc.add(extract(p.s(cpsStmts)))
if optStackTrace in prc.options: generatedProc.add(deinitFrame(p))
generatedProc.add(returnStmt)
if m.config.cmd == cmdNifC:
# definition directive for the cnif artifact: groups the proc's text
# under its name and carries the root-relevant flags. The end directive
# right after the text makes the definition self-delimiting, so raw
# cfsProcs emitters (NimMain block, trav markers, ...) never end up
# inside a definition's span.
var defFlags = ""
if sfExportc in prc.flags or sfConstructor in prc.flags: defFlags.add 'x'
if sfCompilerProc in prc.flags: defFlags.add 'c'
if prc.kind == skMethod or sfDispatcher in prc.flags: defFlags.add 'm'
if (prc.typ == nil or prc.typ.callConv != ccInline) and
sfDispatcher notin prc.flags:
# A unique program-wide definition: external linkage, so exactly one
# translation unit may embed its body and everyone else declares it.
# Each module's `cg` process emits the body (emit-everywhere); this flag
# tells the merge stage which definitions to assign a single owner and
# prototype in the rest. The complement — inline procs and method
# dispatchers — is emitted into every using TU (`static`/main-only) and
# must never be deduplicated.
defFlags.add 'u'
if not hasCnifMarks(prc.loc.snippet):
# The C name was not minted through `fillBackendName` (e.g. set by an
# `extern`/`rtl` pragma at sem time), so its uses are invisible to the
# artifact's liveness walk — conservatively keep the definition.
defFlags.add 'x'
m.s[cfsProcs].add(cnifDefDirective(stripCnifMarks(prc.loc.snippet), defFlags,
icNifName(m, prc)))
m.s[cfsProcs].add(extract(generatedProc))
m.s[cfsProcs].add(cnifEndDefs())
else:
m.s[cfsProcs].add(extract(generatedProc))
m.s[cfsProcs].add(extract(generatedProc))
if isReloadable(m, prc):
m.s[cfsDynLibInit].add('\t')
m.s[cfsDynLibInit].addAssignmentWithValue(prc.loc.snippet):
@@ -1663,15 +1459,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
useHeader(m, sym)
if lfNoDecl in sym.loc.flags or sfCppMember * sym.flags != {}: return
if lfDynamicLib in sym.loc.flags:
if m.config.cmd == cmdNifC and m.config.icBackendStage == "cg":
# Under IC per-module cg every demander emits the dynlib proc's DEFINITION
# locally (findPendingModule returns `m`, so symInDynamicLib follows this
# call and the merge stage keeps one def per C name). Emitting the
# cross-module `extern` proto here would register `sym.id` in
# `m.declaredThings` and thereby make that `symInDynamicLib` skip, leaving
# the `Dl_*` symbol declared-but-never-defined -> undefined at link.
discard "definition emitted by symInDynamicLib"
elif sym.itemId.module != m.module.position and
if sym.itemId.module != m.module.position and
not containsOrIncl(m.declaredThings, sym.id):
let vis = if isReloadable(m, sym): StaticProc else: Extern
let name = mangleDynLibProc(sym)
@@ -1694,15 +1482,10 @@ proc genProcPrototype(m: BModule, sym: PSym) =
var header = newBuilder("")
var visibility: DeclVisibility = None
genProcHeader(m, sym, header, visibility, asPtr = asPtr, addAttributes = true)
# A prototype is not a *use*: strip the cnif name marks so the artifact's
# liveness walk does not see every forward-declared proc as referenced.
var headerText = extract(header)
if m.config.cmd == cmdNifC:
headerText = stripCnifMarks(headerText)
if asPtr:
m.s[cfsProcHeaders].addDeclWithVisibility(visibility):
# genProcHeader would give variable declaration, add it directly
m.s[cfsProcHeaders].add(headerText)
m.s[cfsProcHeaders].add(extract(header))
else:
let extraVis =
if sym.typ.callConv != ccInline and requiresExternC(m, sym):
@@ -1711,7 +1494,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
None
m.s[cfsProcHeaders].addDeclWithVisibility(extraVis):
m.s[cfsProcHeaders].addDeclWithVisibility(visibility):
m.s[cfsProcHeaders].add(headerText)
m.s[cfsProcHeaders].add(extract(header))
m.s[cfsProcHeaders].finishProcHeaderAsProto()
include inliner
@@ -1789,8 +1572,7 @@ proc genProcLvl2(m: BModule, prc: PSym) =
# which will actually become a function pointer
if isReloadable(m, prc):
genProcPrototype(q, prc)
if emitsBodyInThisModule(m, prc):
genProcLvl3(q, prc)
genProcLvl3(q, prc)
else:
fillProcLoc(m, prc.ast[namePos])
useHeader(m, prc)
@@ -1800,7 +1582,7 @@ proc requestConstImpl(p: BProc, sym: PSym) =
if genConstSetup(p, sym):
let m = p.module
# declare implementation:
let q = findPendingModule(m, sym)
var q = findPendingModule(m, sym)
if q != nil and not containsOrIncl(q.declaredThings, sym.id):
assert q.initProc.module == q
genConstDefinition(q, p, sym)
@@ -1824,12 +1606,6 @@ proc genProc(m: BModule, prc: PSym) =
if not containsOrIncl(m.g.generatedHeader.declaredThings, prc.id):
genProcLvl3(m.g.generatedHeader, prc)
proc requestProcDef*(m: BModule, prc: PSym) =
## Public demand entry: request `prc`'s definition; it is routed to the
## module that owns it and generated once, exactly as if some generated
## code had referenced it.
genProc(m, prc)
proc genVarPrototype(m: BModule, n: PNode) =
#assert(sfGlobal in sym.flags)
let sym = n.sym
@@ -1899,7 +1675,7 @@ proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope =
## Returns a mangled module name.
result = mangleModuleName(conf, filename).mangle
proc getSomeNameForModule*(m: BModule): Rope =
proc getSomeNameForModule(m: BModule): Rope =
## Returns a mangled module name.
assert m.module.kind == skModule
assert m.module.owner.kind == skPackage
@@ -2286,40 +2062,6 @@ proc registerModuleToMain(g: BModuleList; m: BModule) =
else:
g.otherModsInit.addCallStmt(init)
proc registerReusedModuleToMain*(g: BModuleList; m: BModule;
initRequired, datInitRequired: bool) =
## `registerModuleToMain` for a module whose cached translation unit is
## reused: the init/datInit presence comes from the artifact's meta head
## instead of the (never generated) sections. Mirrors the non-hcr path of
## `registerModuleToMain` — reuse is disabled when hcr is on.
let
init = m.getInitName
datInit = m.getDatInitName
if datInitRequired:
g.mainModProcs.addDeclWithVisibility(Private):
g.mainModProcs.addProcHeader(ccNimCall, datInit, CVoid, cProcParams())
g.mainModProcs.finishProcHeaderAsProto()
g.mainDatInit.addCallStmt(datInit)
if sfSystemModule in m.module.flags:
if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
g.mainDatInit.addCallStmt(cgsymValue(m, "initThreadVarsEmulation"))
if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcAtomicArc, gcOrc, gcYrc}:
g.mainDatInit.addCallStmt(cgsymValue(m, "initStackBottomWith"),
cCast(CPointer, cAddr("inner")))
if initRequired:
g.mainModProcs.addDeclWithVisibility(Private):
g.mainModProcs.addProcHeader(ccNimCall, init, CVoid, cProcParams())
g.mainModProcs.finishProcHeaderAsProto()
if sfMainModule in m.module.flags:
g.mainModInit.addCallStmt(init)
elif sfSystemModule in m.module.flags:
g.mainDatInit.addCallStmt(init) # systemInit right after systemDatInit
else:
g.otherModsInit.addCallStmt(init)
proc genDatInitCode(m: BModule) =
## this function is called in cgenWriteModules after all modules are closed,
## it means raising dependency on the symbols is too late as it will not propagate
@@ -2567,16 +2309,6 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
moduleIsEmpty = false
res.add(extract(m.s[i]))
# what `registerModuleToMain` will announce for this module; recorded in
# the artifact's meta head so a later run can reuse the TU
let initRequired = m.s[cfsInitProc].buf.len > 0
let datInitRequired = m.s[cfsDatInitProc].buf.len > 0
if m.config.cmd == cmdNifC:
# close the definitions section: the init procs that follow belong to
# the artifact's top level (always-run code, hence liveness roots)
res.add(cnifEndDefs())
if m.s[cfsInitProc].buf.len > 0:
moduleIsEmpty = false
res.add(extract(m.s[cfsInitProc]))
@@ -2599,22 +2331,6 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
postprocessCode(m.config, result)
if m.config.cmd == cmdNifC and result.len > 0:
let artifact = cfile.cname.string & ".nif"
var implDeps: seq[string] = @[]
for pos in m.icImplMods.items:
if pos != m.module.position:
implDeps.add modname(pos, m.config)
sort implDeps
writeCnifArtifact(result, artifact, initRequired, datInitRequired,
m.icDataDefs,
semmedNif = toNifFilename(m.config, FileIndex m.module.position),
moduleBase = getSomeNameForModule(m),
implDeps = implDeps)
m.g.graph.icCnifFiles.add artifact
# NB: under cmdNifC the returned text still carries the cnif marks; the
# caller renders it (dropping dead definitions) or strips it.
proc initProcOptions(m: BModule): TOptions =
let opts = m.config.options
if sfSystemModule in m.module.flags: opts-{optStackTrace} else: opts
@@ -2626,8 +2342,6 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
result.headerFiles = @[]
result.declaredThings = initIntSet()
result.declaredProtos = initIntSet()
result.emittedContentDefs = initHashSet[string]()
result.icImplMods = initIntSet()
result.cfilename = filename
result.filename = filename
result.typeCache = initTable[SigHash, Rope]()
@@ -2699,13 +2413,10 @@ proc writeHeader(m: BModule) =
result.finishProcHeaderAsProto()
if m.config.cppCustomNamespace.len > 0: closeNamespaceNim(result)
result.addf("#endif /* $1 */$n", [guard])
var headerText = extract(result)
if m.config.cmd == cmdNifC:
headerText = stripCnifMarks(headerText)
if not writeRope(headerText, m.filename):
if not writeRope(extract(result), m.filename):
rawMessage(m.config, errCannotOpenFile, m.filename.string)
proc getCFile*(m: BModule): AbsoluteFile =
proc getCFile(m: BModule): AbsoluteFile =
let ext =
if m.compileToCpp: ".nim.cpp"
elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
@@ -2799,9 +2510,8 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool =
rawMessage(m.config, errCannotOpenFile, cfile.cname.string)
result = true
proc genModuleCode(m: BModule; cf: var Cfile): string =
## First half of `writeModule`: finalizes the module and produces its code
## text. Under cmdNifC the text still carries the cnif marks.
proc writeModule(m: BModule) =
let cfile = getCFile(m)
if moduleHasChanged(m.g.graph, m.module):
genInitCode(m)
@@ -2816,11 +2526,9 @@ proc genModuleCode(m: BModule; cf: var Cfile): string =
m.s[cfsProcHeaders].add(extract(m.g.mainModProcs))
generateThreadVarsSize(m)
result = genModule(m, cf)
proc registerModuleCode(m: BModule; cf: var Cfile; code: string) =
## Second half of `writeModule`: writes the .c file if it changed and
## registers it for compilation.
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
var code = genModule(m, cf)
if code != "" or m.config.symbolFiles != disabledSf:
when hasTinyCBackend:
if m.config.cmd == cmdTcc:
@@ -2830,15 +2538,6 @@ proc registerModuleCode(m: BModule; cf: var Cfile; code: string) =
if not shouldRecompile(m, code, cf): cf.flags = {CfileFlag.Cached}
addFileToCompile(m.config, cf)
proc writeModule(m: BModule) =
let cfile = getCFile(m)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
var code = genModuleCode(m, cf)
if m.config.cmd == cmdNifC:
code = stripCnifMarks(code)
registerModuleCode(m, cf, code)
proc updateCachedModule(m: BModule) =
let cfile = getCFile(m)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
@@ -2924,12 +2623,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
if m.g.forwardedProcs.len == 0:
incl m.flags, objHasKidsValid
if m.config.cmd == cmdNifC:
# nifbackend synthesizes the dispatchers between the module loop
# and the finish loop (emitMethodDispatchers): TUs demand-created
# by the dispatcher bodies must still reach `modulesClosed`
discard
elif optMultiMethods in m.g.config.globalOptions or
if optMultiMethods in m.g.config.globalOptions or
m.g.config.selectedGC notin {gcArc, gcOrc, gcAtomicArc, gcYrc} or
vtables notin m.g.config.features:
generateIfMethodDispatchers(graph, m.idgen)
@@ -2943,8 +2637,9 @@ proc genForwardedProcs(g: BModuleList) =
# a second pass here
# Note: ``genProcLvl2`` may add to ``forwardedProcs``
while g.forwardedProcs.len > 0:
let prc = g.forwardedProcs.pop()
let m = g.mods[prc.itemId.module]
let
prc = g.forwardedProcs.pop()
m = g.mods[prc.itemId.module]
if sfForward in prc.flags:
internalError(m.config, prc.info, "still forwarded: " & prc.name.s)
@@ -2959,32 +2654,7 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) =
# order anyway)
genForwardedProcs(g)
if config.cmd == cmdNifC and not isDefined(config, "icNoCDce"):
# Two-phase write: produce every module's marked text and artifact
# first, then compute global liveness over the artifacts and render
# the .c files with dead definitions dropped. Demand-driven codegen
# over-approximates (it cannot retract a definition once some path
# requested it); this is where the surplus is removed.
var mods: seq[BModule] = @[]
var cfs: seq[Cfile] = @[]
var codes: seq[string] = @[]
for m in cgenModules(g):
let cfile = getCFile(m)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(m.config, toObjFile(m.config, cfile)), flags: {})
let code = genModuleCode(m, cf)
mods.add m
cfs.add cf
codes.add code
let cl = computeLiveFromCArtifacts(g.graph.icCnifFiles)
var dropped = 0
for i in 0..<mods.len:
let rendered =
if cl.broken: stripCnifMarks(codes[i])
else: renderMarkedC(codes[i], cl.live, dropped)
registerModuleCode(mods[i], cfs[i], rendered)
else:
for m in cgenModules(g):
m.writeModule()
for m in cgenModules(g):
m.writeModule()
writeMapping(config, g.mapping)
if g.generatedHeader != nil: writeHeader(g.generatedHeader)

View File

@@ -158,12 +158,6 @@ type
forwTypeCache*: TypeCache # cache for forward declarations of types
declaredThings*: IntSet # things we have declared in this .c file
declaredProtos*: IntSet # prototypes we have declared in this .c file
emittedContentDefs*: HashSet[string]
# cmdNifC per-module backend: content-addressed C names (generic
# instances and synthesized hooks) whose body this TU already emitted.
# Distinct symbols (minted in different source modules) can share one
# `_i<disamb>` name; `declaredThings` keys on symbol id and lets the
# second one through, so we dedup the body by name here instead.
queue*: seq[PSym] # queue of procs to generate
alive*: IntSet # symbol IDs of alive data as computed by `dce.nim`
headerFiles*: seq[string] # needed headers to include
@@ -182,17 +176,6 @@ type
extensionLoaders*: array['0'..'9', Builder] # special procs for the
# OpenGL wrapper
sigConflicts*: CountTable[SigHash]
icImplMods*: IntSet # module ids whose routine BODIES this TU
# embeds (redirected defs, shared instances,
# hooks); recorded as the artifact's cdeps so
# the reuse gate can check their impl cookies
icDataDefs*: seq[tuple[cname, nifname: string]]
# C names of data definitions (consts, globals,
# RTTI) this TU embeds plus their NIF symbol
# names (empty for RTTI, which has no symbol);
# recorded in the cnif artifact so a later run
# can reuse the TU and re-demand definitions
# that cached TUs still reference
g*: BModuleList
template config*(m: BModule): ConfigRef = m.g.config

View File

@@ -160,17 +160,7 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
var witness: PSym = nil
if s.typ.firstParamType.owner.getModule != s.getModule and vtables in g.config.features and not
g.config.isDefined("nimInternalNonVtablesTesting") and sfFromGeneric notin s.flags:
# `sfFromGeneric` excepted: this is the same-module restriction for vtable
# slot placement, and it must be judged on the GENERIC method, not on an
# instance. The generic `method skip[T](x: Input[T])` never reaches here
# (`semMethodPrototype` registers generic methods via `addMethodToGeneric`,
# bypassing `methodDef`); only its instance `skip[string]` does, and that
# instance's first-param type `Input[string]` is owned by whichever module
# first instantiated it (`tparsecombnum`, which `import parsecomb`s and uses
# it), NOT by `Input[T]`'s defining module — so the comparison spuriously
# fails for a method that is perfectly legal at the generic level. (Concrete
# methods, `sfFromGeneric notin flags`, are still checked.)
g.config.isDefined("nimInternalNonVtablesTesting"):
localError(g.config, s.info, errGenerated, "method `" & s.name.s &
"` can be defined only in the same module with its type (" & s.typ.firstParamType.typeToString() & ")")
if sfImportc in s.flags:
@@ -190,7 +180,6 @@ proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
g.methods[i].methods[0] != s:
# already exists due to forwarding definition?
localError(g.config, s.info, "method is not a base")
logMethodDef(g, s)
return
of No: discard
of Invalid:
@@ -202,7 +191,6 @@ proc methodDef*(g: ModuleGraph; idgen: IdGenerator; s: PSym) =
else:
g.bucketTable.inc(s.typ.firstParamType.skipTypes(skipPtrs).itemId)
g.methods.add((methods: @[s], dispatcher: createDispatcher(s, g, idgen)))
logMethodDef(g, s)
#echo "adding ", s.info
if witness != nil:
localError(g.config, s.info, "invalid declaration order; cannot attach '" & s.name.s &

View File

@@ -167,8 +167,6 @@ type
curExcSym: PSym # Current exception
externExcSym: PSym # Extern exception: what would getCurrentException() return outside of closure iter
enclosingPragmas: seq[PNode] # stack of pragma blocks wrapping stmtlist
states: seq[State] # The resulting states. Label is int literal.
finallyPathStack: seq[FinallyTarget] # Stack of split blocks, whiles and finallies
stateLoopLabel: PSym # Label to break on, when jumping between states.
@@ -254,8 +252,7 @@ proc newCurExcAccess(ctx: var Ctx): PNode =
ctx.newEnvVarAccess(ctx.curExcSym)
proc newStateLabel(ctx: Ctx): PNode =
result = nkIntLit.newIntNode(0)
result.typ = getSysType(ctx.g, TLineInfo(), tyInt16)
ctx.g.newIntLit(TLineInfo(), 0)
proc newState(ctx: var Ctx, n: PNode, inlinable: bool, label: PNode): PNode =
# Creates a new state, adds it to the context
@@ -336,14 +333,9 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
var cond: PNode = nil
for i in 0..<c.len - 1:
assert(c[i].kind == nkType)
# Use the :curExc env field (set by the wrapper before entering the
# except landing state) instead of calling getCurrentException():
# injectdestructors does not process the args of this raw generic
# `of` magic call, so an owning getCurrentException() temp would
# never be destroyed and the caught exception would leak (#23615).
let nextCond = newTreeIT(nkCall, c.info, ctx.g.getSysType(c.info, tyBool),
newSymNode(g.getSysMagic(c.info, "of", mOf)),
ctx.newCurExcAccess(),
g.callCodegenProc("getCurrentException"),
c[i])
cond = if cond.isNil: nextCond
@@ -600,7 +592,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
let branch = n[i]
case branch.kind
of nkExceptBranch:
branch[^1] = ctx.convertExprBodyToAsgn(branch[^1], tmp)
if branch[0].kind == nkType:
branch[1] = ctx.convertExprBodyToAsgn(branch[1], tmp)
else:
branch[0] = ctx.convertExprBodyToAsgn(branch[0], tmp)
of nkFinally:
discard
else:
@@ -990,14 +985,9 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
for j in i + 1..<n.len:
s.add(n[j])
var body = s
for pragma in ctx.enclosingPragmas:
body = newTreeI(nkPragmaBlock, n[i + 1].info,
pragma[0].copyTree, body)
n.sons.setLen(i + 1)
discard ctx.newState(body, true, label)
if ctx.transformClosureIteratorBody(body, gotoOut) != body:
discard ctx.newState(s, true, label)
if ctx.transformClosureIteratorBody(s, gotoOut) != s:
internalError(ctx.g.config, "transformClosureIteratorBody != s")
break
else:
@@ -1135,14 +1125,6 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
finallyBody = ctx.transformClosureIteratorBody(finallyBody, finallyExit)
dec ctx.curFinallyLevel
of nkPragmaBlock:
# Propagate the pragma blocks so that blocks like {.cast(uncheckedAssign).}
# remain effective
ctx.enclosingPragmas.add(n)
n[1] = ctx.transformClosureIteratorBody(n[1], gotoOut)
discard ctx.enclosingPragmas.pop()
result = n
of nkGotoState, nkForStmt:
internalError(ctx.g.config, "closure iter " & $n.kind)

View File

@@ -53,8 +53,7 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: ConfigRef;
graph: ModuleGraph): bool =
if self.suggestMode:
conf.setCmd cmdCheck
conf.ideActive = true
conf.setCmd cmdIdeTools
if conf.cmd == cmdNimscript:
incl(conf.globalOptions, optWasNimscript)
loadConfigs(DefaultConfig, cache, conf, graph.idgen) # load all config files

View File

@@ -1,726 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## The "cnif" artifact: the C code generator's output as a NIF file.
##
## This is deliberately *not* NIFC: the C text is kept verbatim (Nim's
## C-level machinery — exception handling in particular — is more refined
## than what NIFC models today; the gap can be closed incrementally later).
## The only structure the artifact adds is the part dead code elimination
## and generic-instance merging need:
##
## - raw C text as string literals
## - every *global* entity's C name as a `Symbol` token
## - every emitted proc definition as a `(cdef SymbolDef flags ...)` group
##
## The C generator marks names with control characters at the single place
## a global's C name is minted (`fillBackendName`) and emits a definition
## directive at the single place finished procs are appended; the marks then
## ride through all of the snippet composition untouched. This module turns
## the final marked module text into the `.c.nif` artifact and strips the
## marks for the actual `.c` output. Rendering C from the artifact is a
## plain token walk: string literals verbatim, symbols by name — which is
## also where a later merge step redirects losing generic instances.
##
## Marker scheme (cannot collide: C string literals escape control chars,
## and `\1`/`\31`/`\23` of cgen's postprocess directives are distinct):
## \2 name \3 a global's C name
## \4 name \31 flags \31 nif \5 start of the definition of `name`;
## `nif` is the defining symbol's NIF name
## (empty for backend-minted symbols) so a
## later run can re-demand the definition
## \4 \5 end of the definitions section
import std / [tables, sets, os, assertions, syncio, algorithm]
import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse]
const
CnifSymStart* = '\2'
CnifSymEnd* = '\3'
CnifDefStart* = '\4'
CnifDefSep* = '\31' # same separator char as cgen's postprocess directives
CnifDefEnd* = '\5'
proc markCName*(name: string): string {.inline.} =
CnifSymStart & name & CnifSymEnd
proc hasCnifMarks*(s: string): bool =
for c in s:
if c in {CnifSymStart, CnifSymEnd, CnifDefStart}: return true
false
proc stripCnifMarks*(s: string): string =
## Removes the symbol marks (keeping the names) and the definition
## directives (entirely) so the result is plain C.
if not hasCnifMarks(s): return s
result = newStringOfCap(s.len)
var i = 0
while i < s.len:
case s[i]
of CnifSymStart, CnifSymEnd:
inc i
of CnifDefStart:
while i < s.len and s[i] != CnifDefEnd: inc i
inc i # skip CnifDefEnd
else:
result.add s[i]
inc i
const
CnifVersion* = "4"
## Artifact format version, stored in the meta head. Artifacts written
## by an older compiler lack the NIF names and the cref group the
## def-retention check needs (v2), the cdeps group the fine-grained
## reuse gate needs (v3), or the type NIF names and cnif-marked extern
## RTTI references the typeinfo flavor of the def-retention check
## needs (v4); `readCnifHeads` reports them as invalid so their TUs
## simply regenerate once.
proc cnifDefDirective*(name, flags, nifName: string): string =
CnifDefStart & name & CnifDefSep & flags & CnifDefSep & nifName & CnifDefEnd
proc cnifEndDefs*(): string =
CnifDefStart & CnifDefEnd
proc writeCnifArtifact*(code: string; outfile: string;
initRequired = false; datInitRequired = false;
dataDefs: openArray[tuple[cname, nifname: string]] = [];
semmedNif = ""; moduleBase = "";
implDeps: openArray[string] = []) =
## Splits the marked module text into the `.c.nif` artifact.
## The artifact starts with a `(meta <flags> "semmedNif" "moduleBase"
## "version")` head — whether the module has an init/datInit proc
## ('i'/'d'), which semmed NIF it was generated from and the module's
## mangled base name (what `registerModuleToMain` and the reuse decision
## need when the TU is reused in a later run, possibly without the module
## ever being loaded again) — a `(cdata (SymbolDef StrLit)*)` group naming
## the data definitions (consts, globals, RTTI) the TU embeds together
## with their NIF names, a `(cref Ident*)` group naming every C name
## the TU references but does not define itself (what the def-retention
## check consults when some *other* TU regenerates), and a
## `(cdeps Ident*)` group naming the modules whose routine *bodies* this
## TU embeds (redirected defs, shared instances, hooks): the fine-grained
## reuse gate checks their `.impl.nif` cookies on top of the direct
## imports' `.iface.nif` cookies.
# pre-pass: every marked name is a use, every definition directive (and
# every data def) is a definition; external references = uses - defs
var uses = initHashSet[string]()
var defs = initHashSet[string]()
block prePass:
var i = 0
while i < code.len:
case code[i]
of CnifSymStart:
inc i
var name = ""
while i < code.len and code[i] != CnifSymEnd:
name.add code[i]
inc i
inc i
uses.incl name
of CnifDefStart:
inc i
var payload = ""
while i < code.len and code[i] != CnifDefEnd:
payload.add code[i]
inc i
inc i
let sep = find(payload, CnifDefSep)
if sep > 0: defs.incl payload[0..<sep]
elif payload.len > 0: defs.incl payload
else:
inc i
for d in dataDefs: defs.incl d.cname
var crefs: seq[string] = @[]
for u in uses:
if u notin defs: crefs.add u
sort crefs
var b = nifbuilder.open(outfile)
b.withTree "stmts":
b.withTree "meta":
var metaFlags = ""
if initRequired: metaFlags.add 'i'
if datInitRequired: metaFlags.add 'd'
if metaFlags.len > 0: b.addIdent metaFlags
else: b.addEmpty
b.addStrLit semmedNif
b.addStrLit moduleBase
b.addStrLit CnifVersion
b.withTree "cdata":
for d in dataDefs:
b.addSymbolDef d.cname
b.addStrLit d.nifname
b.withTree "cref":
for r in crefs:
b.addIdent r
b.withTree "cdeps":
for s in implDeps:
b.addIdent s
var raw = ""
var inDef = false
template flushRaw() =
if raw.len > 0:
b.addStrLit raw
raw.setLen 0
var i = 0
while i < code.len:
case code[i]
of CnifSymStart:
flushRaw()
inc i
var name = ""
while i < code.len and code[i] != CnifSymEnd:
name.add code[i]
inc i
inc i # skip CnifSymEnd
b.addSymbol name, ""
of CnifDefStart:
flushRaw()
inc i
var payload = ""
while i < code.len and code[i] != CnifDefEnd:
payload.add code[i]
inc i
inc i # skip CnifDefEnd
if inDef:
b.endTree()
inDef = false
if payload.len > 0:
let sep = find(payload, CnifDefSep)
let name = if sep >= 0: payload[0..<sep] else: payload
var flags = if sep >= 0: payload[sep+1..^1] else: ""
var nifName = ""
let sep2 = find(flags, CnifDefSep)
if sep2 >= 0:
nifName = flags[sep2+1..^1]
flags = flags[0..<sep2]
b.addTree "cdef"
b.addSymbolDef name
if flags.len > 0: b.addIdent flags
else: b.addEmpty
b.addStrLit nifName
inDef = true
else:
raw.add code[i]
inc i
flushRaw()
if inDef:
b.endTree()
b.close()
proc renderMarkedC*(code: string; live: HashSet[string]; dropped: var int): string =
## Renders the final C text from the marked module text: symbol marks are
## removed (keeping the names — a later merge step substitutes them here),
## and definitions whose name is not in `live` are dropped entirely. Each
## definition is self-delimiting (genProcAux emits an end directive right
## after the proc's text), so text written by other emitters is never part
## of a definition's span and survives unconditionally.
result = newStringOfCap(code.len)
var i = 0
while i < code.len:
case code[i]
of CnifSymStart, CnifSymEnd:
inc i
of CnifDefStart:
var payload = ""
inc i
while i < code.len and code[i] != CnifDefEnd:
payload.add code[i]
inc i
inc i # skip CnifDefEnd
if payload.len > 0:
let sep = find(payload, CnifDefSep)
let name = if sep >= 0: payload[0..<sep] else: payload
if name notin live:
inc dropped
# drop the definition's text: everything up to its end directive
while i < code.len and code[i] != CnifDefStart: inc i
else:
result.add code[i]
inc i
# ---- Liveness over the artifact -------------------------------------------
proc symOrIdentName(c: Cursor): string {.inline.} =
if c.kind == Ident: strVal(c) else: symName(c)
type
CnifHeads* = object
## The cheap-to-parse part of an artifact that a later run needs in
## order to reuse the TU without regenerating it.
valid*: bool ## file parsed, carries the meta head and has
## the current format version
initRequired*: bool
datInitRequired*: bool
semmedNif*: string ## the semmed NIF this TU was generated from
moduleBase*: string ## the module's mangled base name
cdefs*: seq[tuple[cname, nifname: string]] ## the proc definitions
cdata*: seq[tuple[cname, nifname: string]] ## the data definitions
crefs*: seq[string] ## C names referenced but not defined here
cdeps*: seq[string] ## module suffixes whose routine bodies this
## TU embeds (impl-cookie gated on reuse)
proc readCnifHeads*(f: string): CnifHeads =
## Reads `(meta ...)`, `(cdata ...)`, `(cref ...)` and the `(cdef ...)`
## head names from an artifact. Artifacts written by an older compiler
## (no meta head or a different format version) report `valid=false`.
result = CnifHeads()
if not fileExists(f): return
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
let cdataTag = tags.registerTag("cdata")
let crefTag = tags.registerTag("cref")
let cdepsTag = tags.registerTag("cdeps")
let metaTag = tags.registerTag("meta")
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return
var version = ""
var sawMeta = false
c.loopInto:
if c.kind == TagLit:
if c.cursorTagId == metaTag:
sawMeta = true
var strIdx = 0
c.loopInto:
if c.kind == Ident:
for ch in strVal(c):
if ch == 'i': result.initRequired = true
elif ch == 'd': result.datInitRequired = true
inc c
elif c.kind == StrLit:
if strIdx == 0: result.semmedNif = strVal(c)
elif strIdx == 1: result.moduleBase = strVal(c)
elif strIdx == 2: version = strVal(c)
inc strIdx
inc c
else:
skip c
elif c.cursorTagId == cdataTag:
c.loopInto:
if c.kind == SymbolDef:
result.cdata.add (symName(c), "")
inc c
elif c.kind == StrLit:
if result.cdata.len > 0:
result.cdata[^1].nifname = strVal(c)
inc c
else:
skip c
elif c.cursorTagId == crefTag:
c.loopInto:
if c.kind in {Ident, Symbol, SymbolDef}:
result.crefs.add symOrIdentName(c)
inc c
else:
skip c
elif c.cursorTagId == cdepsTag:
c.loopInto:
if c.kind in {Ident, Symbol, SymbolDef}:
result.cdeps.add symOrIdentName(c)
inc c
else:
skip c
elif c.cursorTagId == cdefTag:
# fixed head: SymbolDef, flags (Ident or empty), NIF name StrLit;
# everything after that is the definition's body text
var state = 0
c.loopInto:
if c.kind == SymbolDef:
result.cdefs.add (symName(c), "")
state = 1
inc c
elif state == 1: # the flags field
state = 2
skip c
elif state == 2: # the NIF name
if c.kind == StrLit and result.cdefs.len > 0:
result.cdefs[^1].nifname = strVal(c)
state = 3
skip c
else:
skip c
else:
skip c
else:
skip c
endRead(c)
result.valid = sawMeta and version == CnifVersion
type
CnifLiveness* = object
defs*: int ## proc definitions emitted across all modules
liveDefs*: int ## of those, reachable from the roots
live*: HashSet[string] ## live C names
broken*: bool
proc computeLiveFromCArtifacts*(files: openArray[string]): CnifLiveness =
## dce1-style mark&sweep over the C-shaped artifacts: a `(cdef ...)`
## group is a definition (flags 'x'/'c'/'m' — exportc, compilerproc,
## method/dispatcher — make it a root), names at the top level (data,
## globals, init code) are roots, names inside a group are its uses.
## Because the artifact is *fully lowered* output, no conservative
## modelling is needed: every call the C code contains is a token here.
##
## NB: mangled C names contain no dots, so NIF's text reader classifies
## them as `Ident` rather than `Symbol`; the dialect therefore treats
## Ident tokens as name uses. Inside a `(cdef ...)` the flags ident is
## the one immediately following the SymbolDef; everything after is a use.
result = CnifLiveness(live: initHashSet[string]())
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
let cdataTag = tags.registerTag("cdata")
let crefTag = tags.registerTag("cref")
let cdepsTag = tags.registerTag("cdeps")
let metaTag = tags.registerTag("meta")
var uses = initTable[string, HashSet[string]]()
var roots = initHashSet[string]()
var defs = initHashSet[string]()
for f in files:
if not fileExists(f):
result.broken = true
return
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
result.broken = true
endRead(c)
return
c.loopInto:
case c.kind
of Symbol, Ident:
roots.incl symOrIdentName(c)
inc c
of TagLit:
if c.cursorTagId == metaTag or c.cursorTagId == cdataTag or
c.cursorTagId == crefTag or c.cursorTagId == cdepsTag:
# bookkeeping for TU reuse, irrelevant for liveness
skip c
elif c.cursorTagId == cdefTag:
var owner = ""
var flagsSeen = false
c.loopInto:
case c.kind
of SymbolDef:
owner = symName(c)
defs.incl owner
flagsSeen = false
inc c
of Symbol, Ident:
let name = symOrIdentName(c)
if not flagsSeen:
# the flags field right after the SymbolDef
flagsSeen = true
for ch in name:
# 'd' marks a data definition (const/RTTI): never DCE'd, so it
# is a root whose body keeps its referenced procs live
if ch in {'x', 'c', 'm', 'd'}:
roots.incl owner
break
else:
uses.mgetOrPut(owner, initHashSet[string]()).incl name
inc c
of DotToken:
flagsSeen = true # empty flags field
inc c
else:
skip c
else:
c.loopInto:
if c.kind in {Symbol, Ident}:
roots.incl symOrIdentName(c)
inc c
else:
skip c
else:
skip c
endRead(c)
# mark & sweep
var work = newSeqOfCap[string](roots.len)
for r in roots: work.add r
while work.len > 0:
let s = work.pop()
if not result.live.containsOrIncl(s):
if uses.hasKey(s):
for dep in uses[s]:
if dep notin result.live:
work.add dep
result.defs = defs.len
for d in defs:
if d in result.live: inc result.liveDefs
# ---- The merge stage: liveness + owner assignment -------------------------
type
MergeDecision* = object
## What the per-module backend's `merge` stage computes from every
## module's `.c.nif` and what its `emit` stage consumes to render the
## final `.c` of one module.
live*: HashSet[string] ## globally reachable C names (dead cdefs
## are dropped from every module)
owners*: Table[string, string] ## for each `'u'`-flagged (unique,
## externally-linked) definition, the single
## artifact base name allowed to embed its
## body; every other module prototypes it
broken*: bool ## an artifact was missing or unparsable —
## the caller should fall back / regenerate
defs*, liveDefs*: int
proc computeMergeDecision*(files: openArray[string]): MergeDecision =
## One pass over every `.c.nif`: the same mark&sweep as
## `computeLiveFromCArtifacts` plus, per definition, owner assignment.
##
## Each `cg` process emits the body of every definition it demands
## (emit-everywhere), so the same externally-linked definition appears in
## several artifacts. A `'u'` flag on the `(cdef ...)` marks those that need
## exactly one owner, assigned here across processes: the owner is the
## lexicographically smallest artifact that emits it — a pure function of the
## claimant set, hence stable across rebuilds. Definitions without `'u'`
## (inline procs, dispatchers) are `static`/main-only and emitted into every
## using TU, so they get no owner entry and are never deduplicated.
result = MergeDecision(live: initHashSet[string](),
owners: initTable[string, string]())
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
let cdataTag = tags.registerTag("cdata")
let crefTag = tags.registerTag("cref")
let cdepsTag = tags.registerTag("cdeps")
let metaTag = tags.registerTag("meta")
var uses = initTable[string, HashSet[string]]()
var roots = initHashSet[string]()
var defs = initHashSet[string]()
for f in files:
if not fileExists(f):
result.broken = true
return
let owner = extractFilename(f)
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
result.broken = true
endRead(c)
return
c.loopInto:
case c.kind
of Symbol, Ident:
roots.incl symOrIdentName(c)
inc c
of TagLit:
if c.cursorTagId == metaTag or c.cursorTagId == cdataTag or
c.cursorTagId == crefTag or c.cursorTagId == cdepsTag:
skip c
elif c.cursorTagId == cdefTag:
var ownerName = ""
var flagsSeen = false
var needsOwner = false
c.loopInto:
case c.kind
of SymbolDef:
ownerName = symName(c)
defs.incl ownerName
flagsSeen = false
inc c
of Symbol, Ident:
let name = symOrIdentName(c)
if not flagsSeen:
flagsSeen = true
for ch in name:
if ch in {'x', 'c', 'm'}: roots.incl ownerName
# 'u' = unique proc (DCE'd), 'd' = data (never DCE'd, hence a
# root); both need a single owner across the emit-everywhere
# processes
elif ch == 'u': needsOwner = true
elif ch == 'd':
needsOwner = true
roots.incl ownerName
else:
uses.mgetOrPut(ownerName, initHashSet[string]()).incl name
inc c
of DotToken:
flagsSeen = true # empty flags field
inc c
else:
skip c
if needsOwner and ownerName.len > 0:
# smallest claimant wins; ties impossible (one entry per name)
let prev = result.owners.getOrDefault(ownerName, "")
if prev.len == 0 or owner < prev:
result.owners[ownerName] = owner
else:
c.loopInto:
if c.kind in {Symbol, Ident}:
roots.incl symOrIdentName(c)
inc c
else:
skip c
else:
skip c
endRead(c)
var work = newSeqOfCap[string](roots.len)
for r in roots: work.add r
while work.len > 0:
let s = work.pop()
if not result.live.containsOrIncl(s):
if uses.hasKey(s):
for dep in uses[s]:
if dep notin result.live:
work.add dep
result.defs = defs.len
for d in defs:
if d in result.live: inc result.liveDefs
const MergeDecisionFile* = "ic.backend.merge.nif"
## Fixed name of the merge stage's output in the nimcache, read by `emit`.
proc writeMergeDecision*(outfile: string; d: MergeDecision) =
## Serializes the merge decision: `(merge (live Symbol*) (owners (own
## Symbol StrLit)*))`. C names are mangled (no dots) so they serialize as
## symbols; owner artifact base names go in string literals.
var live: seq[string] = @[]
for n in d.live: live.add n
sort live
var keys: seq[string] = @[]
for k in d.owners.keys: keys.add k
sort keys
var b = nifbuilder.open(outfile)
b.withTree "merge":
b.withTree "live":
for n in live: b.addSymbol n, ""
b.withTree "owners":
for k in keys:
b.withTree "own":
b.addSymbol k, ""
b.addStrLit d.owners[k]
b.close()
proc readMergeDecision*(f: string): MergeDecision =
## Reads back a `writeMergeDecision` file; `broken=true` if absent/unparsable.
result = MergeDecision(live: initHashSet[string](),
owners: initTable[string, string]())
if not fileExists(f):
result.broken = true
return
var pool = newPool()
var tags = newTagPool()
let mergeTag = tags.registerTag("merge")
let liveTag = tags.registerTag("live")
let ownersTag = tags.registerTag("owners")
let ownTag = tags.registerTag("own")
var buf = parseFromFile(f, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != mergeTag:
result.broken = true
endRead(c)
return
c.loopInto:
if c.kind == TagLit and c.cursorTagId == liveTag:
c.loopInto:
if c.kind in {Symbol, Ident}:
result.live.incl symOrIdentName(c)
inc c
else:
skip c
elif c.kind == TagLit and c.cursorTagId == ownersTag:
c.loopInto:
if c.kind == TagLit and c.cursorTagId == ownTag:
var key = ""
c.loopInto:
if c.kind in {Symbol, Ident}:
key = symOrIdentName(c)
inc c
elif c.kind == StrLit:
if key.len > 0: result.owners[key] = strVal(c)
inc c
else:
skip c
else:
skip c
else:
skip c
endRead(c)
proc renderCFromArtifact*(artifact: string; d: MergeDecision; ownerId: string;
dropped: var int): string =
## The per-module backend's `emit` stage: render one module's final `.c` from
## its `.c.nif` and the merge decision. String literals are emitted verbatim,
## symbols by name; a `(cdef ...)` body is dropped when the name is dead, or
## when it is a `'u'` unique definition this module does not own. The body's
## prototype lives in the surrounding raw text (cgen emits a forward
## declaration for every *used* proc, independent of where the body lands), so
## a dropped body still leaves a valid declaration — no synthesis needed. The
## head groups (meta/cdata/cref/cdeps) carry no C text.
result = ""
if not fileExists(artifact): return
var pool = newPool()
var tags = newTagPool()
let stmtsTag = tags.registerTag("stmts")
let cdefTag = tags.registerTag("cdef")
var buf = parseFromFile(artifact, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return
c.loopInto:
case c.kind
of StrLit:
result.add strVal(c)
inc c
of Symbol, Ident:
result.add symOrIdentName(c)
inc c
of TagLit:
if c.cursorTagId == cdefTag:
# fixed head: SymbolDef, flags (Ident or empty), nifname StrLit; the
# rest is the definition's body text. `state` counts past the head.
var name = ""
var isUnique = false
var isData = false
var keep = true
var state = 0
c.loopInto:
if state == 0 and c.kind == SymbolDef:
name = symName(c)
state = 1
inc c
elif state == 1: # the flags field (one token: Ident/Symbol or empty)
if c.kind in {Ident, Symbol}:
for ch in symOrIdentName(c):
if ch == 'u': isUnique = true
elif ch == 'd': isData = true
state = 2
inc c
elif state == 2: # the NIF name (one StrLit) — decide keep here
let owned = d.owners.getOrDefault(name, ownerId) == ownerId
keep =
if isData: owned # data: kept by its owner only
elif isUnique: (name in d.live) and owned
else: name in d.live # inline/dispatcher: per-TU
if not keep: inc dropped
state = 3
inc c
else: # body tokens
if keep:
if c.kind == StrLit: result.add strVal(c)
elif c.kind in {Symbol, Ident}: result.add symOrIdentName(c)
inc c
else:
# head groups (meta/cdata/cref/cdeps) carry no C text
skip c
else:
inc c
endRead(c)

View File

@@ -24,7 +24,7 @@ bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
bootSwitch(usedGoGC, defined(gogc), "--gc:go")
bootSwitch(usedNoGC, defined(nogc), "--gc:none")
import std/[setutils, sets, os, strutils, parseutils, parseopt, sequtils, strtabs, enumutils]
import std/[setutils, os, strutils, parseutils, parseopt, sequtils, strtabs, enumutils]
import
msgs, options, nversion, condsyms, extccomp, platform,
wordrecg, nimblecmd, lineinfos, pathutils
@@ -508,8 +508,6 @@ proc parseCommand*(command: string): Command =
of "jsonscript": cmdJsonscript
of "nifc": cmdNifC # generate C from NIF files
of "ic": cmdIc # generate .build.nif for nifmake
of "icconfig": cmdIcConfig # produce the precompiled config artifact
of "track": cmdTrack # IDE goto-def / find-usages over `nim ic`'s NIF output
else: cmdUnknown
proc setCmd*(conf: ConfigRef, cmd: Command) =
@@ -655,18 +653,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
conf: ConfigRef) =
var key = ""
var val = ""
# Record config-file switches so the `nim ic` driver can serialise them into a
# precompiled-config artifact and have its per-module child processes replay
# them instead of re-parsing the `nim.cfg` chain (and re-running `config.nims`
# in the VM) on every invocation. Only `passPP` (config-file) switches are
# captured; command-line switches are forwarded by the build graph as usual.
# Path-search switches are skipped: their net effect already lives in the
# resolved `searchPaths` the driver forwards as `--path`, and replaying their
# raw (often relative-to-config-dir) arguments here would misresolve.
if pass == passPP and switch.normalize notin
["path", "p", "nimblepath", "lazypath", "excludepath",
"nonimblepath", "clearnimblepath", "nimcache"]:
conf.icConfigSwitches.add (switch, arg)
case switch.normalize
of "eval":
expectArg(conf, switch, arg, pass, info)
@@ -719,14 +705,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
of "usenimcache":
processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info)
of "ideimports":
# nimsuggest: where the import closure comes from. IC is opt-in.
# nif|on load unchanged imports from precompiled NIF (cmdM)
# source|off (default) recompile the whole closure from source (cmdCheck)
case arg.normalize
of "nif", "on", "": conf.ideImportsFromNif = true
of "source", "off": conf.ideImportsFromNif = false
else: localError(conf, info, "'--ideImports' expects 'nif' or 'source', got: '$1'" % arg)
of "docseesrcurl":
expectArg(conf, switch, arg, pass, info)
conf.docSeeSrcUrl = arg
@@ -826,8 +804,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
localError(conf, info, "expected nim|cpp but found " & arg)
of "compress":
conf.globalOptions.incl optCompress
of "genbif":
processOnOffSwitchG(conf, {optGenBif}, arg, pass, info)
of "g": # alias for --debugger:native
conf.globalOptions.incl optCDebug
conf.options.incl optLineDir
@@ -947,49 +923,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
of "noimportdoc":
processOnOffSwitchG(conf, {optNoImportdoc}, arg, pass, info)
of "ismainmodule":
# `nim m` (IC) only: marks the single module being checked as the program's
# real entry point so that `isMainModule` and `when isMainModule:` resolve
# correctly even though every module is compiled with `sfMainModule` set.
conf.isMainModule = switchOn(arg)
of "icgroup":
# `nim m` only: register a module that belongs to the current strongly-
# connected import group, so it is compiled from source (not loaded from a
# precompiled NIF) and gets its own NIF written. `deps.nim` emits one
# `--icGroup:<path>` per member of a dependency cycle. The argument is an
# absolute .nim path produced by the dependency scanner.
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icGroup.incl(canonicalizePath(conf, AbsoluteFile arg).string)
of "icproject":
# `nim m`/`nim nifc` only: the ORIGINAL project file (see options.icProject)
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icProject = canonicalizePath(conf, AbsoluteFile arg).string
of "icpreparsedconfig":
# `nim m`/`nim nifc` only: path of the precompiled-config artifact (see
# options.icPreparsedConfig). Read in `passCmd1`, before `loadConfigs`, so
# config loading can replay it instead of re-parsing the `nim.cfg` chain.
expectArg(conf, switch, arg, pass, info)
conf.icPreparsedConfig = arg
of "icconfigout":
# `nim icconfig` only: where to write the precompiled config artifact (see
# options.icConfigOut). The `nim ic` driver spawns the producer with this.
expectArg(conf, switch, arg, pass, info)
conf.icConfigOut = arg
of "icbackendstage":
# `nim nifc` only: per-module backend stage, one of cg|merge|emit (see
# options.icBackendStage). Empty (switch unused) keeps the whole-program
# backend. Emitted by `deps.nim`'s backend build file.
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icBackendStage = arg
of "icbackendmodule":
# `nim nifc` only: the NIF module suffix the cg/emit stage operates on (see
# options.icBackendModule).
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
conf.icBackendModule = arg
of "import":
expectArg(conf, switch, arg, pass, info)
if pass in {passCmd2, passPP}:
@@ -1327,16 +1260,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
# support UNIX style filenames everywhere for portable build scripts:
if config.projectName.len == 0:
config.projectName = unixToNativePath(p.key)
if config.cmd == cmdTrack:
# `nim track PROJ --def:...`: unlike a normal command (where everything
# after the project file is passed to the compiled program), `track`
# accepts its IDE-query switches AFTER the project — the natural,
# nimsuggest-like invocation form. So don't swallow the rest of the line
# into `arguments`; keep parsing the remaining tokens as switches.
result = false
else:
config.arguments = cmdLineRest(p)
result = true
config.arguments = cmdLineRest(p)
result = true
else:
result = false
inc argsCount

File diff suppressed because it is too large Load Diff

View File

@@ -48,7 +48,6 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph; idgen: IdGener
n[resultPos] = newSymNode(res)
result.ast = n
incl result.flagsImpl, {sfFromGeneric, sfNeverRaises}
setHookDisamb(g, result, "$enumtostr", t)
proc searchObjCaseImpl(obj: PNode; field: PSym): PNode =
case obj.kind

View File

@@ -1473,8 +1473,6 @@ proc genFlags*(s: set[TNodeFlag]; dest: var string) =
of nfSkipFieldChecking: dest.add "s0"
of nfDisabledOpenSym: dest.add "d3"
of nfLazyType: dest.add "l1"
of nfLazyBody: discard # process-local placeholder; never serialized
of nfBroadcast: dest.add "v"
proc parse*(t: typedesc[TNodeFlag]; s: string): set[TNodeFlag] =
@@ -1535,7 +1533,6 @@ proc parse*(t: typedesc[TNodeFlag]; s: string): set[TNodeFlag] =
inc i
else: result.incl nfSem
of 't': result.incl nfTransf
of 'v': result.incl nfBroadcast
of 'w': result.incl nfFirstWrite
else: discard
inc i

View File

@@ -19,11 +19,8 @@ import std/tables
when defined(nimPreviewSlimSystem):
import std/assertions
proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) =
## `list` is an `nkStmtList` of `nkReplayAction` nodes (macro-cache puts/incs/
## adds/incls and a few pragmas) recorded for `module`. Under the NIF backend a
## loaded module's `ast` is never reconstructed, so the caller passes the replay
## actions it parsed out of the module's NIF directly.
proc replayStateChanges*(module: PSym; g: ModuleGraph) =
let list = module.ast
assert list != nil
assert list.kind == nkStmtList
for n in list:
@@ -67,9 +64,8 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) =
g.cacheTables[destKey] = initBTree[string, PNode]()
if not contains(g.cacheTables[destKey], key):
g.cacheTables[destKey].add(key, val)
# else: the same key was already replayed. Under IC the import closure is
# replayed (direct module + transitive deps), so the same registration can
# legitimately be reached twice; re-applying it is a no-op, not an error.
else:
internalError(g.config, n.info, "key already exists: " & key)
of "incl":
let destKey = n[1].strVal
let val = n[2]
@@ -90,37 +86,3 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) =
g.cacheSeqs[destKey].add val
else:
internalAssert g.config, false
proc replayBackendActions*(g: ModuleGraph; module: PSym; list: PNode) =
## Applies the backend-relevant replay actions (C compile/link directives)
## found in a NIF-loaded module's top-level statement list. The `nifc`
## backend loads modules without going through sem's `replayStateChanges`,
## so e.g. math's `{.passL: "-lm".}` was lost and the final link failed
## with undefined references. VM cache actions are deliberately NOT
## replayed here — codegen does not run macros.
if list == nil: return
for n in list:
if n.kind == nkReplayAction and n.len >= 2 and
n[0].kind == nkStrLit and n[1].kind == nkStrLit:
case n[0].strVal
of "compile":
if n.len == 4 and n[2].kind == nkStrLit:
let cname = AbsoluteFile n[1].strVal
var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
obj: AbsoluteFile n[2].strVal,
flags: {CfileFlag.External},
customArgs: n[3].strVal)
extccomp.addExternalFileToCompile(g.config, cf)
of "link":
extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal)
of "passl":
extccomp.addLinkOption(g.config, n[1].strVal)
of "passc":
extccomp.addCompileOption(g.config, n[1].strVal)
of "localpassc":
extccomp.addLocalCompileOption(g.config, n[1].strVal,
toFullPathConsiderDirty(g.config, module.info.fileIndex))
of "cppdefine":
options.cppDefine(g.config, n[1].strVal)
else:
discard

View File

@@ -1,292 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Precompiled config for the incremental compiler (`nim ic`).
##
## `nim ic` builds the program by spawning one `nim m` child per module (or
## strongly-connected import group) plus a final `nim nifc`. Each child is a
## full Nim process, so each would normally re-read the whole `nim.cfg` chain
## *and* re-run `config.nims` through the VM — work that is identical for every
## child and, because of the VM run, far from free. With ~85 modules in the
## compiler itself that config work is paid ~85 times during `koch bootic`.
##
## The fix mirrors Nimony's `.cfg.nif`: the driver parses config once, records
## the net effect, and the children replay it. Every config-file switch funnels
## through `processSwitch(..., passPP, ...)` (`nimconf.parseAssignment` and the
## `switch()` callback in `scriptconfig`), so the recorded sequence of those
## switches, replayed in order, reproduces an identical `ConfigRef` without any
## file read or VM run. The one config side effect that does not go through
## `processSwitch` is `cppDefine` (it mutates `conf.cppDefines` directly), so the
## resolved set is serialised alongside.
##
## Path-search switches are deliberately excluded from the recording (see
## `commands.processSwitch`): their resolved result already lives in
## `conf.searchPaths`, which the driver forwards to every child as absolute
## `--path` arguments; replaying their raw, config-dir-relative arguments here
## would misresolve.
import options, commands, lineinfos, pathutils, msgs
import std/[algorithm, os, sets, osproc, times, streams, syncio]
import "../dist/nimony/src/lib" / [nifbuilder, nifcoreparse]
const
IcConfigVersion* = "2"
## Artifact format version. Bump on any layout change here so a child built
## by an older compiler rejects a stale artifact and falls back to normal
## config loading instead of replaying a format it cannot parse.
proc writeIcConfig*(conf: ConfigRef; outfile: string) =
## Serialise the resolved config (the config-file switches recorded during
## `loadConfigs`, the resolved `cppDefines`/`searchPaths`, the nimcache dir, and
## the list of config *source* files for staleness detection) into `outfile`.
## `OnlyIfChanged`: when the content is byte-identical to what is already on
## disk the file is left untouched so its mtime does not advance — otherwise
## every `nim ic` run would re-fire the whole nifmake graph (see `nifler`'s
## `produceConfig`, whose model this mirrors).
var b = nifbuilder.open(outfile, writeMode = OnlyIfChanged)
b.withTree "stmts":
b.withTree "meta":
b.addStrLit IcConfigVersion
b.withTree "sources":
# Every config file read while loading (nim.cfg chain + config.nims), so a
# later run can decide via mtimes whether this artifact is still current
# (see `sourcesChanged`).
for f in conf.configFiles:
b.addStrLit f.string
b.withTree "nimcache":
# Resolved build nimcache. Recorded (unlike the path-search switches) so the
# driver, which replays this artifact instead of parsing `nim.cfg`, still
# learns a `--nimcache:` set inside `nim.cfg` and builds in the right place.
b.addStrLit conf.nimcacheDir.string
b.withTree "cppdefines":
# HashSet iteration order is unspecified; sort so the artifact is
# byte-stable across runs (nifmake keys rebuilds off content changes).
var defs: seq[string] = @[]
for d in conf.cppDefines: defs.add d
sort defs
for d in defs: b.addStrLit d
b.withTree "searchpaths":
# The resolved (absolute) search paths. Path-search *switches* are skipped
# below because their raw arguments are config-dir-relative; the net effect
# lives here instead, so a replayer with no `--path` command-line arguments
# (the `nim ic` driver itself) still resolves imports. `nim m`/`nim nifc`
# children also receive these as forwarded `--path` args; the dedup on
# replay makes the overlap harmless.
for p in conf.searchPaths:
b.addStrLit p.string
b.withTree "switches":
for sw in conf.icConfigSwitches:
b.addTree "sw"
b.addStrLit sw.switch
b.addStrLit sw.arg
b.endTree()
b.close()
proc applyIcConfig*(conf: ConfigRef; infile: string): bool =
## Replay the precompiled config into `conf`. Returns false (and applies
## nothing meaningful) when the artifact is missing or written by a compiler
## with an incompatible format version, so the caller can fall back to reading
## the config files normally.
if not fileExists(infile): return false
var pool = newPool()
var tags = newTagPool()
let
stmtsTag = tags.registerTag("stmts")
metaTag = tags.registerTag("meta")
sourcesTag = tags.registerTag("sources")
nimcacheTag = tags.registerTag("nimcache")
cppTag = tags.registerTag("cppdefines")
pathsTag = tags.registerTag("searchpaths")
switchesTag = tags.registerTag("switches")
swTag = tags.registerTag("sw")
var buf = parseFromFile(infile, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return false
var version = ""
var sawMeta = false
let info = unknownLineInfo
c.loopInto:
if c.kind == TagLit:
if c.cursorTagId == metaTag:
sawMeta = true
c.loopInto:
if c.kind == StrLit:
version = strVal(c)
inc c
else:
skip c
elif c.cursorTagId == nimcacheTag:
c.loopInto:
if c.kind == StrLit:
let nc = strVal(c)
# Only when nimcache was not already pinned on the command line: a
# `--nimcache:` argument the driver/child was launched with must win
# over whatever `nim.cfg` recorded into the artifact.
if nc.len > 0 and conf.nimcacheDir.isEmpty:
conf.nimcacheDir = AbsoluteDir(nc)
inc c
else:
skip c
elif c.cursorTagId == sourcesTag:
# Replay does not need the source list; it exists only for
# `sourcesChanged`. Skip the whole section.
skip c
elif c.cursorTagId == cppTag:
c.loopInto:
if c.kind == StrLit:
cppDefine(conf, strVal(c))
inc c
else:
skip c
elif c.cursorTagId == pathsTag:
c.loopInto:
if c.kind == StrLit:
# Append preserving the serialised order (which already reflects the
# driver's addPath insert-at-front sequence), deduping against any
# path a child already received via a forwarded `--path` argument.
let d = AbsoluteDir(strVal(c))
if not conf.searchPaths.contains(d): conf.searchPaths.add d
inc c
else:
skip c
elif c.cursorTagId == switchesTag:
c.loopInto:
if c.kind == TagLit and c.cursorTagId == swTag:
var sw = ""
var arg = ""
var idx = 0
c.loopInto:
if c.kind == StrLit:
if idx == 0: sw = strVal(c)
else: arg = strVal(c)
inc idx
inc c
else:
skip c
processSwitch(sw, arg, passPP, info, conf)
else:
skip c
else:
skip c
else:
skip c
endRead(c)
result = sawMeta and version == IcConfigVersion
proc sourcesChanged*(configFile: string): bool =
## True when the precompiled config at `configFile` is missing, malformed,
## written by an incompatible version, or any recorded config *source* file is
## newer than it (or has vanished) — i.e. the artifact must be regenerated.
## Mirrors nifler's `sourcesChanged`: the source list lives inside the artifact
## so this needs no out-of-band knowledge of which `nim.cfg`s were read.
if not fileExists(configFile): return true
let modtime = getLastModificationTime(configFile)
var pool = newPool()
var tags = newTagPool()
let
stmtsTag = tags.registerTag("stmts")
metaTag = tags.registerTag("meta")
sourcesTag = tags.registerTag("sources")
var buf = parseFromFile(configFile, 1000, pool, tags)
var c = beginRead(buf)
if c.kind != TagLit or c.cursorTagId != stmtsTag:
endRead(c)
return true
var version = ""
var depsChanged = false
c.loopInto:
if c.kind == TagLit and c.cursorTagId == metaTag:
c.loopInto:
if c.kind == StrLit:
version = strVal(c)
inc c
else:
skip c
elif c.kind == TagLit and c.cursorTagId == sourcesTag:
c.loopInto:
if c.kind == StrLit:
let dep = strVal(c)
if not fileExists(dep) or getLastModificationTime(dep) >= modtime:
depsChanged = true
inc c
else:
skip c
else:
skip c
endRead(c)
result = depsChanged or version != IcConfigVersion
proc produceIcConfig*(conf: ConfigRef) =
## The `cmdIcConfig` command. By the time it runs, the normal pipeline has
## already fully parsed the `nim.cfg` chain and run `config.nims`, so the
## resolved config is sitting in `conf`; just serialise it to `--o`.
let outPath = conf.icConfigOut
if outPath.len == 0:
rawMessage(conf, errGenerated, "icconfig: missing output path (--icConfigOut)")
return
createDir(parentDir(outPath))
writeIcConfig(conf, outPath)
proc ensureIcConfig*(conf: ConfigRef) =
## Driver-side (`cmdIc`). Make sure an up-to-date precompiled config exists,
## (re)producing it in a *separate* process when missing or stale, then point
## `conf.icPreparsedConfig` at it so the driver replays the very same config its
## `nim m`/`nim nifc` children will — perfect speed (config parsed at most once,
## skipped entirely when nothing changed) and consistency (one producer, every
## process replays its output). The artifact lives in the nimcache derived from
## the command line (pre-config-parse), which is the one the children are told;
## a `--nimcache:` set inside `nim.cfg` is recovered from the artifact itself.
let cacheDir = getNimcacheDir(conf).string
# Start from a clean cache when the on-disk NIF format stamp is absent or stale
# (see `icFormatVersion`). This must happen HERE, before the config artifact is
# produced — `commandIc` performs the same check later, but by then the artifact
# would already live in the cache and the wipe would delete it.
createDir(cacheDir)
let versionFile = cacheDir / "ic.version"
let stamp = if fileExists(versionFile): readFile(versionFile) else: ""
if stamp != icFormatVersion:
removeDir(cacheDir)
createDir(cacheDir)
writeFile(versionFile, icFormatVersion)
let outPath = cacheDir / "ic_config.cfg.nif"
if not fileExists(outPath) or sourcesChanged(outPath):
createDir(cacheDir)
# Re-invoke ourselves as the config producer: reuse this process's command
# line, dropping the command argument (`ic`/`track`) in favour of `icconfig`
# and the explicit output path. Every switch must land BEFORE the project
# file, because anything after the project is swallowed into
# `config.arguments` by `cmdLineRest` (and a non-empty `arguments` without
# `--run` is a hard error). Callers may legitimately put switches after the
# project — `nim track PROJ --def:...` — so we re-order rather than replay
# verbatim: all `-`-prefixed switches first (in encounter order), then the
# non-switch project token(s). The producer re-reads `nim.cfg` itself.
var pargs = @["icconfig", "--icConfigOut:" & outPath]
var rest: seq[string] = @[]
var droppedCmd = false
for a in commandLineParams():
if a.len == 0: continue
if a[0] == '-':
pargs.add a
elif not droppedCmd:
droppedCmd = true # drop the original command token (`ic`/`track`)
else:
rest.add a # project file (and any further non-switch tokens) go last
for a in rest: pargs.add a
let p = startProcess(getAppFilename(), args = pargs,
options = {poStdErrToStdOut})
let outp = p.outputStream.readAll()
let code = p.waitForExit()
p.close()
if code != 0 or not fileExists(outPath):
rawMessage(conf, errGenerated,
"failed to produce precompiled config (exit code " & $code & "):\n" & outp)
return
conf.icPreparsedConfig = outPath

View File

@@ -1,55 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## Nim's OWN module-suffix, replacing nimony's `gear2/modnames.moduleSuffix`.
##
## nimony's version hashes a path made RELATIVE to `getCurrentDir()` (or the
## shortest search-path-relative form), so the produced suffix depends on the
## current working directory AND the searchPath set. Under `nim ic` the
## DISCOVERY pass (`deps.nim`, in the driver process) and the COMPILE pass
## (`nifgen`/`typekeys`, in a child `nim m` process) can run with different CWDs
## or `--path` sets, so the SAME file hashes to two different suffixes: e.g.
## `std/staticos` became `sta5rk8sn1` at discovery but `sta4c0qxk` at compile, so
## every importer waited forever for a `.s.bif` that was actually written under
## the other name — a cold `nim ic` build (of anything pulling in `std/os`, whose
## `oscommon` does `from std/staticos import PathComponent`) never converged.
##
## Hashing the CANONICAL ABSOLUTE path makes the suffix a pure function of the
## file, identical across every process and call site. The base-name prefix +
## base-36 `uhash` layout is kept byte-for-byte compatible with the old scheme so
## nothing but the hashed string changes.
import std/os
import "../dist/nimony/src/lib" / tinyhashes
const
PrefixLen = 3 # keep it short: the suffix ends up in every mangled C name
Base36 = "0123456789abcdefghijklmnopqrstuvwxyz"
proc moduleSuffix*(path: string; searchPaths: openArray[string]): string =
## `searchPaths` is accepted for signature-compatibility with the replaced
## `modnames.moduleSuffix` but is deliberately IGNORED — the suffix must not
## depend on the search-path set or the CWD (see the module doc).
# Absolute inputs (the norm at every call site: `toFullPath`/`projectFull`)
# pass straight through `normalizedPath` with no `getCurrentDir` involvement;
# a stray relative path is made absolute against the CWD only as a fallback.
var f = path
if not isAbsolute(f):
try: f = absolutePath(f)
except CatchableError: discard
f = normalizedPath(f)
let m = splitFile(f).name
var id = uhash(f)
result = newStringOfCap(10)
for i in 0 ..< min(m.len, PrefixLen):
result.add m[i]
# base-36 of the hash, low digit first (order is irrelevant for identity).
while id > 0'u32:
result.add Base36[int(id mod 36'u32)]
id = id div 36'u32

View File

@@ -1,252 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## nifcore-based IC serialization helpers — Stage 1 of porting the IC backend
## from the old `nifstreams`/`nifcursors` NIF stack to `nifcore` (see
## `doc/ic_nifcore_port.md`).
##
## It hosts:
## * the process-wide shared `Pool`/`TagPool` that stands in for the old global
## `nifstreams.pool`,
## * `writeFileStable`, the content-stable file writer mirroring
## `nifcursors.writeFile(..., OnlyIfChanged)`,
## * the first ported writer (`writeSemDeps`), used as the migration spike.
##
## No `nifstreams`/`nifcursors` types cross this module's boundary: callers pass
## plain Nim values (config, ids, string lists), so it can coexist with the
## still-old-API `ast2nif.nim` during the migration.
import std / [syncio, algorithm]
from std / os import removeFile, moveFile
import options, pathutils, typekeys
import "../dist/nimony/src/lib" / [nifcore, nifcoreparse, nifreader, bif]
# One shared literals pool + tag pool for the whole process — the nifcore
# analogue of the old global `nifstreams.pool`. A single shared pool keeps
# string/symbol/file ids stable across every TokenBuf the IC backend builds,
# preserving the old global-pool semantics during the migration. (Stage 6 may
# move to fresh per-file pools for bif's fast path; see doc/ic_nifcore_port.md.)
let icPool* = newPool()
let icTags* = newTagPool()
proc createIcBuf*(cap = 16): TokenBuf {.inline.} =
## A `TokenBuf` bound to the shared IC pools.
createTokenBuf(cap, icPool, icTags)
proc tagId*(s: string): TagId {.inline.} =
## Intern a tag name in the shared tag pool.
icTags.registerTag(s)
type
IcBuilder* = object
## A thin nifcore `TokenBuf` builder whose surface is *primitive types only*
## (strings/ints/floats). It lets the still-old-API `ast2nif.nim` drive a
## nifcore buffer without any nifcore type crossing the module boundary —
## the bridge that routes IC output onto the nifcore serializer (Stage 2).
buf*: TokenBuf
proc newIcBuilder*(cap = 16): IcBuilder = IcBuilder(buf: createIcBuf(cap))
proc openTag*(b: var IcBuilder; tag: string) {.inline.} = b.buf.openTag(tagId(tag))
proc closeTag*(b: var IcBuilder) {.inline.} = b.buf.closeTag()
proc addSymUse*(b: var IcBuilder; s: string) {.inline.} = b.buf.addSymUse(s)
proc addSymDef*(b: var IcBuilder; s: string) {.inline.} = b.buf.addSymDef(s)
proc addIdent*(b: var IcBuilder; s: string) {.inline.} = b.buf.addIdent(s)
proc addStrLit*(b: var IcBuilder; s: string) {.inline.} = b.buf.addStrLit(s)
proc addIntLit*(b: var IcBuilder; v: int64) {.inline.} = b.buf.addIntLit(v)
proc addUIntLit*(b: var IcBuilder; v: uint64) {.inline.} = b.buf.addUIntLit(v)
proc addFloatLit*(b: var IcBuilder; v: float64) {.inline.} = b.buf.addFloatLit(v)
proc addCharLit*(b: var IcBuilder; c: char) {.inline.} = b.buf.addCharLit(c)
proc addDotToken*(b: var IcBuilder) {.inline.} = b.buf.addDotToken()
proc lineInfo*(b: var IcBuilder; file: string; line, col: int32; comment = "") =
## Attach line info (+ optional `#comment#`) to the head just emitted. No-op
## when `file` is empty (matches the old "emit only when info is valid").
## Strings are interned in the shared pools; the file/comment ids reproduce
## the old `pool.files`/`pool.strings` entries by string value.
if file.len == 0: return
let fid = icPool.filenames.getOrIncl(file)
let cid = if comment.len > 0: icPool.strings.getOrIncl(comment) else: StrId(0)
b.buf.appendLineInfo(fid, line, col, cid)
proc writeFileStable*(b: var TokenBuf; path: string; onlyIfChanged = false) =
## Serialize `b` to canonical module NIF text and write it. Mirrors
## `nifcursors.writeFile`: the module suffix is derived from `path`
## (`"." & extractModuleSuffix`), and `onlyIfChanged` skips the write when the
## on-disk bytes already match — the content-stability nifmake's incremental
## rebuild depends on.
let content = toModuleString(b, "." & extractModuleSuffix(path))
if onlyIfChanged:
let existing =
try: readFile(path)
except CatchableError: ""
if existing == content: return
writeFile(path, content)
proc writeStable*(b: var IcBuilder; path: string; onlyIfChanged = false) {.inline.} =
writeFileStable(b.buf, path, onlyIfChanged)
proc cursorPool*(c: Cursor): Pool {.inline.} = nifcore.pool(c)
## The literals pool the cursor's buffer was built against. `ast2nif.nim`
## imports `nifcore` with `except pool` (to keep nifstreams' global `pool`
## var the writer uses), so the reader reaches a cursor's pool through here —
## needed once `bif`-loaded buffers carry their OWN fresh pool rather than the
## shared `icPool`.
proc freshModuleCopy(b: var IcBuilder): TokenBuf =
## Re-home `b.buf` into a PRIVATE, module-local pool via `addSubtree` (which
## re-interns only the literals/tags this buffer actually uses). `b.buf` is bound
## to the process-wide shared `icPool`/`icTags`; storing it directly would embed
## the WHOLE shared pool (correct but huge — see `bif.storeToFile`). The copy's
## fresh-pool reload reproduces ids verbatim (the bif fresh-pool INVARIANT).
result = createTokenBuf(b.buf.len, newPool(), newTagPool())
var c = b.buf.beginRead()
while c.hasMore:
addSubtree(result, c)
skip c
proc storeBif*(b: var IcBuilder; path: string; dottedSuffix: string) =
## Persist the buffer as a compact, self-contained binary NIF (`.bif`).
var fresh = freshModuleCopy(b)
bif.store(fresh, path, dottedSuffix)
proc storeBifStable*(b: var IcBuilder; path: string; dottedSuffix: string) =
## Content-stable `bif` write — the binary analogue of `writeFileStable`'s
## `onlyIfChanged`: only replace `path` when the encoded bytes differ, so an
## unchanged sidecar keeps its mtime and nifmake prunes the dependent rebuild
## cascade. Used for the iface/impl cookies + dep sidecars whose byte-stability
## gates incremental builds. (bif encoding is deterministic for a given buffer
## under fresh pools, so equal content ⇒ equal bytes.)
var fresh = freshModuleCopy(b)
let tmp = path & ".tmp"
bif.store(fresh, tmp, dottedSuffix)
let newBytes = readFile(tmp)
let oldBytes =
try: readFile(path)
except CatchableError: ""
if newBytes == oldBytes:
removeFile(tmp)
else:
moveFile(tmp, path)
# --- subtree splicing (shared pool, so a raw subtree copy is exact) ----------
proc addAll*(dest: var IcBuilder; src: var IcBuilder) =
## Append every top-level subtree of `src` into `dest` — the nifcore analogue
## of the old `dest.add wholeBuffer` splice.
var c = src.buf.beginRead()
while c.hasMore:
addSubtree(dest.buf, c)
skip c
proc addStmtsBody*(dest: var IcBuilder; src: var IcBuilder) =
## Append the BODY of a `(stmts . . <body> )` builder into `dest`, dropping the
## wrapper tag and its two leading dot slots (flags/type) — the nifcore
## analogue of the old `for i in 3 ..< content.len-1: dest.add content[i]`.
var c = src.buf.beginRead() # at (stmts
c.into:
skip c # flags dot
skip c # type dot
while c.hasMore:
addSubtree(dest.buf, c)
skip c
# --- cookie input: a line-info-free logical token list of the module ---------
# The cookie hashers (ast2nif) need a flat, ParRi-bearing, index-addressable
# view of the serialized module. nifcore has no ParRi kind and variable-width
# tokens, so we flatten the buffer here (in the clean nifcore world) into a
# neutral `CookieTok` list — no nifcore type crosses into ast2nif.
type
CookieKind* = enum
ckParLe, ckParRi, ckSym, ckSymDef, ckIdent, ckStr, ckInt, ckUInt, ckFloat, ckChar, ckDot
CookieTok* = object
kind*: CookieKind
tag*: string # ckParLe
name*: string # ckSym / ckSymDef
sym*: uint32 # ckSym / ckSymDef id (identity key)
str*: string # ckIdent / ckStr
ival*: int64
uval*: uint64
fval*: float64
cval*: uint32
proc flattenGo(c: var Cursor; b: TokenBuf; acc: var seq[CookieTok]) =
while c.hasMore:
case c.kind
of TagLit:
acc.add CookieTok(kind: ckParLe, tag: b.tags.tagName(c.cursorTagId))
c.into:
flattenGo(c, b, acc)
acc.add CookieTok(kind: ckParRi)
of Symbol:
acc.add CookieTok(kind: ckSym, name: symName(c, b.pool), sym: uint32(symId(c, b.pool)))
skip c
of SymbolDef:
acc.add CookieTok(kind: ckSymDef, name: symName(c, b.pool), sym: uint32(symId(c, b.pool)))
skip c
of Ident:
acc.add CookieTok(kind: ckIdent, str: strVal(c, b.pool)); skip c
of StrLit:
acc.add CookieTok(kind: ckStr, str: strVal(c, b.pool)); skip c
of IntLit:
acc.add CookieTok(kind: ckInt, ival: intVal(c)); skip c
of UIntLit:
acc.add CookieTok(kind: ckUInt, uval: uintVal(c)); skip c
of FloatLit:
acc.add CookieTok(kind: ckFloat, fval: floatVal(c)); skip c
of CharLit:
acc.add CookieTok(kind: ckChar, cval: uint32(ord(charLit(c)))); skip c
of DotToken:
acc.add CookieTok(kind: ckDot); skip c
else:
skip c # LineInfoLit / ExtendedSuffix ride on heads, never standalone
proc flattenForCookie*(b: var IcBuilder): seq[CookieTok] =
## Flatten the nifcore module buffer to the cookie hashers' flat token list.
result = newSeqOfCap[CookieTok](b.buf.len)
var cur = b.buf.beginRead()
flattenGo(cur, b.buf, result)
proc collectBifStrLits*(path: string): seq[string] =
## Read a small `(tag "s" "s" …)` bif sidecar (`semdeps`/`edges`) and return every
## string literal it holds, in order — the binary analogue of the old nifstreams
## scan that collected `StrLit`s. Keeps nifcore types out of `deps.nim`, which
## only needs the recorded string list.
##
## Uses `loadFromFile` (a full read into owned memory) rather than the mmap-backed
## `bif.load`, then CLOSES the handle. `bif.load` intentionally leaves the mapping
## resident for the process lifetime; for the `nim ic` driver that reads these
## sidecars while `nim m` children rewrite them, a lingering read mapping is a
## Windows sharing violation: the child's `open(path, fmWrite)` fails with
## `IOError: cannot open`. These sidecars are tiny, so the zero-copy mmap buys
## nothing here anyway.
result = @[]
var f = open(path, fmRead)
var m = bif.loadFromFile(f)
close(f)
var c = m.buf.beginRead()
while c.hasMore:
if c.kind == StrLit: result.add strVal(c)
inc c
proc writeSemDeps*(config: ConfigRef; thisModule: int32; importPaths: seq[string]) =
## Stage 1 spike: the nifcore port of `ast2nif.writeSemDeps`. Serializes the
## module's resolved direct imports as `(semdeps "path" ...)`. Byte-identical
## to the old writer (verified), so `nim ic` build graphs are unaffected.
let selfSuffix = modname(thisModule, config)
var paths = importPaths
sort paths
var dest = newIcBuilder(4 + 2*paths.len)
dest.openTag "semdeps"
for p in paths:
dest.addStrLit p
dest.closeTag()
let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".s.deps.bif").string
storeBifStable(dest, path, "." & extractModuleSuffix(path))

View File

@@ -1,279 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## NIF-based goto-definition / find-all-usages for `nim track`.
##
## This is the mainline-Nim port of nimony's `idetools.nim`. It answers a
## `--def:FILE,LINE,COL` / `--usages:FILE,LINE,COL` query by *scanning the
## `.s.bif` files* (binary NIF, see `dist/nimony/src/lib/bif.nim`) that the
## preceding `nim ic` frontend (`nim track`) emitted into the nimcache directory
## — NOT by re-running sem. NIF distinguishes a definition (`SymbolDef` token) from a use
## (`Symbol` token) syntactically, so goto-def / find-uses become plain token
## scans over type-checked NIF, which is more reliable than the classic PSym
## engine because generics and macros are type-checked in the NIF too.
##
## Two passes (mirroring nimony's `usages`):
## 1. Load the queried module's `.s.bif` and find the `Symbol`/`SymbolDef`
## token whose line info + identifier length contains `conf.m.trackPos`.
## That yields the mangled symbol NAME and whether it is global (>= 2 dots).
## 2. `--usages`: emit every `Symbol` (use) token; `--def`: every `SymbolDef`.
## A global symbol is scanned across every module `.s.bif`; a local one only
## within the queried module.
##
## IMPORTANT porting note: `bif.load` mints FRESH per-file pools, so a `SymId`
## from module A's buffer is meaningless in module B's. The cross-module match is
## therefore by the mangled NAME string, never by `SymId` (nimony can compare ids
## because it parses every text NIF into one shared global pool; we cannot).
import std / [os, strutils, sets]
import options, msgs, pathutils
import lineinfos as astli
import ast2nif # toNifFilename
from deps import includerSbifs # deps-guided include-file lookup
import "../dist/nimony/src/lib/nifcore"
from "../dist/nimony/src/lib" / bif import load, BifModule, containsSym
proc identLen(name: string): int =
## Length of the displayed identifier: the run before the first `.` of a
## mangled NIF name (`ident.disamb[.moduleSuffix]`). Bounds the column match.
let d = name.find('.')
result = if d < 0: name.len else: d
proc isGlobalName(name: string): bool =
## A global symbol carries `ident.disamb.moduleSuffix` (>= 2 dots); a local at
## most `ident.disamb` (<= 1 dot). `moduleSuffix` is a dot-free hash, so a raw
## dot count is equivalent to nifbuilder's suffix-compressed test for our use.
var dots = 0
for i in 1 ..< name.len:
if name[i] == '.': inc dots
result = dots >= 2
proc posMatch(c: Cursor; conf: ConfigRef; target: TLineInfo; tokenLen: int): bool =
## True when `target` (the queried position) falls within the identifier span
## of the Symbol/SymbolDef token at `c`. Mirrors nimony's `lineInfoMatch`; the
## filename is resolved through the loaded buffer's own pool (fresh per file),
## then mapped to a `FileIndex` exactly like `ast2nif.oldLineInfo`.
let li = rawLineInfo(c)
if not li.isValid: return false
if li.line.int != target.line.int: return false
let f = fileInfoIdx(conf, AbsoluteFile lineInfoFile(c))
if f != target.fileIndex: return false
if target.col.int < li.col.int: return false
if target.col.int > li.col.int + tokenLen: return false
result = true
const sep = '\t'
proc formatSuggest(s: Suggest): string =
## Reproduce `suggest.$Suggest` for the `ideDef`/`ideUse` sections without
## importing `suggest` (which would create an import cycle). Layout:
## `section⭾symkind⭾qualifiedPath⭾forth⭾filePath⭾line⭾column⭾⭾quality`.
## symkind is always `skUnknown` here — the raw NIF scan has no PSym to give a
## real kind (like nimony's `foundSymbol`, which leaves it empty).
result = $s.section
result.add sep
result.add "skUnknown"
result.add sep
if s.qualifiedPath.len != 0:
result.add s.qualifiedPath.join(".")
result.add sep
result.add s.forth
result.add sep
result.add s.filePath
result.add sep
result.add $s.line
result.add sep
result.add $s.column
result.add sep # empty doc field (docgen is off outside nimsuggest)
if s.version == 0 or s.version == 3:
result.add sep
result.add $s.quality
proc emit(conf: ConfigRef; c: Cursor; section: IdeCmd; name: string;
seen: var HashSet[string]) =
## Report one hit as a nimsuggest-compatible result (routed through the
## structured-output hook / `--stdout`). We only have the mangled name + line
## info from the raw NIF, so symkind/type are left empty — like nimony's
## `foundSymbol`. `seen` deduplicates: the same source location can back
## several NIF `Symbol` tokens (e.g. a call argument re-emitted in a lowered
## form), which must surface as one hit.
let li = rawLineInfo(c)
if not li.isValid: return
let key = $section.int & ":" & lineInfoFile(c) & ":" & $li.line.int & ":" & $li.col.int
if seen.containsOrIncl(key):
return # already reported this location for this section
let s = Suggest(section: section,
qualifiedPath: @[name[0 ..< identLen(name)]],
filePath: lineInfoFile(c),
line: li.line.int,
column: li.col.int,
tokenLen: identLen(name),
forth: "",
symkind: 0'u8,
quality: 100,
version: conf.suggestVersion)
if conf.suggestionResultHook != nil:
conf.suggestionResultHook(s)
else:
conf.suggestWriteln(formatSuggest(s))
proc tokenSymId(c: Cursor): SymId {.inline.} =
## SymId (in the cursor's own per-file pool) of a `Symbol`/`SymbolDef` token,
## or `SymId(0)` for an inline-encoded one — which is never our search target:
## a mangled name (`ident.disamb.suffix`) is always longer than
## `StrInlineMaxLen`, so every occurrence of the symbol we look for is stored by
## pool id, decoded here with a shift and no string materialization.
if isInlineLit(c): SymId(0) else: SymId(combinedPayload(c) shr 1)
template symMatches(c: Cursor): bool =
## True when the token at `c` is the searched symbol. The fast path is a pure
## integer compare against `targetSym` (the symbol's id in THIS module's pool,
## resolved once per file by the caller). `targetSym == 0` means the name is not
## representable as a pool id (a rare <=3-byte local): fall back to a string
## compare, correct for both inline and pooled encodings.
(if targetSym != SymId(0): tokenSymId(c) == targetSym else: symName(c) == targetName)
proc scanUses(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string;
seen: var HashSet[string]) =
## `--usages`: report every `Symbol` (use) occurrence with valid line info.
if m.buf.len == 0: return
var c = m.buf.beginRead()
while c.hasMore:
if c.kind == Symbol and symMatches(c) and rawLineInfo(c).isValid:
emit(conf, c, ideUse, targetName, seen)
inc c
c.endRead()
proc scanDef(conf: ConfigRef; m: var BifModule; targetSym: SymId; targetName: string;
seen: var HashSet[string]) =
## `--def`: report the declaration of the target symbol if this module owns it
## (has its `SymbolDef`). The `SymbolDef` token itself carries no line info; the
## declaration location lives on the *enclosing tag* (e.g. `(sd @file:line:col`,
## like `bif.buildIndex`'s `mostRecentTagPos`). When that tag has no line info
## either, fall back to the declaration-site `Symbol` occurrence — but only in
## the owning module, so a plain user of the symbol is never reported as a def.
if m.buf.len == 0: return
var c = m.buf.beginRead()
var mostRecentTagPos = 0
var sawDef = false
var emitted = false
var fallbackPos = -1
while c.hasMore:
case c.kind
of TagLit:
mostRecentTagPos = cursorToPosition(m.buf, c)
inc c
of SymbolDef:
if symMatches(c):
sawDef = true
var tc = cursorAt(m.buf, mostRecentTagPos)
if rawLineInfo(tc).isValid:
emit(conf, tc, ideDef, targetName, seen)
emitted = true
tc.endRead()
inc c
of Symbol:
if fallbackPos < 0 and symMatches(c) and rawLineInfo(c).isValid:
fallbackPos = cursorToPosition(m.buf, c)
inc c
else:
inc c
c.endRead()
if sawDef and not emitted and fallbackPos >= 0:
var fc = cursorAt(m.buf, fallbackPos)
emit(conf, fc, ideDef, targetName, seen)
fc.endRead()
proc scanBuf(conf: ConfigRef; m: var BifModule; section: IdeCmd;
targetSym: SymId; targetName: string; seen: var HashSet[string]) =
## Emit hits for the target symbol in `m` per the query kind. `ideDus`
## (`--defusages`) reports both the definition and every usage.
if section in {ideDef, ideDus}:
scanDef(conf, m, targetSym, targetName, seen)
if section in {ideUse, ideDus}:
scanUses(conf, m, targetSym, targetName, seen)
proc findPos(conf: ConfigRef; m: var BifModule; target: TLineInfo;
foundName: var string): bool =
## Scan `m` for the `Symbol`/`SymbolDef` token covering the queried position
## `target` and set `foundName` to its mangled name. Returns true on a hit.
if m.buf.len == 0: return false
var c = m.buf.beginRead()
result = false
while c.hasMore:
let k = c.kind
if k == Symbol or k == SymbolDef:
let nm = symName(c)
if posMatch(c, conf, target, identLen(nm)):
foundName = nm
result = true
break
inc c
c.endRead()
proc runIdeQuery*(conf: ConfigRef) =
## Entry point: called from `main.nim` after `commandCheck` when a
## `--def`/`--usages` query is active. Assumes the check just emitted the
## project's `.s.bif` files into `getNimcacheDir(conf)`.
let section = conf.ideCmd
if section notin {ideDef, ideUse, ideDus}: return
let target = conf.m.trackPos
if target.fileIndex.int32 < 0: return
# Pass 1: position -> symbol. Try the queried file's own module bif first (the
# fast path when the position is inside a real module). An include file has no
# module bif of its own — its tokens live in the *including* module's bif with
# include-file line info — so when the direct lookup misses, consult the
# `.deps.nif` preludes (`includerSbifs`) to load only the module(s) that
# include the queried file (directly or transitively), never every bif in the
# nimcache. `ownerFile` is the bif that owns the hit.
let modFile = toNifFilename(conf, target.fileIndex)
var foundName = ""
var ownerFile = ""
if fileExists(modFile):
var qm = load(modFile)
if findPos(conf, qm, target, foundName):
ownerFile = modFile
if foundName.len == 0:
for cand in includerSbifs(conf, toFullPath(conf, target.fileIndex).AbsoluteFile):
if cand == modFile: continue
var m = load(cand)
if findPos(conf, m, target, foundName):
ownerFile = cand
break
if foundName.len == 0: return
# Pass 2: emit definition / usages. `seen` spans every module so a location is
# reported once even when scanned across the whole nimcache.
#
# Cross-file matching is by SymId, not by decoding every token's name. Two
# filters keep it cheap:
# 1. `bif.containsSym` — a sym-table-only probe that reads just the small
# trailing pools, NOT the token block or any `BiTable`. A module that never
# references the symbol is rejected here without a full `load` (no pools
# built, no token block mapped) — so a query whose symbol lives in a few
# modules no longer pays to load the whole nimcache.
# 2. For a module that does contain it, `bif.load` mints a fresh per-file pool,
# so the name is resolved to THIS file's SymId once via `getKeyId`; the scan
# then compares integer ids per token instead of materializing a string for
# each (see `symMatches`).
var seen = initHashSet[string]()
if isGlobalName(foundName):
for f in walkFiles((getNimcacheDir(conf).string) / "*.s.bif"):
if not containsSym(f, foundName): continue
var m = load(f)
let tid = m.buf.pool.syms.getKeyId(foundName)
if tid != SymId(0):
scanBuf(conf, m, section, tid, foundName, seen)
else:
# Local symbol: its mangled name is not unique across modules, so restrict
# the scan to the module it lives in (the one that owns the queried position).
var qm = load(ownerFile)
let tid = qm.buf.pool.syms.getKeyId(foundName)
scanBuf(conf, qm, section, tid, foundName, seen)

View File

@@ -24,7 +24,7 @@ import std/[strtabs, tables, strutils, intsets]
when defined(nimPreviewSlimSystem):
import std/assertions
from trees import exprStructuralEquivalent, getRoot, isCursor, whichPragma, getPotentialWrites
from trees import exprStructuralEquivalent, getRoot, whichPragma, getPotentialWrites
type
Con = object
@@ -180,6 +180,17 @@ proc isFirstWrite(n: PNode; c: var Con): bool =
let m = skipConvDfa(n)
result = nfFirstWrite in m.flags
proc isCursor(n: PNode): bool =
case n.kind
of nkSym:
sfCursor in n.sym.flags
of nkDotExpr:
isCursor(n[1])
of nkCheckedFieldExpr:
isCursor(n[0])
else:
false
template isFullyUnpackedTuple(n: PNode): bool =
## we move out all elements of unpacked tuples,
## hence unpacked tuples themselves don't need to be destroyed
@@ -234,18 +245,6 @@ proc genOp(c: var Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode
let canon = c.graph.canonTypes.getOrDefault(h)
if canon != nil:
op = getAttachedOp(c.graph, canon, kind)
if op == nil or op.ast.isGenericRoutine:
# IC: injectDestructorCalls is demand-driven and runs HERE (cg), not in the
# `lower` stage, so a structural, env-agnostic op the lower stage never had
# reason to serialize — most often a closure PROC type's `=destroy`/`=sink`
# (which act on the `(ClP_0, ClE_0)` tuple, NOT the concrete env) — must be
# lifted on demand, exactly as the lazy path's cg does. This is safe now:
# closure-env identity resolves via `attachedOps[itemId]`/env-erased typeKey,
# env objects load complete, and atomicRefOp's type-erased path covers any
# still-incomplete env (so the lift never walks a nil field).
excl t.flagsImpl, tfCheckedForDestructor
createTypeBoundOps(c.graph, nil, t, dest.info, c.idgen)
op = getAttachedOp(c.graph, t, kind)
if op == nil:
#echo dest.typ.id
globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
@@ -804,23 +803,6 @@ proc hasCustomDestructor(c: Con, t: PType): bool =
obj = skipTypes(obj.baseClass, abstractPtrs)
result = result or isCustomDestructor(c, obj)
const
exprBranchKinds = {nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt,
nkTryStmt, nkPragmaBlock}
proc distributeAsgn(asgnKind: TNodeKind; dest, ri: PNode; c: var Con; s: var Scope): PNode =
## Distributes an assignment ``dest = ri`` into the leaf expressions of
## ``ri`` when ``ri`` is an expression-based control flow construct. This
## avoids creating pointless intermediate temporaries (bug #25850). The
## descent is recursive so that nestings like ``block: ...; if c: a else: b``
## assign directly to ``dest`` instead of going through a temp per branch.
if ri.kind in exprBranchKinds:
template process(child, s): untyped =
distributeAsgn(asgnKind, dest, child, c, s)
handleNestedTempl(ri, process, willProduceStmt = true)
else:
result = newTree(asgnKind, dest, p(ri, c, s, consumed))
proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSingleUsedTemp}; inReturn = false): PNode =
if n.kind in {nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkIfStmt,
nkIfExpr, nkCaseStmt, nkWhen, nkWhileStmt, nkParForStmt, nkTryStmt, nkPragmaBlock}:
@@ -1022,11 +1004,13 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing
result = moveOrCopy(p(n[0], c, s, mode), n[1], c, s, flags)
elif isDiscriminantField(n[0]):
result = c.genDiscriminantAsgn(s, n)
elif n[1].kind in exprBranchKinds:
elif n[1].kind in {nkStmtListExpr, nkBlockExpr, nkIfExpr, nkCaseStmt, nkTryStmt, nkPragmaBlock}:
# Distribute the assignment into each branch to avoid
# creating pointless temporaries for expression-based control flow.
let dest = p(n[0], c, s, mode)
result = distributeAsgn(n.kind, dest, n[1], c, s)
template process(child, s): untyped =
newTree(n.kind, dest, p(child, c, s, consumed))
handleNestedTempl(n[1], process, willProduceStmt = true)
else:
result = copyNode(n)
result.add p(n[0], c, s, mode)

View File

@@ -1,98 +0,0 @@
#
#
# The Nim Compiler
# (c) Copyright 2026 Andreas Rumpf
#
# See the file "copying.txt", included in this
# distribution, for details about the copyright.
#
## `ItemId` is the identity of a symbol or type: a `(module, item)` pair.
##
## The fields are private on purpose: the module half reserves bit 30 as the
## "backend minted" marker, so all construction and inspection has to go
## through this module's API and the marker bit can never leak into module
## indexing or arithmetic.
##
## Three id spaces coexist per module:
## - Semantic-phase and NIF-loader ids: `itemId(module, item)` with `item > 0`.
## - Backend-minted ids (IC codegen, `nim nifc`: transf labels and temps,
## lifted hooks): `backendItemId` sets `BackendModuleBit`, so these can
## never compare equal to a loader id even though both counters mint the
## same small `item` range in one process. They never cross a process
## boundary and must never be written to a NIF file.
## - Derived env/tuple-field ids (`lowerings.addField`): the source local's
## id with `item` negated. `derivedFieldId` preserves the backend marker,
## keeping the derivation collision-free for both id spaces above.
import std/hashes
when defined(nimPreviewSlimSystem):
import std/assertions
const
BackendModuleBit = 0x4000_0000'i32
# Bit 30 of the module field. Bit 31 stays clear so marked module values
# remain non-negative and cannot be mistaken for the special negative
# module ids like `PackageModuleId`.
PackageModuleId* = -3'i32
type
ItemId* = object
moduleBits: int32
itemBits: int32
proc itemId*(module, item: int32): ItemId {.inline.} =
assert module < 0 or (module and BackendModuleBit) == 0
ItemId(moduleBits: module, itemBits: item)
proc backendItemId*(module, item: int32): ItemId {.inline.} =
## An id minted during IC codegen; distinct from every `itemId` of the
## same module so that the loader's stub counter and the backend's counter
## cannot collide in id-keyed tables.
assert module >= 0 and (module and BackendModuleBit) == 0
ItemId(moduleBits: module or BackendModuleBit, itemBits: item)
proc module*(x: ItemId): int32 {.inline.} =
if x.moduleBits >= 0: x.moduleBits and not BackendModuleBit
else: x.moduleBits
proc item*(x: ItemId): int32 {.inline.} = x.itemBits
proc isBackendMinted*(x: ItemId): bool {.inline.} =
x.moduleBits >= 0 and (x.moduleBits and BackendModuleBit) != 0
proc derivedFieldId*(source: ItemId): ItemId {.inline.} =
## The id of the env/tuple field that `lowerings.addField` derives for a
## captured local: `item` negated, module bits (including the backend
## marker) preserved.
ItemId(moduleBits: source.moduleBits, itemBits: -abs(source.itemBits))
proc matchesDerivedFieldId*(field, source: ItemId): bool {.inline.} =
## Does `field` carry the id `derivedFieldId` would derive for `source`?
## `source` may itself already be the derived field id.
field.moduleBits == source.moduleBits and
field.itemBits == -abs(source.itemBits)
proc `==`*(a, b: ItemId): bool {.inline.} =
# raw bit comparison: a backend-minted id never equals a loader id
a.itemBits == b.itemBits and a.moduleBits == b.moduleBits
proc hash*(x: ItemId): Hash =
var h: Hash = hash(x.moduleBits)
h = h !& hash(x.itemBits)
result = !$h
proc `$`*(x: ItemId): string =
result = "(module: " & $x.module & ", item: " & $x.itemBits
if x.isBackendMinted: result.add ", backend"
result.add ")"
const
moduleShift = when defined(cpu32): 20 else: 24
proc toId*(a: ItemId): int {.inline.} =
## Packs an ItemId into a single int. Uses the raw module bits so the
## backend marker keeps the two id spaces disjoint (bit 30 shifts to
## bit 54; like the module/item split itself this needs a 64-bit int).
(a.moduleBits.int shl moduleShift) + a.itemBits.int

View File

@@ -34,7 +34,7 @@ import
ropes, wordrecg, renderer,
cgmeth, lowerings, sighashes, modulegraphs, lineinfos,
transf, injectdestructors, sourcemap, astmsgs, pushpoppragmas,
mangleutils, varpartitions
mangleutils
import pipelineutils
@@ -1298,16 +1298,14 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
xtyp = etySeq
case xtyp
of etySeq:
if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded or
(x.kind == nkSym and sfCursor in x.sym.flags):
if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
else:
useMagic(p, "nimCopy")
lineF(p, "$1 = nimCopy(null, $2, $3);$n",
[a.rdLoc, b.res, genTypeInfo(p, y.typ)])
of etyObject:
if x.typ.kind in {tyVar, tyLent, tyOpenArray, tyVarargs} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded or
(x.kind == nkSym and sfCursor in x.sym.flags):
if x.typ.kind in {tyVar, tyLent, tyOpenArray, tyVarargs} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
else:
useMagic(p, "nimCopy")
@@ -2094,8 +2092,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
gen(p, n, a)
case mapType(p, v.typ)
of etyObject, etySeq:
if v.typ.kind in {tyOpenArray, tyVarargs} or needsNoCopy(p, n) or
sfCursor in v.flags:
if v.typ.kind in {tyOpenArray, tyVarargs} or needsNoCopy(p, n):
s = a.res
else:
useMagic(p, "nimCopy")
@@ -2801,11 +2798,6 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
var transformedBody = transformBody(p.module.graph, p.module.idgen, prc, {})
if sfInjectDestructors in prc.flags:
transformedBody = injectDestructorCalls(p.module.graph, p.module.idgen, prc, transformedBody)
else:
# JS has a GC, so the destructor pass is off; but the cursor (alias) analysis
# is independent of ownership and always memory-safe on a traced target.
# Running it lets last-use `var b = a` aliases skip the deep `nimCopy`.
computeCursors(prc, transformedBody, p.module.graph)
p.nested: genStmt(p, transformedBody)

View File

@@ -164,21 +164,9 @@ proc getClosureIterResult*(g: ModuleGraph; iter: PSym; idgen: IdGenerator): PSym
incl(result.flagsImpl, sfUsed)
iter.ast.add newSymNode(result)
proc closureParams(routine: PSym): PNode =
## The formal parameters node lambda lifting reads and extends. In a
## from-source compilation `routine.ast[paramsPos]` and `routine.typ.n` are the
## very same node (see the `typ.n.len` based position math below). Under IC the
## loaded proc AST omits the parameters (they are kept only in `typ.n`), so
## restore the shared node here.
result = routine.ast[paramsPos]
if (result == nil or result.kind == nkEmpty) and routine.typ != nil and
routine.typ.n != nil and routine.ast.len > paramsPos:
result = routine.typ.n
routine.ast[paramsPos] = result
proc addHiddenParam*(routine: PSym, param: PSym) =
proc addHiddenParam(routine: PSym, param: PSym) =
assert param.kind == skParam
var params = closureParams(routine)
var params = routine.ast[paramsPos]
# -1 is correct here as param.position is 0 based but we have at position 0
# some nkEffect node:
param.position = routine.typ.n.len-1
@@ -189,8 +177,7 @@ proc addHiddenParam*(routine: PSym, param: PSym) =
proc getEnvParam*(routine: PSym): PSym =
if routine.ast.isNil: return nil
let params = closureParams(routine)
if params == nil or params.len == 0: return nil
let params = routine.ast[paramsPos]
let hidden = lastSon(params)
if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
result = hidden.sym
@@ -307,27 +294,7 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
elif not (owner.typ.isClosure or owner.isNimcall and not owner.isExplicitCallConv or isEnv):
localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
[s.name.s, owner.name.s, $owner.typ.callConv])
unsealForTransform(owner.typ)
incl(owner.typ, tfCapturesEnv)
# A closure proc type that captures an env owns a REF to it: copying the closure
# value must incref the env and destroying it must decref. That is exactly what
# `tfHasAsgn` signals to `injectDestructorCalls` (so a closure assignment becomes
# `=copy`, not a raw field store).
#
# Set it HERE (closure-type creation) so the flag is DETERMINISTIC and serializes
# with the type — but ONLY under `nim ic`. The per-module `lower` stage is a
# separate process that lowers routines in index order; if a consumer (e.g.
# `workNimAsyncContinue`) was lowered before the closure type's ops were lifted,
# its env store emitted a RAW assign with no incref → freed env → async
# "yielded `nil`". A normal single-process `nim c` build does NOT need this —
# `createTypeBoundOps` sets the flag lazily, in lift order, before it matters
# (the old `liftdestructors ~1498` "XXX Breaks IC!" side effect) — and setting it
# eagerly there REGRESSES codegen: a `=destroy` hook gets generated against the
# bare `void(*)(void)` proc representation but is then called with closure structs
# (`eqdestroy__u2__stdZtypedthreads` type mismatch — broke megatest). So gate on
# `cmdNifC`; normal builds keep the lazy (devel) behavior.
if g.config.cmd == cmdNifC:
incl(owner.typ, tfHasAsgn)
if not isEnv:
owner.typ.callConv = ccClosure

View File

@@ -75,11 +75,6 @@ proc newAsgnStmt(le, ri: PNode): PNode =
result[0] = le
result[1] = ri
proc newSinkAsgnStmt(le, ri: PNode): PNode =
result = newNodeI(nkSinkAsgn, le.info, 2)
result[0] = le
result[1] = ri
proc genBuiltin*(g: ModuleGraph; idgen: IdGenerator; magic: TMagic; name: string; i: PNode): PNode =
result = newNodeI(nkCall, i.info)
result.add createMagic(g, idgen, name, magic).newSymNode
@@ -89,9 +84,7 @@ proc genBuiltin(c: var TLiftCtx; magic: TMagic; name: string; i: PNode): PNode =
result = genBuiltin(c.g, c.idgen, magic, name, i)
proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
if c.kind == attachedSink:
body.add newSinkAsgnStmt(x, y)
elif c.kind in {attachedAsgn, attachedDeepCopy, attachedDup}:
if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink, attachedDup}:
body.add newAsgnStmt(x, y)
elif c.kind == attachedDestructor and c.addMemReset:
let call = genBuiltin(c, mDefault, "default", x)
@@ -101,21 +94,11 @@ proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
body.add genBuiltin(c, mWasMoved, "wasMoved", x)
proc genAddr(c: var TLiftCtx; x: PNode): PNode =
# These synthesized addresses are always passed to codegen procs that expect a
# genuine pointer (nimAsgnYrc, nimSinkYrc, destructors, ...). `addr(deref x)`
# collapses to `x` only when `x` is a real pointer; on the C++ backend a `var`
# parameter is a C++ reference, so we must keep the `nkHiddenAddr` to actually
# take its address (`&dest`) instead of passing the reference's value. Likewise
# `tfVarIsPtr` keeps the C++ backend from lowering the synthesized address back
# to a reference and dropping the `&` (e.g. a closure's `tyPointer` env). See
# #26026 CI (yrc + cpp).
if x.kind == nkHiddenDeref and c.g.config.backend != backendCpp:
if x.kind == nkHiddenDeref:
checkSonsLen(x, 1, c.g.config)
result = x[0]
else:
let addrTyp = makeVarType(x.typ.owner, x.typ, c.idgen)
addrTyp.incl tfVarIsPtr
result = newNodeIT(nkHiddenAddr, x.info, addrTyp)
result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ, c.idgen))
result.add x
proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
@@ -728,11 +711,6 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
doAssert t.asink != nil
body.add newHookCall(c, t.asink, x, y)
of attachedDestructor:
when defined(icDbg):
if t.destructor == nil:
echo "MISSING destructor: ", typeToString(t), " kind=", t.kind,
" itemId=", t.itemId, " uniqueId=", t.uniqueId, " state=", t.state,
" owner=", (if t.owner != nil: t.owner.name.s else: "nil")
doAssert t.destructor != nil
body.add destructorCall(c, t.destructor, x)
of attachedTrace:
@@ -838,15 +816,13 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
tfAcyclic in skipTypes(elemType, abstractInst+{tyOwned}-{tyTypeDesc}).flags
# dynamic Acyclic refs need to use dyn decRef
let useStatic = isFinal(elemType)
let tmp =
if isCyclic and c.kind in {attachedAsgn, attachedSink, attachedDup}:
declareTempOf(c, body, x)
else:
x
if useStatic:
if isFinal(elemType):
addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr))
var alignOf = genBuiltin(c, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
alignOf.typ = getSysType(c.g, c.info, tyInt)
@@ -857,7 +833,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
var cond: PNode
if isCyclic:
if useStatic:
if isFinal(elemType):
let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
typInfo.typ = getSysType(c.g, c.info, tyPointer)
cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
@@ -892,7 +868,7 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
of attachedDeepCopy: assert(false, "cannot happen")
of attachedTrace:
if isCyclic:
if useStatic:
if isFinal(elemType):
let typInfo = genBuiltin(c, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
typInfo.typ = getSysType(c.g, c.info, tyPointer)
body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x, c.idgen), typInfo, y)
@@ -1104,17 +1080,8 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
case t.kind
of tyNone, tyEmpty, tyVoid: discard
of tyUncheckedArray:
# An UncheckedArray has no known length, so it cannot be copied, moved or
# destroyed as a value: it only ever lives behind a pointer and its bytes
# are managed manually (element ops for seqs/strings go through the
# seq/string hooks, which know the length). Emitting `x = y` for it (as the
# pointer-like group below does) produces an assignment of an unsized array,
# which the C backend cannot lower (genAssignment: tyUncheckedArray). So all
# value hooks for it are no-ops.
discard
of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCstring,
tyPtr, tyVar, tyLent:
tyPtr, tyUncheckedArray, tyVar, tyLent:
defaultOp(c, t, body, x, y)
of tyRef:
if c.g.config.selectedGC in {gcArc, gcOrc, gcYrc, gcAtomicArc}:
@@ -1254,7 +1221,6 @@ proc symDupPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttache
n[resultPos] = newSymNode(res)
result.ast = n
incl result.flagsImpl, {sfFromGeneric, sfGeneratedOp}
setHookDisamb(g, result, AttachedOpToStr[kind], typ)
proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
info: TLineInfo; idgen: IdGenerator; isDiscriminant = false): PSym =
@@ -1301,10 +1267,6 @@ proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp
if kind == attachedWasMoved:
incl result.flagsImpl, sfNoSideEffect
incl result.typ, tfNoSideEffect
if not isDiscriminant:
# discriminant destructors derive their body from the enclosing object
# AND the selected field; their key is set at the call site
setHookDisamb(g, result, AttachedOpToStr[kind], typ)
proc genTypeFieldCopy(c: var TLiftCtx; t: PType; body, x, y: PNode) =
let xx = genBuiltin(c, mAccessTypeField, "accessTypeField", x)
@@ -1397,7 +1359,6 @@ proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym,
assert(typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject)
# discrimantor assignments needs pointers to destroy fields; alas, we cannot use non-var destructor here
result = symPrototype(g, field.typ, typ.owner, attachedDestructor, info, idgen, isDiscriminant = true)
setHookDisamb(g, result, "=destroy¦" & field.name.s & "¦" & $field.position, typ)
var a = TLiftCtx(info: info, g: g, kind: attachedDestructor, asgnForType: typ, idgen: idgen,
fn: result)
a.asgnForType = typ

View File

@@ -100,7 +100,6 @@ type
warnGlobalVarConstructorTemporary = "GlobalVarConstructorTemporary",
warnImplicitRangeConversion = "ImplicitRangeConversion",
warnSystemRangeConversion = "SystemRangeConversion",
warnInvalidCmpOp = "InvalidCmpOp",
# hints
hintSuccess = "Success", hintSuccessX = "SuccessX",
hintCC = "CC",
@@ -211,7 +210,6 @@ const
warnGlobalVarConstructorTemporary: "global variable '$1' initialization requires a temporary variable",
warnImplicitRangeConversion: "implicit range conversion $1",
warnSystemRangeConversion: "implicit range conversion $1",
warnInvalidCmpOp: "$1",
hintSuccess: "operation successful: $#",
# keep in sync with `testament.isSuccess`
hintSuccessX: "$build\n$loc lines; ${sec}s; $mem; proj: $project; out: $output",

View File

@@ -378,9 +378,6 @@ proc wrongRedefinition*(c: PContext; info: TLineInfo, s: string;
conflictsWith: TLineInfo, note = errGenerated) =
## Emit a redefinition error if in non-interactive mode
if c.config.cmd != cmdInteractive:
when defined(icDbgRefc):
echo "[icRedef] ", s
echo getStackTrace()
localError(c.config, info, note,
"redefinition of '$1'; previous declaration here: $2" %
[s, c.config $ conflictsWith])
@@ -462,15 +459,6 @@ proc openShadowScope*(c: PContext) =
symbols: initStrTable(),
depthLevel: c.scopeDepth)
proc rememberShadowDefs*(c: PContext) =
## bug #25693: a template/macro operand's local definitions are sem-checked in
## a shadow scope that is then discarded. Record those definitions so that a
## later re-emission (e.g. a captured `typed` fragment expanded more than once)
## can be detected as a redefinition rather than silently miscompiled.
for s in c.currentScope.symbols:
if s.kind in {skVar, skLet, skForVar} and {sfGenSym, sfWasGenSym} * s.flags == {}:
c.shadowDiscardedDefs.incl s.id
proc closeShadowScope*(c: PContext) =
## closes the shadow scope, but doesn't merge any of the symbols
## Does not check for unused symbols or missing forward decls since a macro

View File

@@ -207,70 +207,15 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym =
if result != nil: return
else: discard
of nkSym:
if matchesDerivedFieldId(n.sym.itemId, id): result = n.sym
else: discard
proc lookupCapturedField(n: PNode, s: PSym): PSym =
## Find an env field that `addField` would have produced for the captured
## local `s`. Used as a fallback when the derived-itemId match fails because
## `s` is a macro-generated gensym whose process-local id diverges from the
## loaded env field's (see `addField`). `addField` always names a field
## `s.name & $field.position`, so that pair uniquely identifies the field for a
## local of this name without relying on the (unstable) item id.
result = nil
case n.kind
of nkRecList:
for i in 0..<n.len:
result = lookupCapturedField(n[i], s)
if result != nil: return
of nkRecCase:
if n[0].kind != nkSym: return
result = lookupCapturedField(n[0], s)
if result != nil: return
for i in 1..<n.len:
case n[i].kind
of nkOfBranch, nkElse:
result = lookupCapturedField(lastSon(n[i]), s)
if result != nil: return
else: discard
of nkSym:
if n.sym.kind == skField and n.sym.name.s == s.name.s & $n.sym.position:
result = n.sym
if n.sym.itemId.module == id.module and n.sym.itemId.item == -abs(id.item): result = n.sym
else: discard
proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym =
# Idempotent w.r.t. the captured symbol (mirrors `addUniqueField`): re-lifting
# a LOADED routine re-derives its transformed body (never serialized under IC)
# and re-captures the same locals, but the env object loaded from the NIF
# already carries their fields. Re-adding would duplicate the field and, worse,
# mutate a Sealed loaded type via `propagateToOwner` (the `t.state != Sealed`
# crash). Return the existing field instead.
let existing = lookupInRecord(obj.n, s.itemId)
if existing != nil:
return existing
# Re-lifting a LOADED routine during a VM transform (its transformed body is
# re-derived per process, never serialized) re-captures the same locals, but
# for a macro-generated gensym (e.g. libp2p `p2pProtocolBackendImpl`'s
# `msgVar`) its process-local id diverges from the one baked into the loaded
# env field, so the id match above misses. Reuse the existing same-named field
# rather than appending a divergent duplicate, which keeps the re-derived
# closure consistent (else a stale `:env` access reaches `cannotEval`).
# Confined to a loaded (Sealed) env: in a freshly built env ids are consistent,
# and two distinct same-named captures legitimately get distinct fields there.
if obj.state == Sealed:
let byName = lookupCapturedField(obj.n, s)
if byName != nil:
return byName
# Genuinely new field. Under IC the env may be a loaded Sealed type whose
# transform-time mutation is process-local (the body is discarded after the
# macro runs), so downgrade it to mutable instead of crashing on
# `t.state != Sealed` (mirrors `markAsClosure`).
unsealForTransform(obj)
# because of 'gensym' support, we have to mangle the name with its ID.
# This is hacky but the clean solution is much more complex than it looks.
var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
idgen, s.owner, s.info, s.options)
field.itemId = derivedFieldId(s.itemId)
field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
let t = skipIntLit(s.typ, idgen)
field.typ = t
if s.kind in {skLet, skVar, skField, skForVar}:
@@ -290,7 +235,7 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator)
if result == nil:
var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len), idgen,
s.owner, s.info, s.options)
field.itemId = derivedFieldId(s.itemId)
field.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
let t = skipIntLit(s.typ, idgen)
field.typ = t
assert t.kind != tyTyped
@@ -361,16 +306,6 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
assert t.kind == tyObject
result = lookupInRecord(t.n, v.itemId)
if result != nil: break
# A LOADED (Sealed) env object carries fields baked by the producer process;
# re-lifting a NIF-loaded routine in a consumer (e.g. a macro VM-evaluating an
# imported `p2pProtocolBackendImpl`) re-captures the same local under a
# divergent process-local id, so the derived-itemId match misses. Fall back to
# the name+position identity `addField` uses — SYMMETRIC with `addField`'s
# Sealed by-name reuse — so the access resolves the field `addField` produced
# instead of failing with `not part of closure object type`.
if t.state == Sealed:
result = lookupCapturedField(t.n, v)
if result != nil: break
t = t.baseClass
if t == nil: break
t = t.skipTypes(skipPtrs)

View File

@@ -29,12 +29,10 @@ when defined(nimPreviewSlimSystem):
import ../dist/checksums/src/checksums/sha1
import pipelines
from icconfig import produceIcConfig
when not defined(nimKochBootstrap):
import nifbackend
import deps
import idetools
when not defined(leanCompiler):
import docgen
@@ -417,32 +415,13 @@ proc mainCommand*(graph: ModuleGraph) =
for it in conf.searchPaths: msgWriteln(conf, it.string)
of cmdCheck:
commandCheck(graph)
of cmdTrack:
# `nim track --def:/--usages:/--track:` — IDE goto-definition / find-usages.
# Runs `nim ic`'s incremental frontend (nifler + per-module `nim m`, so only
# changed modules recompile and each writes a faithful, VM-executed `.s.bif`
# — covering stdlib too), then scans those NIF files (idetools.runIdeQuery).
# Shares the `nim ic` nimcache dir, so a prior `nim ic` build is reused.
setUseIc(true)
wantMainModule(conf)
setOutFile(conf)
when not defined(nimKochBootstrap):
commandIc(conf, frontendOnly = true)
runIdeQuery(conf)
else:
rawMessage(conf, errGenerated, "nim track not available in bootstrap build")
of cmdM:
# cmdM uses NIF files, not ROD files
graph.config.symbolFiles = disabledSf
setUseIc(true)
# vtable dispatch needs a whole-program vtable layout, which the
# per-module compilation model cannot provide (yet); methods dispatch
# through the classic if-chain dispatchers instead
excl conf.features, Feature.vtables
commandCheck(graph)
of cmdNifC:
setUseIc(true)
excl conf.features, Feature.vtables
# Generate C code from NIF files
wantMainModule(conf)
setOutFile(conf)
@@ -451,18 +430,10 @@ proc mainCommand*(graph: ModuleGraph) =
# Generate .build.nif for nifmake
setUseIc(true)
wantMainModule(conf)
# Resolve the output binary path (honoring `--out`) up front, like cmdNifC:
# the backend build file derives the link target from `conf.absOutFile`.
setOutFile(conf)
when not defined(nimKochBootstrap):
commandIc(conf)
else:
rawMessage(conf, errGenerated, "nim deps not available in bootstrap build")
of cmdIcConfig:
# Produce the precompiled config artifact for `nim ic` (config already
# parsed by the normal pipeline); a separate process spawned by the driver.
wantMainModule(conf)
produceIcConfig(conf)
of cmdParse:
wantMainModule(conf)
discard parseFile(conf.projectMainIdx, cache, conf)
@@ -476,17 +447,10 @@ proc mainCommand*(graph: ModuleGraph) =
of cmdJsonscript:
setOutFile(graph.config)
commandJsonScript(graph)
of cmdUnknown, cmdNone:
of cmdUnknown, cmdNone, cmdIdeTools:
rawMessage(conf, errGenerated, "invalid command: " & conf.command)
if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop, cmdM} and
not (conf.cmd == cmdNifC and conf.icBackendStage.len > 0):
# The IC build runs hundreds of internal per-module child processes — the
# frontend `nim m` (cmdM) and the per-module backend stages (cg/emit/merge/
# link). Each would print a `[SuccessX]` summary that is pure noise (and
# misleading: `out: unknownOutput`, or `out: <the whole compiler>` for a
# step that only wrote one `.c.nif`/`.c`). The driving `nim ic` (and koch)
# reports the real result.
if conf.errorCounter == 0 and conf.cmd notin {cmdTcc, cmdDump, cmdNop}:
if optProfileVM in conf.globalOptions:
echo conf.dump(conf.vmProfileData)
genSuccessX(conf)

View File

@@ -53,39 +53,7 @@ proc mangleParamExt*(s: PSym): string =
result.addInt s.position
proc mangleProcNameExt*(graph: ModuleGraph, s: PSym): string =
# The disambiguator comes first and the module suffix LAST, so the suffix is
# a strippable trailing token: content-addressed cross-module merging chops
# everything from the final `__` to recover a mint-site-independent name.
if s.itemId.isBackendMinted:
# A symbol minted during IC codegen (`idGeneratorForBackend`): its idgen
# starts with an EMPTY per-name disamb table, so its `disamb` restarts at 0
# and collides with same-named sem-time symbols loaded from NIFs (two
# `=destroy` hooks both mangling to `_u2` → "conflicting types for ..." in
# the generated C). Most such symbols never cross a process boundary (nifc
# lifts, emits and compiles them in one run), so the per-module-unique
# item id is a safe and deterministic discriminator; the `_c` marker keeps
# the namespace disjoint from `_u<disamb>`.
result = "_c"
if (s.disamb and HookDisambBit) != 0'i32:
# EXCEPTION: a backend-minted sym whose `disamb` is content-derived
# (setHookDisamb gave it HookDisambBit) — e.g. the `rttiDestroy` wrapper —
# DOES cross process boundaries: its C name is baked into the type's RTTI
# table, which is emit-everywhere and merge-deduped, so one process's
# `_c<item>` (a per-process backend counter) ends up referenced while the
# wrapper is defined with another's → undefined at link (`rttiDestroy_c23`).
# The content-derived disamb is stable across processes; use it.
result.addInt s.disamb
else:
result.addInt s.itemId.item
else:
result = "_u"
# Use `disamb` rather than `itemId.item`: under incremental compilation a
# symbol loaded from a NIF file gets a fresh, load-order-dependent `itemId.item`
# (from the per-module symbol counter), which is neither stable across the
# processes that compile vs. use a module nor guaranteed distinct from another
# loaded symbol's. `disamb` is assigned deterministically per (module, name)
# and, together with the already-prepended mangled name, yields a unique and
# stable C identifier.
result.addInt s.disamb
result.add "__"
result = "__"
result.add graph.ifaces[s.itemId.module].uniqueName
result.add "_u"
result.addInt s.itemId.item # s.disamb #

View File

@@ -11,7 +11,7 @@
## represents a complete Nim project. Single modules can either be kept in RAM
## or stored in a rod-file.
import std/[intsets, tables, hashes, strtabs, os, strutils, parseutils, sets]
import std/[intsets, tables, hashes, strtabs, os, strutils, parseutils]
import ../dist/checksums/src/checksums/md5
import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb
@@ -68,42 +68,6 @@ type
enumToStringProcs*: Table[ItemId, PSym]
loadedEnumToStringProcs: Table[string, PSym]
emittedTypeInfo*: Table[string, FileIndex]
instDisambs: Table[(int, int32), ItemId] # (name id, content disamb) ->
# instance, for collision probing in
# `setInstanceDisamb`
icCnifFiles*: seq[string] # `.c.nif` artifacts written by this run
pendingMethodReplays*: seq[PSym] # method registrations loaded under
# `nim nifc`, bucketed only after every
# module is loaded (`flushMethodReplays`)
icImplDeps*: IntSet # NeedsImpl edge tracking under `nim m`:
# module ids (FileIndex) whose routine BODIES
# this compilation consumed at compile time.
# Written to the `.edges` sidecar; deps.nim
# then gates the dependent on those modules'
# IMPL cookie instead of the iface cookie, so
# e.g. `const x = dep.foo()` re-sems when foo's
# body changes. Uniform across body-access
# kinds — the iface cookie hashes signatures
# ONLY (see ast2nif.cookieSd), so every body
# consumer records an edge here: VM-compiled /
# getImpl'ed bodies (recordIcImplDep from vm/
# vmgen), expanded templates (semTemplateExpr)
# and instantiated generics (generateInstance).
# Inline iterators / `inline` procs are NOT
# tracked: they are inlined at codegen, where
# the nifc backend's NIF-mtime invalidation
# already re-codegens their users.
icQualIfaces*: IntSet # module positions whose interface tables were
# populated ONLY for qualified access through a
# module re-export (`import x; export x`); the
# Iface.module stays nil so a later direct
# import still takes the full load path
inVMTransform*: int # >0 while the VM compiles a routine body
# (vmgen.genProc's transformBody): hooks lifted
# there (e.g. for closure-env types of LOADED
# routines) are process-local VM artifacts —
# serializing them would embed references to
# derived env-field syms that no module defines
packageSyms*: TStrTable
deps*: IntSet # the dependency graph or potentially its transitive closure.
@@ -146,19 +110,6 @@ type
cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented
cacheCounters*: Table[string, BiggestInt] # IC: implemented
cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
pendingNifInit*: seq[tuple[module: PSym; topLevel: PNode]]
# EVERY module loaded from a NIF — whether a direct import (moduleFromNifFile)
# or only a dep-of-a-dep (loadTransitiveHooks) — is recorded here with its
# serialized top-level AST. The sem driver drains it once
# (pipelines.finalizeLoadedModules) and applies the module's VM-level load
# effects UNIFORMLY: macro-cache replay (std/macrocache put/inc/add/incl) and
# eager `{.compileTime.}` global init. This is the single place "what a loaded
# module does to global state" lives, so a transitively-reached module — which
# never passes through compilePipelineModule — gets the SAME treatment as a
# direct import instead of silently skipping it (its macrocache state would be
# lost; its CT globals would stay nil and a macro splicing one, e.g.
# chronicles' `chroniclesBlockName`, emits `break nil` / `nil == 0`). To add a
# new per-load VM effect, extend the drain — never a parallel buffer.
passes*: seq[TPass]
pipelinePass*: PipelinePass
onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -168,22 +119,13 @@ type
strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
idgen*: IdGenerator
vmTransfIdgen*: IdGenerator # process-local backend idgen for closure envs
# minted while the VM compiles a routine body
# (inVMTransform); see lambdalifting / ast2nif @bk
operators*: Operators
cachedFiles*: StringTableRef
procGlobals*: seq[PNode]
nifReplayActions*: Table[int32, seq[PNode]] # module position -> replay actions for NIF
nifExpansions*: Table[int32, seq[(PSym, TLineInfo)]]
# module position -> (template/macro sym, call-site info) for every expansion
# in that module. Templates/macros leave no trace in the sem'checked AST, so
# this side-channel (written into the `.bif`, see ast2nif) is what lets
# `nim track --usages`/`--def` find them. Populated by `rememberExpansion`.
cachedMods: IntSet
hookClosure: IntSet # modules whose serialized hooks were already registered
TPassContext* = object of RootObj # the pass's context
idgen*: IdGenerator
@@ -293,18 +235,6 @@ iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
if s != nil:
yield s
proc reexportedModuleSyms*(g: ModuleGraph; m: PSym): seq[(string, string)] =
## (name, NIF module suffix) of MODULE syms in `m`'s interface — these are
## re-exports (`import x; export x`, added by `reexportSym`) acting as
## qualifiers (`m.x.sym`). Consumed by the NIF writer; semExport does not
## put them into the nkExportStmt children, so the AST walk cannot see them.
result = @[]
var seen = initIntSet()
for s in g.ifaces[m.position].interf.data:
if s != nil and s.kind == skModule and s.position != m.position and
not seen.containsOrIncl(s.position):
result.add (s.name.s, cachedModuleSuffix(g.config, FileIndex s.position))
proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
let importHidden = optImportHidden in m.options
result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name)
@@ -350,67 +280,21 @@ proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
result = g.loadedOps[op].getOrDefault(key)
#echo "fallback ", key, " ", op, " ", result
when defined(icDbgHash):
if result == nil and op == attachedDestructor:
echo "HOOK MISS key=", key, " table.len=", g.loadedOps[op].len,
" kind=", t.kind, " sym=", (if t.sym != nil: t.sym.name.s else: "NIL")
if key.len > 10:
let probe = key[3 ..< min(key.len, 18)]
for k in g.loadedOps[op].keys:
if probe in k: echo " candidate: ", k
else:
result = nil
proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
## we also need to record this to the packed module.
# Key-based deduplication for opsLog: different type objects (e.g. canon vs
# orig) can have different itemIds but the same structural key.
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
if g.inVMTransform > 0 and g.config.cmd == cmdM:
# hook lifted while the VM compiles a routine body (closure-env types of
# loaded routines): register it for in-process lookup but keep it out of
# the serialized log — it is a process-local artifact whose type graph
# references derived env-field syms that no module's NIF defines
if g.loadedOps[op].getOrDefault(key) == nil:
if not g.attachedOps[op].contains(t.itemId):
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
# Use key-based deduplication for opsLog because different type objects
# (e.g. canon vs orig) can have different itemIds but same structural key
if key notin g.loadedOps[op]:
# Hooks should be written to the module where the type is defined,
# not the module that triggered the registration
let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module
g.opsLog.add LogEntry(kind: HookEntry, op: op, module: ownerModule, key: key, sym: value)
g.loadedOps[op][key] = value
g.attachedOps[op][t.itemId] = value
return
let existing = g.loadedOps[op].getOrDefault(key)
if existing == nil:
# Stamp the entry with the module whose compilation produced the hook
# (`module`), NOT the type's def module: each `nim m` is a separate
# process, so a hook lifted while compiling a *downstream* module simply
# does not exist in the def module's process — stamping it with the def
# module produced a `LogEntry` that no module ever writes (the def
# module's writer ran in another process that never lifted it; this
# module's writer skips it because `op.module != thisModule`) and codegen
# failed with "'=destroy' operator not found" (e.g. astdef's `TStrTable`,
# whose destroy is first needed by modulegraphs). This holds for nominal
# types as much as for generic/structural instances. Duplicate
# registrations across lifting modules are reconciled deterministically
# at load time (see the HookEntry replay in `replayStateChanges`).
g.opsLog.add LogEntry(kind: HookEntry, op: op, module: module, key: key, sym: value)
g.loadedOps[op][key] = value
elif existing != value:
# Re-registration replacing an earlier sym for the same key. This happens
# legitimately: `createTypeBoundOps` first registers empty `symPrototype`
# placeholders, then `produceSym` replaces them — in particular
# `produceSymDistinctType` replaces a distinct type's placeholder with the
# BASE type's hook (a `distinct string` uses string's `=sink`). The log
# must follow the replacement, otherwise the NIF ships the dead,
# empty-bodied prototype and codegen in another process calls a no-op
# `=sink`/`=copy`, silently losing the value (e.g. `conf.projectPath`
# ended up empty: "cannot open '/'").
g.loadedOps[op][key] = value
var updated = false
for e in mitems(g.opsLog):
if e.kind == HookEntry and e.op == op and e.key == key:
e.sym = value
e.module = module
updated = true
break
if not updated:
g.opsLog.add LogEntry(kind: HookEntry, op: op, module: module, key: key, sym: value)
g.attachedOps[op][t.itemId] = value
proc setAttachedOp*(g: ModuleGraph; module: int; typeId: ItemId; op: TTypeAttachedOp; value: PSym) =
@@ -459,10 +343,8 @@ proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) =
g.enumToStringProcs[t.itemId] = value
let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback)
# Stamp with the module that owns the generated proc, not the enum's def
# module: the def module's process may never have generated it (same
# "written by nobody" failure as hook entries, see setAttachedOp).
g.opsLog.add LogEntry(kind: EnumToStrEntry, module: value.itemId.module.int, key: key, sym: value)
let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: value.itemId.module.int
g.opsLog.add LogEntry(kind: EnumToStrEntry, module: ownerModule, key: key, sym: value)
iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) =
if g.methodsPerGenericType.contains(t.itemId):
@@ -475,49 +357,6 @@ proc addMethodToGeneric*(g: ModuleGraph; module: int; t: PType; col: int; m: PSy
let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module
g.opsLog.add LogEntry(kind: MethodEntry, module: ownerModule, key: key, sym: m)
proc logMethodDef*(g: ModuleGraph; s: PSym) =
## Log a method registration (`cgmeth.methodDef`) so that importers and
## the backend can rebuild the dispatch buckets (`g.methods`) from the
## NIF replay log — the serialized method ast carries its dispatcher sym
## at `dispatcherPos`, so replay reuses the original dispatcher that all
## call sites reference by name (see `registerLoadedMethod`).
if g.config.cmd in {cmdNifC, cmdM}:
g.opsLog.add LogEntry(kind: MethodEntry, module: s.itemId.module.int,
key: "", sym: s)
proc registerLoadedMethod*(g: ModuleGraph; m: PSym) =
## Rebuild the dispatch buckets from a serialized method registration.
## Buckets group the methods sharing a dispatcher; the dispatcher's BODY
## does not exist in serialized form — `generateIfMethodDispatchers`
## synthesizes it in the backend from the complete bucket.
template dbg(msg: string) =
when defined(icDbgMeth):
echo "[icMeth] replay ", (if m != nil: m.name.s else: "nil"), ": ", msg
if m == nil or sfDispatcher in m.flags: dbg "skip self/nil"; return
if m.ast == nil or dispatcherPos >= m.ast.len:
dbg "no dispatcherPos (len " & $(if m.ast != nil: m.ast.len else: -1) & ")"
return
let dn = m.ast[dispatcherPos]
if dn == nil or dn.kind != nkSym or dn.sym == nil: dbg "empty dispatcher slot"; return
let disp = dn.sym
if sfDispatcher notin disp.flags: dbg "slot sym not a dispatcher"; return
dbg "ok -> bucket of " & disp.name.s & "." & $disp.disamb
for i in 0..<g.methods.len:
if g.methods[i].dispatcher.itemId == disp.itemId:
for existing in g.methods[i].methods:
if existing.itemId == m.itemId: return
g.methods[i].methods.add m
return
g.methods.add (methods: @[m], dispatcher: disp)
proc flushMethodReplays*(g: ModuleGraph) =
## Builds the dispatch buckets from the method registrations collected
## during module loading; called once every module of the program is
## loaded (`nifbackend.generateCode`).
for s in g.pendingMethodReplays:
registerLoadedMethod(g, s)
g.pendingMethodReplays.setLen 0
proc logGenericInstance*(g: ModuleGraph; inst: PSym) =
## Log a generic instance so it gets written to the NIF file.
## This is needed when generic instances are created during compile-time
@@ -526,86 +365,6 @@ proc logGenericInstance*(g: ModuleGraph; inst: PSym) =
let ownerModule = inst.itemId.module.int
g.opsLog.add LogEntry(kind: GenericInstEntry, module: ownerModule, sym: inst)
const
InstanceDisambBit* = 0x4000_0000'i32
## Set in the `disamb` of routine instances whose value is content-derived
## (see `setInstanceDisamb`); keeps them disjoint from the small counter
## range ordinary symbols draw from, so the NIF name `name.disamb.module`
## stays collision-free within a module.
proc setInstanceDisamb*(g: ModuleGraph; inst, generic: PSym;
concreteTypes: openArray[PType]) =
## Under IC, replace a fresh routine instance's counter-based `disamb` with
## a content-derived one: a hash of the generic's identity plus the
## `typeKey` of every concrete type argument — exactly the identity the
## instantiation cache compares. The instance's NIF name
## `name.disamb.modsuffix` then differs only in the module suffix when the
## same instantiation is made by different modules, which is the
## prerequisite for cross-module generic-instance merging (and gives the
## dce analysis its `offers` keys). The hash is computed once, here; it is
## never recomputed — the value travels in the serialized `disamb` field.
if g.config.cmd notin {cmdNifC, cmdM}: return
if isDefined(g.config, "icNoInstKey"): return
var key = generic.name.s
key.add '.'
key.addInt generic.disamb
key.add '.'
key.add modname(generic.itemId.module, g.config)
for t in concreteTypes:
key.add '|'
key.add typeKey(t, g.config, loadTypeCallback, loadSymCallback)
let d = toMD5(key)
var h = (int32(d[0]) or (int32(d[1]) shl 8) or (int32(d[2]) shl 16) or
(int32(d[3] and 0x3F'u8) shl 24)) or InstanceDisambBit
# Same-name hash collisions inside this process get probed to the next
# free value; the loser stays correct (its name keeps the module suffix),
# it merely won't merge cross-module.
while true:
let probe = (inst.name.id, h)
if g.instDisambs.hasKey(probe):
if g.instDisambs[probe] == inst.itemId: break
h = if h == high(int32): InstanceDisambBit else: h + 1
else:
g.instDisambs[probe] = inst.itemId
break
inst.disamb = h
const
HookDisambBit* = 0x2000_0000'i32
## Set in the `disamb` of synthesized type-bound operators and `$enum`
## procs whose value is content-derived (see `setHookDisamb`); disjoint
## from both the small counter range and the `InstanceDisambBit` range.
proc setHookDisamb*(g: ModuleGraph; hook: PSym; opName: string; typ: PType) =
## Under IC, replace a synthesized hook's counter-based `disamb` with a
## content-derived one: a hash of the operation name plus the `typeKey` of
## the type it is bound to. Counter disambs renumber whenever an *earlier*
## hook appears in a re-semmed module, so cached translation units keep
## calling the old `_u<disamb>` C name while the regenerated producer
## defines a new one — the hook flavor of the backend def-migration hole.
## With a content-derived value the hook's NIF name (and hence its C name)
## is stable as long as the type itself is unchanged.
if g.config.cmd notin {cmdNifC, cmdM}: return
if isDefined(g.config, "icNoHookKey"): return
var key = opName
key.add '|'
key.add typeKey(typ, g.config, loadTypeCallback, loadSymCallback)
let d = toMD5(key)
var h = (int32(d[0]) or (int32(d[1]) shl 8) or (int32(d[2]) shl 16) or
(int32(d[3] and 0x1F'u8) shl 24)) or HookDisambBit
# Same-name hash collisions inside this process get probed to the next
# free value (staying below InstanceDisambBit); the loser merely loses
# cross-run name stability.
while true:
let probe = (hook.name.id, h)
if g.instDisambs.hasKey(probe):
if g.instDisambs[probe] == hook.itemId: break
h = if h == InstanceDisambBit - 1'i32: HookDisambBit else: h + 1
else:
g.instDisambs[probe] = hook.itemId
break
hook.disamb = h
proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool =
let op = getAttachedOp(g, t, attachedAsgn)
result = op != nil and sfError in op.flags
@@ -623,14 +382,10 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym =
when not defined(nimKochBootstrap):
# Try to resolve from NIF for both cmdNifC and cmdM (which uses NIF files)
if g.config.cmd in {cmdNifC, cmdM}:
# First try system module (most compilerprocs are there).
# Only consult the NIF if it actually exists: under nimsuggest's cold
# cache (ideActive) system is compiled from source and has no NIF yet,
# in which case the proc is already registered in-memory and the caller
# found/falls back to it — so degrade to nil instead of asserting.
# First try system module (most compilerprocs are there)
let systemFileIdx = g.config.m.systemFileIdx
if systemFileIdx != InvalidFileIdx and not g.withinSystem and
fileExists(toNifFilename(g.config, systemFileIdx)):
if systemFileIdx != InvalidFileIdx and not g.withinSystem:
# Only try to load from NIF if the file exists (it may not during initial ic build)
result = tryResolveCompilerProc(ast.program, name, systemFileIdx)
if result != nil:
strTableAdd(g.compilerprocs, result)
@@ -642,7 +397,6 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym =
let module = g.ifaces[moduleIdx].module
if module != nil and module.name.s == "threadpool":
let threadpoolFileIdx = module.position.FileIndex
if not fileExists(toNifFilename(g.config, threadpoolFileIdx)): break
result = tryResolveCompilerProc(ast.program, name, threadpoolFileIdx)
if result != nil:
strTableAdd(g.compilerprocs, result)
@@ -789,7 +543,6 @@ proc initModuleGraphFields(result: ModuleGraph) =
result.emittedTypeInfo = initTable[string, FileIndex]()
result.cachedFiles = newStringTable()
result.cachedMods = initIntSet()
result.hookClosure = initIntSet()
proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
result = ModuleGraph()
@@ -820,15 +573,6 @@ proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} =
result = true
proc recordIcImplDep*(g: ModuleGraph; s: PSym) =
## NeedsImpl edge tracking, see `icImplDeps`. Called from the compile-time
## body consumption sites (vmgen's proc compilation, the getImpl opcodes).
## Own-module and group-member entries are filtered out when the `.edges`
## sidecar is written.
if g.config.cmd == cmdM and s != nil and s.kind in routineKinds and
s.itemId.module >= 0 and not isBackendMinted(s.itemId):
g.icImplDeps.incl module(s.itemId).int
proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b
proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
@@ -911,131 +655,9 @@ proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool =
proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
result = s.ast[bodyPos]
if result != nil and nfLazyBody in result.flags and forceLazyBodyHook != nil:
# Sanctioned body-access gate (see astdef.bodyPos): materialize the deferred
# IC body so callers may safely touch `.sons` directly, not only via `len`.
forceLazyBodyHook(result)
assert result != nil
when not defined(nimKochBootstrap):
proc registerLoadedHooks*(g: ModuleGraph; logOps: seq[LogEntry]) =
let mainSuffix = getMainModuleSuffix(ast.program)
for x in logOps:
# A dependency's NIF may carry hooks whose syms belong to the module we
# are compiling fresh (e.g. a stale NIF of that very module written by an
# earlier in-process compilation). Loading those would collide with the
# freshly semchecked hook declarations.
if mainSuffix.len > 0 and
cachedModuleSuffix(g.config, x.sym.itemId.module.FileIndex) == mainSuffix:
continue
case x.kind
of HookEntry:
# The same structural hook may be serialized by several instantiating
# modules (a generic/structural instance has no single def site, so each
# using module owns its copy). Pick one deterministic program-wide winner
# by the smaller owning-module name, so every lookup resolves to the same
# sym regardless of module load order.
let existing = g.loadedOps[x.op].getOrDefault(x.key)
if existing == nil or
cachedModuleSuffix(g.config, x.sym.itemId.module.FileIndex) <
cachedModuleSuffix(g.config, existing.itemId.module.FileIndex):
g.loadedOps[x.op][x.key] = x.sym
of EnumToStrEntry:
g.loadedEnumToStringProcs[x.key] = x.sym
of MethodEntry:
# only `methodDef` registrations (empty key) rebuild dispatch
# buckets; the `addMethodToGeneric` flavor (typeKey key) announces
# the uninstantiated generic method, which must never enter a
# bucket (methodsPerGenericType replay is still a todo).
# Under `nim nifc` the replay is deferred: building a bucket forces
# the method's body, and a body loaded mid `loadModuleDependencies`
# registers modules it references in a different path context than
# the lazy loads during codegen do (`flushMethodReplays`).
if x.key.len == 0:
if g.config.cmd == cmdNifC:
g.pendingMethodReplays.add x.sym
else:
registerLoadedMethod(g, x.sym)
else:
discard
proc loadTransitiveHooks(g: ModuleGraph; deps: seq[ModuleSuffix]) =
## Registers the serialized hooks (and enum-to-string procs) of every module
## in the import closure of `deps`. Deliberately does NOT use
## `moduleFromNifFile`: that would register the dep as a fully loaded module
## and a later direct import of it would then skip `replayStateChanges`.
var stack = deps
var interf = initStrTable()
var interfHidden = initStrTable()
while stack.len > 0:
let suffix = stack.pop()
var isKnownFile = false
let fileIdx = g.config.registerNifSuffix(string suffix, isKnownFile)
if not g.hookClosure.containsOrIncl(fileIdx.int):
let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {})
registerLoadedHooks(g, precomp.logOps)
# Record this transitively-loaded module so the sem driver applies its
# VM-level load effects (macro-cache replay + `{.compileTime.}` global init)
# exactly as for a direct import — see `pendingNifInit`. A throwaway module
# symbol (same shape as moduleFromNifFile's) gives the drain an idgen/info
# context; it is not registered, so a later direct import still loads fully.
if g.config.cmd == cmdM:
let m = PSym(kindImpl: skModule, itemId: itemId(int32(fileIdx), 0'i32),
name: getIdent(g.cache, splitFile(toFullPath(g.config, fileIdx)).name),
infoImpl: newLineInfo(fileIdx, 1, 1), positionImpl: int(fileIdx))
setOwner(m, getPackage(g.config, g.cache, fileIdx))
g.pendingNifInit.add (m, precomp.topLevel)
# Rebuild generic TYPE- and PROC-instance offers across the WHOLE closure,
# not just direct imports (`moduleFromNifFile`). An instance is frozen at
# the FIRST module to create it (in a scope where its body's symbols
# resolve unambiguously); a consumer many imports away must REUSE it rather
# than re-instantiate in its own scope, which may resolve a body symbol
# differently — a divergent `compiles()`-dependent array bound (SSZ
# `HashArray[8192, Gwei]`, type offer), or an ambiguous unqualified ident
# leaked from an unrelated import (`fromRaw` -> `SkRawPublicKeySize` from
# both `secp` and `secp256k1`, proc offer). Direct-only rebuild left the
# deep offer invisible when the clean instance lives a transitive hop away.
for off in precomp.typeOffers:
g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst
for off in precomp.genericOffers:
g.procInstCache.mgetOrPut(off.generic.itemId, @[]).add PInstantiation(
sym: off.inst, concreteTypes: off.concreteTypes,
genericParamsCount: off.genericParamsCount, compilesId: 0)
for d in precomp.deps: stack.add d
proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym =
## A re-exported MODULE (`import x; export x`) acts as a qualifier in the
## re-exporting module's interface (`asmm.x86.nd`). Reconstruct a module
## symbol for it and make its interface tables available for qualified
## lookup (`someSym` reads `g.ifaces[position]`) — WITHOUT registering
## the module: `Iface.module` stays nil so a later direct import still
## takes the full load path (replayStateChanges etc.).
var isKnown = false
let fIdx = g.config.registerNifSuffix(msuffix, isKnown)
if fIdx.int >= g.ifaces.len: setLen(g.ifaces, fIdx.int + 1)
if g.ifaces[fIdx.int].module != nil and
g.ifaces[fIdx.int].module.name.s == mname:
# properly registered already (directly imported earlier): reuse it
return g.ifaces[fIdx.int].module
result = PSym(kindImpl: skModule, itemId: itemId(int32(fIdx), 0'i32),
name: getIdent(g.cache, mname),
infoImpl: newLineInfo(fIdx, 1, 1),
positionImpl: int(fIdx))
setOwner(result, getPackage(g.config, g.cache, fIdx))
if g.ifaces[fIdx.int].module == nil and
not g.icQualIfaces.containsOrIncl(fIdx.int):
var interf = initStrTable()
var interfHidden = initStrTable()
let precomp = loadNifModule(ast.program, ModuleSuffix(msuffix),
interf, interfHidden, {})
# chains: the re-exported module may itself re-export modules
for (n2, s2) in precomp.reexportedModules:
let inner = materializeReexportedModule(g, n2, s2)
if inner != nil:
strTableAdd(interf, inner)
g.ifaces[fIdx.int].interf = interf
g.ifaces[fIdx.int].interfHidden = interfHidden
proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex;
flags: set[LoadFlag] = {}): PrecompiledModule =
## Returns 'nil' if the module needs to be recompiled.
@@ -1044,23 +666,12 @@ when not defined(nimKochBootstrap):
if not fileExists(toNifFilename(g.config, fileIdx)):
return PrecompiledModule(module: nil)
# NOTE: direction-(c) experiment (refuse to NIF-serve include-bearing modules
# under ideActive, forcing a source compile) is disabled — it reproduces the
# known sibling-resolution corruption (system.string -> excpt.nim:746). The
# cold-include *discovery* scan (scanIncludeGraph) stays; the round-trip
# fidelity of included symbols is the separate, still-open loader problem.
when false:
if g.config.ideActive and not g.withinSystem and
fileIdx != g.config.m.systemFileIdx and
nifModuleHasIncludes(g.config, fileIdx):
return PrecompiledModule(module: nil)
# Create module symbol
let filename = AbsoluteFile toFullPath(g.config, fileIdx)
let m = PSym(
kindImpl: skModule,
itemId: itemId(int32(fileIdx), 0'i32),
itemId: ItemId(module: int32(fileIdx), item: 0'i32),
name: getIdent(g.cache, splitFile(filename).name),
infoImpl: newLineInfo(fileIdx, 1, 1),
positionImpl: int(fileIdx))
@@ -1072,98 +683,25 @@ when not defined(nimKochBootstrap):
g.ifaces[fileIdx.int].interf,
g.ifaces[fileIdx.int].interfHidden, flags)
result.module = m
for (mname, msuffix) in result.reexportedModules:
let ms = materializeReexportedModule(g, mname, msuffix)
if ms != nil:
strTableAdd(g.ifaces[fileIdx.int].interf, ms)
# Re-establish include->module mapping so nimsuggest's `parentModule` can map
# a query in an included file back to this (NIF-loaded) module and recompile
# it, exactly as it does for a from-source module. Without this the include
# relationship is invisible for NIF-served modules.
for incPath in result.includes:
g.addIncludeDep(fileIdx, fileInfoIdx(g.config, AbsoluteFile incPath))
# Rebuild `procInstCache` from this module's generic-instance OFFERS so a
# consumer's `genericCacheGet` finds the instance and SKIPS re-running
# `instantiateBody` in its own module scope (which lacks symbols visible only
# at the generic's definition site — see ast2nif's `(offer …)`).
for off in result.genericOffers:
g.procInstCache.mgetOrPut(off.generic.itemId, @[]).add PInstantiation(
sym: off.inst, concreteTypes: off.concreteTypes,
genericParamsCount: off.genericParamsCount, compilesId: 0)
# Rebuild `typeInstCache` from this module's generic TYPE-instance OFFERS so a
# consumer's `searchInstTypes` reuses the baked instance (e.g. an SSZ
# `HashArray` whose array bound depends on import-scope-sensitive `compiles()`)
# rather than re-instantiating it with a divergent bound — see ast2nif's
# `(toffer …)`. Keyed by the generic body sym's itemId, as `searchInstTypes`.
for off in result.typeOffers:
g.typeInstCache.mgetOrPut(off.generic.itemId, @[]).add off.inst
# Mark module as cached
g.cachedMods.incl fileIdx.int
g.hookClosure.incl fileIdx.int
# Register hooks from NIF index with the module graph
registerLoadedHooks(g, result.logOps)
for x in result.logOps:
case x.kind
of HookEntry:
g.loadedOps[x.op][x.key] = x.sym
of ConverterEntry:
g.ifaces[fileIdx.int].converters.add x.sym
of PureEnumEntry:
# rebuild the pure-enum list (source path: `addPureEnum`) so importers can
# offer this loaded `{.pure.}` enum's fields as the restricted pure-enum
# fallback (`importPureEnumFields`).
g.ifaces[fileIdx.int].pureEnums.add x.sym
of MethodEntry:
discard "dispatch buckets already rebuilt by registerLoadedHooks"
discard "todo"
of EnumToStrEntry:
g.loadedEnumToStringProcs[x.key] = x.sym
of GenericInstEntry:
raiseAssert "GenericInstEntry should not be in the NIF index"
of HookEntry, EnumToStrEntry:
discard "already done by registerLoadedHooks"
# Register methods per type from NIF index
discard "todo"
# `nim m` loads only its *direct* imports through this proc, but a hook for
# a structural type (e.g. `=destroy` for `seq[PNode]`) lives in the NIF of
# whichever module first lifted it — possibly a dependency of a dependency
# that the current module never imports directly. Walk the whole import
# closure so every serialized hook is visible. (Codegen, `nim nifc`, already
# walks the closure in nifbackend.loadModuleDependencies.)
if g.config.cmd == cmdM:
loadTransitiveHooks(g, result.deps)
# Record the directly-loaded module for the same VM-level load effects as its
# transitive deps (`pendingNifInit`). AFTER loadTransitiveHooks so the drain
# applies deps before the dependent (macro-cache order).
g.pendingNifInit.add (m, result.topLevel)
proc isModuleFile(g: ModuleGraph; fileIdx: FileIndex): bool =
let i = fileIdx.int32
i >= 0 and i < g.ifaces.len and g.ifaces[i].module != nil
proc registerIncluderFromNif*(g: ModuleGraph; fileIdx: FileIndex): bool =
## Targeted cold-include discovery for nimsuggest: scan the nimcache NIFs
## (`scanIncludeGraph`) for a module whose include-set contains *this* file
## and register only that single include->module edge in `inclToMod`, so a
## query inside the include file resolves its includer via `parentModule`.
##
## Deliberately targeted: registering *every* include relationship (i.e. also
## `system`'s own `include`s) eagerly assigns FileIndexes and pollutes
## `inclToMod`, which perturbs the NIF line-info decode of unrelated modules
## (`system.string` then resolves into `excpt.nim`). Touch nothing but the
## one edge we need.
let target = toFullPath(g.config, fileIdx)
for (includer, includes) in scanIncludeGraph(g.config):
for incFile in includes:
if cmpPaths(incFile, target) == 0:
g.addIncludeDep(fileInfoIdx(g.config, AbsoluteFile includer), fileIdx)
return true
result = false
proc needsIncludeScan*(g: ModuleGraph; fileIdx: FileIndex): bool =
## True when `fileIdx` is neither a known module of its own nor an
## already-known include file — i.e. a cold-opened file whose includer we
## must still discover via `registerIncluderFromNif`.
not g.isModuleFile(fileIdx) and not g.inclToMod.hasKey(fileIdx)
proc configComplete*(g: ModuleGraph) =
#rememberStartupConfig(g.startupPackedConfig, g.config)
@@ -1192,16 +730,7 @@ proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
proc belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
## Check if symbol belongs to the 'stdlib' package.
# Compare the package *name* (an interned ident), not the package symbol's
# `.id`. Under per-module IC (`nim m`) the system module is loaded from a NIF
# in a process that does not compile it from source, so its package symbol is
# reconstructed with a fresh `.id` that no longer matches the freshly-interned
# package of a stdlib module compiled standalone here — making the old id
# comparison wrongly report `false` and inject `--import`ed modules into the
# stdlib. Both are canonically named `stdlib` (lib/stdlib.nimble); in a normal
# `nim c` build (system compiled from source) the ids match too, so this is a
# no-op there.
sym.getPackageSymbol.name.id == graph.systemModule.getPackageSymbol.name.id
sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId
proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): SuggestFileSymbolDatabase =
result = graph.suggestSymbols.getOrDefault(fileIdx, newSuggestFileSymbolDatabase(fileIdx, optIdeExceptionInlayHints in graph.config.globalOptions))

View File

@@ -32,7 +32,7 @@ proc newModule*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
# We cannot call ``newSym`` here, because we have to circumvent the ID
# mechanism, which we do in order to assign each module a persistent ID.
result = PSym(kindImpl: skModule, itemId: itemId(int32(fileIdx), 0'i32),
result = PSym(kindImpl: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32),
name: getModuleIdent(graph, filename),
infoImpl: newLineInfo(fileIdx, 1, 1))
if not isNimIdentifier(result.name.s):

View File

@@ -125,25 +125,12 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
var dummy: bool = false
result = fileInfoIdx(conf, filename, dummy)
proc expandOrPseudo(filename: string): AbsoluteFile =
# `expandFilename` raises OSError when the path does not exist on disk. That is
# fine for a real source path, but a macro can legitimately set a node's
# line-info file to a name that has no file behind it — e.g. the `???` sentinel
# produced by `toFilename` for a NIF-loaded node whose `fileIndex` is unknown
# (FileIndex(-1)). Falling back to the raw name lets the `AbsoluteFile` overload
# register it as a pseudo-path (like `command line`/`stdin`) instead of crashing
# the whole `nim m` child with an unhandled OSError.
try:
result = AbsoluteFile expandFilename(filename)
except OSError:
result = AbsoluteFile filename
proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile; isKnownFile: var bool): FileIndex =
fileInfoIdx(conf, expandOrPseudo(filename.string), isKnownFile)
fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), isKnownFile)
proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile): FileIndex =
var dummy: bool = false
fileInfoIdx(conf, expandOrPseudo(filename.string), dummy)
fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), dummy)
proc registerNifSuffix*(conf: ConfigRef; suffix: string; isKnownFile: var bool): FileIndex =
result = conf.m.filenameToIndexTbl.getOrDefault(suffix, InvalidFileIdx)
@@ -351,7 +338,7 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
## This is used for 'nim dump' etc. where we don't have nimsuggest
## support.
#if conf.ideActive and optCDebug notin gGlobalOptions: return
#if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
let sep = if msgNoUnitSep notin flags: conf.unitSep else: ""
if not isNil(conf.writelnHook) and msgSkipHook notin flags:
conf.writelnHook(s & sep)
@@ -457,8 +444,8 @@ To create a stacktrace, rerun compilation with './koch temp $1 <file>', see $2 f
proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string, ignoreMsg: bool) =
if msg in fatalMsgs:
if conf.ideActive: log(s)
if not conf.ideActive or msg != errFatal:
if conf.cmd == cmdIdeTools: log(s)
if conf.cmd != cmdIdeTools or msg != errFatal:
quit(conf, msg)
if msg >= errMin and msg <= errMax or
(msg in warnMin..hintMax and msg in conf.warningAsErrors and not ignoreMsg):
@@ -472,7 +459,7 @@ proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string,
raiseRecoverableError(s)
else:
quit(conf, msg)
elif eh == doAbort and not conf.ideActive:
elif eh == doAbort and conf.cmd != cmdIdeTools:
quit(conf, msg)
elif eh == doRaise:
raiseRecoverableError(s)
@@ -503,7 +490,7 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
info = context.info
proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
msg >= errGenerated and conf.ideActive and optIdeDebug notin conf.globalOptions
msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions
proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
conf.m.fileInfos[fileIdx.int32].lines.add line
@@ -524,9 +511,6 @@ proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
## 1-based index (matches editor line numbers); 1st line is for i.line = 1
## last valid line is `numLines` inclusive
if i.fileIndex.int32 < 0: return ""
# line 0 means "unknown": nodes synthesized from an IC-loaded template or
# macro body carry no source position.
if i.line.int < 1: return ""
let num = numLines(conf, i.fileIndex)
# can happen if the error points to EOF:
if i.line.int > num: return ""
@@ -661,7 +645,7 @@ proc warningDeprecated*(conf: ConfigRef, info: TLineInfo = gCmdLineInfo, msg = "
message(conf, info, warnDeprecated, msg)
proc internalErrorImpl(conf: ConfigRef; info: TLineInfo, errMsg: string, info2: InstantiationInfo) =
if (conf.ideActive or conf.cmd == cmdCheck) and conf.structuredErrorHook.isNil: return
if conf.cmd in {cmdIdeTools, cmdCheck} and conf.structuredErrorHook.isNil: return
writeContext(conf, info)
liMessage(conf, info, errInternal, errMsg, doAbort, info2)

View File

@@ -17,51 +17,18 @@
## 1. Compile modules to NIF: nim m mymodule.nim
## 2. Generate C from NIF: nim nifc myproject.nim
import std/[intsets, tables, sets, os, algorithm, syncio, times, strutils]
import std/[intsets, tables, sets, os]
when defined(nimPreviewSlimSystem):
import std/assertions
import ast, options, lineinfos, modulegraphs, cgendata, cgen,
pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif, typekeys,
cnif, icmodnames
from cgmeth import generateIfMethodDispatchers
from transf import transformBody
from injectdestructors import injectDestructorCalls
import ic / replayer
pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif
proc systemNifSuffix(conf: ConfigRef): string =
## The system module's NIF suffix, derived from `system.nim`'s path EXACTLY as
## the frontend derives it (deps.nim's `toPair` on `libpath/system.nim`), so the
## backend loads the very `.s.bif` the frontend wrote. It must NOT be a constant:
## `moduleSuffix` (icmodnames) now hashes the absolute path, so the system suffix
## is install-dependent (was hardcoded `sysma2dyk`, valid only for the old
## relative-path scheme where `system.nim` always relativized to `system.nim`).
moduleSuffix((conf.libpath / RelativeFile"system.nim").string,
cast[seq[string]](conf.searchPaths))
proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex;
nifFiles: var seq[string];
depFlags: set[LoadFlag] = {LoadFullAst}): seq[PrecompiledModule] =
proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[PrecompiledModule] =
## Traverse the module dependency graph using a stack.
## Returns all modules that need code generation, in dependency order.
##
## The main module is always loaded with its full AST (it is the codegen
## target). `depFlags` governs the rest: the whole-program backend needs every
## module's full AST (it generates code for all of them), but a per-module
## stage codegens only one target, so it loads the others interface-only
## (`depFlags = {}`) — the interface, hooks, methods and the `(replay ...)`
## directives are loaded regardless of `LoadFullAst`, and demanded bodies are
## fetched lazily from the kept-open stream, so the per-module proc-body ASTs
## (the bulk of the memory) are never materialized for non-targets.
# The main module is loaded by its SOURCE FileIndex, but its serialized
# symbols carry the module's NIF suffix. Pre-alias the suffix to the source
# index so that `registerNifSuffix` does not allocate a second FileIndex for
# the same module, which would split its codegen across two C translation
# units (top-level globals in one, procs in the other → undeclared symbols).
g.config.m.filenameToIndexTbl[cachedModuleSuffix(g.config, mainFileIdx)] = mainFileIdx
let mainModule = moduleFromNifFile(g, mainFileIdx, {LoadFullAst})
nifFiles.add toNifFilename(g.config, mainFileIdx)
var stack: seq[ModuleSuffix] = @[]
result = @[]
@@ -79,10 +46,9 @@ proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex;
if not visited.containsOrIncl(suffix.string):
var isKnownFile = false
let fileIdx = g.config.registerNifSuffix(suffix.string, isKnownFile)
let precomp = moduleFromNifFile(g, fileIdx, depFlags)
let precomp = moduleFromNifFile(g, fileIdx, {LoadFullAst})
if precomp.module != nil:
result.add precomp
nifFiles.add toNifFilename(g.config, fileIdx)
for dep in precomp.deps:
if not visited.contains(dep.string):
stack.add dep
@@ -96,13 +62,7 @@ proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule =
## Set up a BModule for code generation from a NIF module.
if g.backend == nil:
g.backend = cgendata.newModuleList(g)
result = cgen.newModule(BModuleList(g.backend), module, g.config, idGeneratorForBackend(module))
proc isMetaIter(t: PType, closure: RootRef): bool =
# openArray/varargs hooks are sem bookkeeping: no real flow ever demands
# them, and generating one pollutes the TU's type cache with a struct
# descriptor for what must remain a (ptr, len) parameter expansion
t.kind in tyMetaTypes + {tyTyped, tyUntyped, tyNone, tyVarargs, tyOpenArray}
result = cgen.newModule(BModuleList(g.backend), module, g.config, idGeneratorFromModule(module))
proc finishModule(g: ModuleGraph; bmod: BModule) =
# Finalize the module (this adds it to modulesClosed)
@@ -110,114 +70,9 @@ proc finishModule(g: ModuleGraph; bmod: BModule) =
let initStmt = newNode(nkStmtList)
finalCodegenActions(g, bmod, initStmt)
# NB: the method dispatchers are emitted in `emitMethodDispatchers`,
# between the module loop and this finish loop: their bodies demand the
# method definitions, which can in turn demand definitions from modules
# the backend never loaded — and a TU demand-created during the LAST
# finishModule call would miss `modulesClosed` and never be written.
proc emitMethodDispatchers(g: ModuleGraph) =
## Synthesizes the method dispatcher bodies from the replayed dispatch
## buckets (`registerLoadedMethod`) and emits their definitions into the
## main TU. Main is regenerated on every run, so a dispatcher — whose
## body enumerates the whole program's method set — can never go stale
## inside a cached TU; cross-TU callers prototype it (see genProcLvl3).
let bl = BModuleList(g.backend)
var mainMod: BModule = nil
for m in bl.mods:
if m != nil and m.module != nil and sfMainModule in m.module.flags:
mainMod = m
break
if mainMod == nil: return
generateIfMethodDispatchers(g, mainMod.idgen)
# Generate dispatcher methods
for disp in getDispatchers(g):
if not containsOrIncl(mainMod.declaredThings, disp.id):
genProcLvl3(mainMod, disp)
proc signatureHasMetaType(t: PType; depth: int = 0): bool =
## Whether a routine signature mentions a compile-time/meta element type
## (`typed`/`untyped` — e.g. `echo`'s `varargs[typed]` — typedesc, static,
## generic param). Such routines are expanded at their call sites and never
## emitted standalone, so the per-module owned-routine seeding must skip them
## (`getTypeDescAux(tyTyped)` otherwise). `tfHasMeta` alone misses the varargs
## element case, hence the explicit scan.
result = false
if t == nil or depth > 8: return false
if t.kind == tyGenericBody:
# The uninstantiated template carried as a `tyGenericInst`'s first child
# always mentions its `tyGenericParam` placeholders, but the instance
# itself is fully concrete (e.g. `var CountTable[SigHash]`). Descending
# here would wrongly flag every routine with a generic-instance parameter
# as meta and drop it from the owned-routine seeding -> undefined symbols
# at link (its only definer never emits it).
return false
if t.kind == tyStatic:
# A RESOLVED static value (the `256` in `MDigest[256]`, the `N` in
# `HashList[T, N]`, …) is carried as a `tyStatic` node inside the otherwise
# fully-concrete `tyGenericInst`, but it is NOT meta: the routine is a normal
# runtime routine the owner must emit. Only an UNRESOLVED `static T` parameter
# (no bound value, `t.n == nil`) is meta. Without this, every routine whose
# signature touches a `static`-parameterized generic instance (the bulk of
# the SSZ/`MDigest` API) is dropped from the owned-routine seeding and ends up
# an undefined reference at link (mirrors the tyGenericBody case above).
return t.n == nil
if t.kind in {tyTyped, tyUntyped, tyTypeDesc, tyGenericParam,
tyAnything, tyFromExpr, tyError}:
return true
for k in t.kids:
if signatureHasMetaType(k, depth + 1): return true
proc ownsRuntimeRoutine(s: PSym; modPos: int): bool =
## A concrete, non-generic, runtime routine with a real body, OWNED by the
## module at `modPos`. Shared by the `cg` stage's owned-routine seeding (so a
## routine called only from other modules is still emitted by somebody) and
## the `lower` stage's owned-routine enumeration, so both stages see exactly
## the same set. The exclusions:
## - nested/closure procs (owner is a proc, not a module): emitted via their
## enclosing routine's lambda-lifting, never standalone;
## - generic instances (`sfFromGeneric`): emitted by demand, deduped by merge;
## - `importc`/`compileTime`/`error`/forward sentinels and meta signatures:
## not real codegen targets.
## - method DISPATCHERS (`sfDispatcher`): their bodies are (re)synthesized into
## the main TU by `emitMethodDispatchers`/`generateIfMethodDispatchers`, never
## per module. A dispatcher is a `copySym` clone of the method that shares the
## method's body sub-tree (incl. its closure iterator); transforming it here
## would lambda-lift that SHARED iterator a SECOND time under a different owner
## identity, baking a conflicting `up` field → "up references do not agree"
## (the divergence is impossible in non-IC, where the dispatcher body is empty
## at lift time). So a dispatcher is never an owned runtime routine.
## A `{.closure.}` iterator IS a standalone runtime routine (unlike an inline
## iterator, which is expanded at each call site) and must be emitted by its
## owner — else a cross-module `for` over it links to nothing.
##
## Generic INSTANCES (`sfFromGeneric`) are NEVER an owned runtime routine — not
## in `cg` and not in the `lower` stage. They are demanded by the backend's
## emit-everywhere path and deduped by `merge` (content C name); the frontend
## materialises them through the `(offer)` mechanism. The `lower` stage must
## not transform an instance: a not-fully-concrete instance (a closure factory
## over a `static` param, or a `$`/`=` op instance whose body resolves only at
## its further-specialised use sites) still carries unresolved overload choices
## and crashes `transformBody` (empty-`namePos` lambda, nil-typed const-fold).
s.itemId.module == modPos and
(s.kind in {skProc, skFunc, skConverter, skMethod} or
(s.kind == skIterator and s.typ != nil and s.typ.callConv == ccClosure)) and
s.skipGenericOwner != nil and s.skipGenericOwner.kind == skModule and
s.magic == mNone and
sfFromGeneric notin s.flags and
sfDispatcher notin s.flags and
{sfForward, sfImportc, sfCompileTime, sfError} * s.flags == {} and
s.typ != nil and not signatureHasMetaType(s.typ) and
s.ast != nil and s.ast.safeLen > bodyPos and
s.ast[genericParamsPos].kind == nkEmpty
# NOTE: an `nkEmpty` body is NOT a disqualifier. A concrete, owned, non-
# forward/-importc/-magic routine whose body folds to nothing is still a real
# definition the owner must emit (`void f(void){}`), exactly as whole-program
# cgen does — else a cross-module caller links to nothing. This bites e.g.
# Nimbus' `extras.incInternalErrors`, a plain `proc` whose sole statement is a
# metrics-counter `.inc()` that the `metrics` library expands to a no-op when
# the importing tool (ncli) builds with `-u:metrics`; the body is then a bare
# `nkEmpty`, but `state_transition_epoch` still calls it. Forward declarations
# (the other empty-body case) carry `sfForward` and are excluded above.
genProcLvl3(bmod, disp)
proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
## Generate C code for a single module.
@@ -226,659 +81,76 @@ proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) =
if bmod == nil:
bmod = setupNifBackendModule(g, precomp.module)
# Apply the module's recorded C compile/link directives (passl/passc/...)
# before generating code: the link step needs them (e.g. math's -lm).
replayBackendActions(g, precomp.module, precomp.topLevel)
# Generate code for the module's top-level statements
if precomp.topLevel != nil:
cgen.genTopLevelStmt(bmod, precomp.topLevel)
# Per-module backend: emit the bodies of the routines this module OWNS, not
# only the ones its top-level happens to demand. Procs are serialized as lazy
# `(sd ...)` defs (never as `nkProcDef` statements), so `genTopLevelStmt` never
# reaches them; a routine called only from *other* modules would otherwise be
# emitted by nobody, because every module now merely prototypes its foreign
# callees instead of funnelling their bodies (see `cgen.emitsBodyInThisModule`).
# The merge stage's DCE drops whatever turns out globally dead.
if g.config.cmd == cmdNifC and g.config.icBackendStage == "cg":
let modPos = precomp.module.position
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if ownsRuntimeRoutine(s, modPos):
requestProcDef(bmod, s)
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.
## Traverses the module dependency graph and generates C code.
proc loadBackendModules(g: ModuleGraph; mainFileIdx: FileIndex):
tuple[modules: seq[PrecompiledModule], precompSys: PrecompiledModule,
nifFiles: seq[string]] =
## Shared by the per-module `cg` and `emit` stages: load system + the main
## module's whole import closure and set up a `BModule` for each, so every
## type/symbol resolves and `getCFile` yields the same path both stages use.
## The main module is loaded by its source index (its NIF suffix is aliased to
## it in `loadModuleDependencies`), so it gets exactly one `BModule`.
##
## Only the main module — the codegen target of the stages that use this — is
## loaded with its full AST; every other module is loaded interface-only so
## the whole program's proc bodies are not materialized into this process (that
## was ~1.8 GB for the compiler's main `cg`). The `link` stage codegens nothing
## and only needs each module's `(replay ...)` directives, which load anyway.
# Reset backend state
resetForBackend(g)
var isKnownFile = false
let systemFileIdx = registerNifSuffix(g.config, systemNifSuffix(g.config), isKnownFile)
g.config.m.systemFileIdx = systemFileIdx
var precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface})
g.systemModule = precompSys.module
if precompSys.module != nil:
# The precompiled-load path does not restore `sfSystemModule` (mirror of the
# `sfMainModule` re-add above). `registerReusedModuleToMain` keys on it to put
# the system module's init right after its datInit AND to emit
# `initStackBottomWith` into `mainDatInit` — so that the main thread's stack
# bottom is set before any module's init runs. Without the flag the system
# init is mis-routed into the regular `otherModsInit` bucket and
# `initStackBottomWith` is never registered, so a GC cycle during a module's
# init (under refc) scans the stack with a nil bottom and crashes.
incl precompSys.module.flagsImpl, sfSystemModule
var nifFiles: seq[string] = @[toNifFilename(g.config, systemFileIdx)]
var modules = loadModuleDependencies(g, mainFileIdx, nifFiles, depFlags = {})
# loadModuleDependencies traverses the project's import closure and stops at
# system. The whole-program backend then demand-loads system's own closure
# (locks, allocators, threads, …) during codegen; the per-module backend
# instead makes every one of those a first-class cg/emit target, so load that
# closure here too — otherwise `findTargetModule` cannot resolve their suffix.
block:
var visited = initHashSet[string]()
visited.incl systemNifSuffix(g.config)
for m in modules:
visited.incl cachedModuleSuffix(g.config, FileIndex m.module.position)
var stack: seq[ModuleSuffix] = @[]
if precompSys.module != nil:
for dep in precompSys.deps: stack.add dep
while stack.len > 0:
let suffix = stack.pop()
if not visited.containsOrIncl(suffix.string):
var isKnown = false
let fileIdx = registerNifSuffix(g.config, suffix.string, isKnown)
let precomp = moduleFromNifFile(g, fileIdx, {})
if precomp.module != nil:
modules.add precomp
nifFiles.add toNifFilename(g.config, fileIdx)
for dep in precomp.deps: stack.add dep
flushMethodReplays(g)
for m in modules:
discard setupNifBackendModule(g, m.module)
if precompSys.module != nil:
discard setupNifBackendModule(g, precompSys.module)
result = (modules, precompSys, nifFiles)
proc loadDepClosure(g: ModuleGraph; targetSuffix: string):
tuple[modules: seq[PrecompiledModule], precompSys: PrecompiledModule,
target: PrecompiledModule] =
## Per-module `cg`/`emit` for a NON-main target: load system + the target
## module + the target's transitive import closure ONLY — not the whole
## program. This is the "process the one file it is passed" model (à la
## Nimony's `hexer c file.nif`): the foreign symbols the target's codegen
## demands are loaded lazily by `ast2nif.moduleId`, which opens any referenced
## module's NIF index on first touch, so a body in a not-loaded module still
## resolves. The closure is loaded as full `BModule`s only so that the
## incidental `g.mods[pos]` accesses during codegen resolve; system's own
## internal closure (allocators, locks, …) is included because a target's
## emit-everywhere codegen can demand those without importing them directly.
##
## The whole program is no longer loaded in this process, which is what bounds
## per-process memory under nifmake's parallel fan-out (the main module's `cg`,
## which still loads everything for NimMain's init list and the method
## dispatchers, runs essentially alone since every other `.c.nif` precedes it).
resetForBackend(g)
var isKnownFile = false
let systemFileIdx = registerNifSuffix(g.config, systemNifSuffix(g.config), isKnownFile)
let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile)
g.config.m.systemFileIdx = systemFileIdx
let precompSys = moduleFromNifFile(g, systemFileIdx, {AlwaysLoadInterface})
#msgs.fileInfoIdx(g.config,
# g.config.libpath / RelativeFile"system.nim")
# Load system module first - it's always needed and contains essential hooks
var precompSys = PrecompiledModule(module: nil)
precompSys = moduleFromNifFile(g, systemFileIdx, {LoadFullAst, AlwaysLoadInterface})
g.systemModule = precompSys.module
var modules: seq[PrecompiledModule] = @[]
var visited = initHashSet[string]()
visited.incl systemNifSuffix(g.config)
# Only the target is codegen'd, so only it needs its full AST; the closure is
# loaded interface-only (demanded bodies come lazily from the kept-open
# streams), which is what keeps a per-module process light under parallel fan-out.
var isKnown = false
let targetIdx = registerNifSuffix(g.config, targetSuffix, isKnown)
let target = moduleFromNifFile(g, targetIdx, {LoadFullAst})
visited.incl targetSuffix
var stack: seq[ModuleSuffix] = @[]
if target.module != nil:
modules.add target
for dep in target.deps: stack.add dep
if precompSys.module != nil:
for dep in precompSys.deps: stack.add dep
while stack.len > 0:
let suffix = stack.pop()
if not visited.containsOrIncl(suffix.string):
var isKnown2 = false
let fileIdx = registerNifSuffix(g.config, suffix.string, isKnown2)
let precomp = moduleFromNifFile(g, fileIdx, {})
if precomp.module != nil:
modules.add precomp
for dep in precomp.deps: stack.add dep
flushMethodReplays(g)
for m in modules:
discard setupNifBackendModule(g, m.module)
if precompSys.module != nil:
discard setupNifBackendModule(g, precompSys.module)
result = (modules, precompSys, target)
proc findTargetModule(g: ModuleGraph; modules: seq[PrecompiledModule];
precompSys: PrecompiledModule; suffix: string): PrecompiledModule =
## The loaded module whose NIF suffix is `suffix` (the `--icBackendModule`
## value), or a nil module if none matches.
result = PrecompiledModule(module: nil)
for m in modules:
if cachedModuleSuffix(g.config, FileIndex m.module.position) == suffix:
return m
if precompSys.module != nil and
cachedModuleSuffix(g.config, FileIndex precompSys.module.position) == suffix:
return precompSys
proc setNestedClosureBodies(g: ModuleGraph; idgen: IdGenerator; n: PNode;
owner: PSym; seen: var IntSet) =
## A closure routine nested in `owner` (the `:anonymous` proc lambda-lifting
## minted, plus any deeper nesting) gets its captured-var→env rewrite produced
## as part of the OWNER's `transformBody`. The nested proc is a module-indexed
## sym whose `.s.nif` sdef carries its PRE-lift body, so without help the whole
## module re-serializer would write that pre-lift body and cg would lose the
## capture mapping (it accesses `x` directly instead of `ClE_0->x0`). Walk the
## owner's transformed body and cache each nested closure's transformed body on
## its sym so `writeSymDef` serializes the lifted body into the routine's
## 2-way-body slot.
if n == nil: return
if n.kind == nkSym:
let s = n.sym
if s != nil and s.kind in routineKinds and s != owner and
s.skipGenericOwner != nil and s.skipGenericOwner.kind != skModule and
not seen.containsOrIncl(s.id):
# Covers ALL nested routines, not only ccClosure ones. A NIMCALL nested proc
# the async transform mints (e.g. workNimAsyncContinue) already has its
# lifted body set by the OWNER's transformBody, but it is NOT in the owned
# loop (owner is a proc, not the module). Without injecting it HERE it is
# serialized transform-only; cg loads it (wasLoaded) and skips injection, so
# a closure-env store stays a raw field assign with no incref -> the env is
# freed before the async callback runs -> "yielded nil". `seen` (shared
# across the owned loop) injects each routine exactly once.
if s.ast != nil and getBody(g, s).kind != nkEmpty:
# Only ccClosure routines are safe to `transformBody` standalone here; a
# nimcall nested proc already has its lifted body from the owner's lift,
# and transforming an arbitrary nested routine with no cached body crashes
# (not in a standalone-transformable state).
let weTransformed = s.transformedBody == nil and
s.typ != nil and s.typ.callConv == ccClosure
if weTransformed:
s.transformedBody = transformBody(g, idgen, s, {})
if s.transformedBody != nil:
# Inject destructors so cg loads a fully-lowered body and never rebuilds
# (mirrors non-IC, which injects every nested proc separately). The
# importer `n2` skField collision this used to trigger is fixed at the
# NIF-naming layer (toNifSymName gives derived env fields a unique
# disamb), so injecting ccClosure nested procs here is safe.
if sfInjectDestructors in s.flags:
s.transformedBody = injectDestructorCalls(g, idgen, s, s.transformedBody)
setNestedClosureBodies(g, idgen, s.transformedBody, s, seen)
else:
for i in 0 ..< n.safeLen:
setNestedClosureBodies(g, idgen, n[i], owner, seen)
proc reownFromTwin(n: PNode; twin, s: PSym) =
## Re-own to `s` every entity the frontend attributed to `s`'s forward-decl
## `twin` (found via the result's owner). lambda-lifting compares owners by
## reference, so a twin-owned `result` is rejected as `illegalCapture`
## ("'result' ... cannot be captured") and, once that is fixed, twin-owned
## locals go missing from `s`'s env ("environment misses: ..."). Both are
## pervasive on chronos `{.async.}` methods. Re-owning to `s` matches the
## single-sym non-IC case. `twin` is ONE specific sym, so only THIS routine's
## result-twin-owned entities match — re-owning entities of OTHER same-name
## twins proved too blunt (it disrupts env construction and reintroduces the
## very capture errors it should fix). `n.sym != s` guards self-ownership.
if n == nil: return
if n.kind == nkSym and n.sym != nil and n.sym != s and n.sym.owner == twin:
setOwner(n.sym, s)
for i in 0 ..< n.safeLen:
reownFromTwin(n[i], twin, s)
proc generateLowerStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend lowering (`--icBackendStage:lower --icBackendModule:<suffix>`):
## enumerate the routines this module OWNS and write them to `<module>.t.nif`.
## Eventually this transforms each owned routine once, in the owner's id space,
## so `cg` reads the result instead of re-deriving it (re-derivation per
## parallel `cg` process is the root of the closure-`:env` identity drift).
## Runs per module in parallel on the shallow backend dep-graph — NOT folded
## into the dense, mostly-serial sem stage.
##
## gate `newSymNode`'s lazy-type marking to the backend (see astdef) — the
## transform builds sym nodes off not-yet-typed stubs, exactly as the `cg`
## stage does.
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
else:
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: module not found for suffix: " & g.config.icBackendModule)
return
let modPos = target.module.position
let tb = BModuleList(g.backend).mods[modPos]
if tb == nil:
rawMessage(g.config, errGenerated,
"per-module lowering: no backend module for suffix: " & g.config.icBackendModule)
return
# Transform every owned routine ONCE in this single process's id space and
# re-serialize the ENTIRE module as a proper indexed NIF (`writeLoweredModule`)
# with the transformed bodies baked into the routine `(sd)` entries. `cg` loads
# it through the normal module loader, so nested procs (incl. async state
# machines) arrive as real defs with their lifted bodies — no re-derivation.
# This single-writer-per-owner is what keeps closure-`:env` identity stable
# across the parallel `cg` processes (re-derivation per process was the root of
# the `:env` identity drift). `transformBody` with flags {} mirrors the cg call
# (cgen.nim); `injectDestructorCalls` is NOT run here — it stays in `cg` on the
# loaded body.
#
# `transformBody`/lambda-lifting LIFTS the closure env's type-bound ops
# (`=destroy` etc.) into `g.opsLog`; snapshot its length so we serialize exactly
# the ops THIS stage created (not those loaded from `.s.nif`).
let opsLogStart = g.opsLog.len
# Shared across the owned loop so a nested routine reachable from more than one
# owner is transformed + destructor-injected EXACTLY once (double injection
# would emit two `=destroy`/`=copy` runs).
var seenNested = initIntSet()
for s in moduleSymbolStubs(ast.program, FileIndex modPos):
if ownsRuntimeRoutine(s, modPos):
# REUSE path (`icReuseSemLowering` ON): a routine already transformed during
# sem (CT eval / macro / VM transform) carries its lowered body in the
# `.s.nif` slot (loaded into `transformedBody`) — don't re-transform it.
# Default OFF: the slot is never loaded (see loadSymFromCursor), so
# `transformedBody` is nil here and we always re-derive below. See
# doc/ic_backend_simplify.md §6a/§6b.
if icReuseSemLowering(g.config) and s.transformedBody != nil: continue
# A routine serialized as a forward-decl + impl pair (writeSymDef's
# "separate forward declaration and implementation") loads as TWO syms; the
# impl `s` we transform here can carry body entities (`result`, locals,
# nested routines) owned by its fwd-decl TWIN, not by `s`. lambda-lifting
# compares owners by reference → `illegalCapture` rejects a twin-owned
# `result` and the lifting pass can't find twin-owned locals in `s`'s env.
# Pervasive on chronos `{.async.}` methods. Re-own them to `s`, matching the
# single-sym non-IC case. Backend-only, so frontend effect/exception
# inference is untouched.
if s.ast != nil and s.ast.len > resultPos and
s.ast[resultPos].kind == nkSym and s.ast[resultPos].sym.owner != s:
reownFromTwin(s.ast, s.ast[resultPos].sym.owner, s)
# Retain the transformed body on the sym so `writeSymDef` serializes it in
# the routine's `(sd)` 2-way-body slot.
s.transformedBody = transformBody(g, tb.idgen, s, {})
# Run the destructor injection HERE so the `.t.bif` body is FULLY lowered:
# `injectDestructorCalls` is demand-driven (it decides where destructors go
# by move analysis) and LIFTS the type-bound ops it needs (e.g. a nested
# closure env's `=destroy`) into `g.opsLog` — which the `hooks` collection
# below then serializes. Done in `cg` instead, those ops were lifted per-cg
# process, owned by nobody, and emitted as a prototype-only → undefined at
# link (the `eqdestroy__c<n>` gap). cg must NOT re-inject a loaded body
# (see genProcLvl3's `wasLoaded` gate) so this stays the single injection.
if sfInjectDestructors in s.flags:
s.transformedBody = injectDestructorCalls(g, tb.idgen, s, s.transformedBody)
# Cache the lifted+injected body on nested ccClosure routines too, so a
# module-indexed nested closure serializes its lifted (capture-rewritten,
# destructor-injected) body.
setNestedClosureBodies(g, tb.idgen, s.transformedBody, s, seenNested)
# Collect the hooks this stage lifted, and transform each hook ROUTINE's body
# too (it is itself lowered into NIFC). The hooks' `(sd)` + transformed body go
# into the `.t.nif`; `cg` re-attaches them so `injectDestructorCalls` resolves
# the loaded env's `=destroy`. Iterate to a fixpoint: a hook body can lift
# further hooks (a field's `=destroy`).
var hooks: seq[LogEntry] = @[]
var i = opsLogStart
while i < g.opsLog.len:
let e = g.opsLog[i]
if e.kind == HookEntry and e.sym != nil and e.sym.kind in routineKinds and
e.sym.transformedBody == nil:
hooks.add e
# Transform the hook routine's body and cache it on the sym so `writeSymDef`
# serializes it in the hook's `(sd)` transformed-body slot (`transformBody
# {}` returns the body but does not cache it). Inject the hook's own
# destructors here too (it can destroy fields/temporaries) so cg loads a
# fully-lowered hook and never re-injects.
e.sym.transformedBody = transformBody(g, tb.idgen, e.sym, {})
if sfInjectDestructors in e.sym.flags:
e.sym.transformedBody = injectDestructorCalls(g, tb.idgen, e.sym, e.sym.transformedBody)
inc i
# Re-serialize the whole module to its suffix-based `.t.nif` (the path
# `toNifFilename` resolves for the cg/emit stages). `writeLoweredModule` seals
# routines itself.
let suffix = cachedModuleSuffix(g.config, FileIndex modPos)
let wholeArtifact = toGeneratedFile(g.config, AbsoluteFile(suffix), ".t.bif").string
writeLoweredModule(ast.program, g.config, target, hooks, wholeArtifact)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icLower] " & extractFilename(wholeArtifact) & " " &
$hooks.len & " hooks"
proc visitDep(suffix: string;
suffixToMod: Table[string, PrecompiledModule];
visited: var HashSet[string]; bl: BModuleList;
ordered: var seq[BModule]) =
## Post-order DFS over a module's import closure used to reconstruct the
## dependency (init) order: a dependency's init must be registered before its
## importer's. Appends each reachable non-main module's `BModule` to `ordered`.
if visited.containsOrIncl(suffix): return
let pm = suffixToMod.getOrDefault(suffix)
if pm.module == nil: return
for dep in pm.deps: # dependencies first (post-order)
visitDep(dep.string, suffixToMod, visited, bl, ordered)
if sfMainModule notin pm.module.flags:
let bm = bl.mods[pm.module.position]
if bm != nil: ordered.add bm
proc generateCgStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend codegen (`--icBackendStage:cg --icBackendModule:<suffix>`):
## generate C for the single module named by `icBackendModule` and write only
## its `.c.nif` artifact (no merge, no `.c` render, no cc/link — those are
## separate nifmake rules).
##
## `findPendingModule` routes every demand into the target (emit-everywhere).
##
## A NON-main target loads only its own import closure (`loadDepClosure`); the
## whole program is no longer pulled into every parallel `cg` process. The main
## module still loads everything (`loadBackendModules`) because NimMain's init
## list and the method dispatchers are whole-program; its `cg` runs essentially
## alone (every other `.c.nif` precedes it), so it does not contend for memory.
# gate `newSymNode`'s lazy-type marking to this stage only (see astdef)
nifcBackendActive = true
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
var modules: seq[PrecompiledModule]
var precompSys: PrecompiledModule
var target: PrecompiledModule
if targetIsMain:
var nifFiles: seq[string]
(modules, precompSys, nifFiles) = loadBackendModules(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
# No whole-program DCE here: each module emits the routines it owns and the
# MERGE stage recomputes the one program-wide live set across all `.c.nif`s.
# Running a whole-program liveness pass over all ~260 NIFs in the main `cg`
# would cost ~900 MB for a result the merge stage throws away.
target = findTargetModule(g, modules, precompSys, g.config.icBackendModule)
else:
# No whole-program load, hence no whole-program DCE: the target emits its
# full demanded closure and the merge stage drops what is globally dead.
(modules, precompSys, target) = loadDepClosure(g, g.config.icBackendModule)
if target.module == nil:
rawMessage(g.config, errGenerated,
"per-module codegen: module not found for suffix: " & g.config.icBackendModule)
return
# The `lower` stage already wrote each module's transformed bodies + lifted
# hooks into its `.t.nif`, which the loaders above read directly (toNifFilename
# resolves the `.t.nif`); transformed bodies arrive via loadSymFromCursor and
# lifted hooks via moduleFromNifFile's registerLoadedHooks. Nothing to apply.
generateCodeForModule(g, target)
let bl = BModuleList(g.backend)
# The main module also owns the whole-program method dispatchers + NimMain.
if sfMainModule in target.module.flags:
emitMethodDispatchers(g)
# NimMain (generated when the main module is finished) must call every other
# module's init/datInit. Those translation units are produced by their own
# `cg` processes, so the calls are registered here from each `.c.nif` meta
# head — which is why the main module's `cg` runs last, after every other
# `.c.nif` exists. Modules without init code (no `.c.nif`) register nothing.
#
# The registration order IS the runtime init order, and it must be the
# DEPENDENCY (post-order) order: an imported module's init has to run before
# its importer's. The whole-program backend gets this for free — it iterates
# `modulesClosed`, built in module-FINISH order (a post-order DFS over
# imports). Iterating `bl.mods` by position is WRONG: an importer gets a
# LOWER position than the modules it imports (its file is registered before
# its `import` statements are processed), so position order runs importers
# before their dependencies. That left chronicles' `topics_registry` — whose
# init sets `mainThreadId` — running AFTER a module that calls `registerTopic`
# from its own init, tripping the `getThreadId() == mainThreadId` assert at
# startup. So reconstruct the post-order DFS over the import closure here.
#
# NOTE: this is deliberately a SEPARATE traversal rather than reusing the
# module LOAD order — the per-module backend's C emit is sensitive to load
# order (it determines the main TU's header composition), so the loader must
# keep its existing order and the init order is derived independently here.
var suffixToMod = initTable[string, PrecompiledModule]()
for pm in modules:
if pm.module != nil:
suffixToMod[cachedModuleSuffix(g.config, FileIndex pm.module.position)] = pm
if precompSys.module != nil:
suffixToMod[cachedModuleSuffix(g.config, FileIndex precompSys.module.position)] = precompSys
var visited = initHashSet[string]()
var ordered: seq[BModule] = @[]
# System (and its include/import closure) must initialize FIRST: its init
# runs `initGC()` (top-level code in `threadimpl`, included into system),
# and every other module's init may allocate — an allocation before the GC
# heap is set up triggers a collection over an uninitialized region and
# crashes (e.g. nim-metrics' `newRegistry` in its init). System is the
# IMPLICIT universal import and appears in no module's explicit `deps`, so a
# DFS rooted at main never reaches it; seed the traversal from system first.
if precompSys.module != nil:
visitDep(cachedModuleSuffix(g.config, FileIndex precompSys.module.position),
suffixToMod, visited, bl, ordered)
# Then order the whole import closure rooted at the main module; main itself
# is excluded above (its init body becomes NimMain).
for pm in modules:
if pm.module != nil and sfMainModule in pm.module.flags:
visitDep(cachedModuleSuffix(g.config, FileIndex pm.module.position),
suffixToMod, visited, bl, ordered)
# Defensive: any loaded module not reachable from main's import closure
# (demand-loaded system internals) keeps its init registered, appended last
# — nothing imports it, so its relative order does not matter.
for m in bl.mods:
if m != nil and sfMainModule notin m.module.flags:
let suffix = cachedModuleSuffix(g.config, FileIndex m.module.position)
if not visited.containsOrIncl(suffix):
ordered.add m
for m in ordered:
let heads = readCnifHeads(getCFile(m).string & ".nif")
registerReusedModuleToMain(bl, m, heads.initRequired, heads.datInitRequired)
let tb = bl.mods[target.module.position]
if tb != nil:
finishModule(g, tb)
# Writes only the target's `.c.nif` (every other loaded module's TU is empty,
# so `cgenWriteModules` emits no artifact for it). cc/link are NOT run here.
cgenWriteModules(g.backend, g.config)
# Always leave a `.c.nif` for the target, even when the module has no code
# (a leaf library whose procs all emit into their users): the per-module
# nifmake graph declares one `.c.nif` output per `cg` rule, so a missing one
# would re-fire the rule forever. An empty artifact renders to an empty `.c`.
if tb != nil:
let artifact = getCFile(tb).string & ".nif"
if not fileExists(artifact):
writeCnifArtifact("", artifact,
semmedNif = toNifFilename(g.config, FileIndex target.module.position),
moduleBase = $getSomeNameForModule(tb))
proc generateMergeStage(g: ModuleGraph) =
## Per-module backend merge (`--icBackendStage:merge`): a pure artifact
## operation, no module graph loaded. Reads every `.c.nif` the `cg` stages
## wrote, computes the global live set and — for each `'u'`-flagged unique
## definition that several `cg` processes emitted (emit-everywhere) — the one
## artifact allowed to embed its body, and writes the decision the `emit`
## stages consume — the cross-process replacement for what used to be
## in-process first-claimant/DCE coordination.
let nimcache = getNimcacheDir(g.config).string
var files: seq[string] = @[]
for artifact in walkFiles(nimcache / "*.c.nif"):
files.add artifact
sort files
let decision = computeMergeDecision(files)
if decision.broken:
rawMessage(g.config, errGenerated,
"per-module backend merge: a .c.nif artifact is missing or unparsable")
return
writeMergeDecision(nimcache / MergeDecisionFile, decision)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icMerge] artifacts: " & $files.len &
" live: " & $decision.live.len & " defs: " & $decision.defs &
" liveDefs: " & $decision.liveDefs & " owned: " & $decision.owners.len
proc generateEmitStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend emit (`--icBackendStage:emit --icBackendModule:<suffix>`):
## render the target module's final `.c` from its `.c.nif` and the merge
## decision. Loads the target the same way `cg` does so `getCFile` returns the
## identical path `cg` wrote to (the main module's source-vs-suffix aliasing in
## particular); no codegen runs. A non-main target loads only its own closure
## (`loadDepClosure`) so emit, like `cg`, stays bounded under parallel fan-out.
let mainSuffix = cachedModuleSuffix(g.config, mainFileIdx)
let targetIsMain = g.config.icBackendModule.len == 0 or
g.config.icBackendModule == mainSuffix
# emit renders a module's final `.c` PURELY from its own `.c.nif` and the merge
# decision (see `renderCFromArtifact` — text filtering, no AST is touched). It
# used to load the target's whole transitive import closure as BModules solely
# to reach `getCFile(bmod)` for the output path. Under the fire-all-every-edit
# merge barrier (every `emit` re-fires whenever `merge` bumps the decision's
# mtime — deliberate insurance so a decision change re-renders all `.c`
# consistently) that per-process `loadDepClosure` was the bulk of a warm
# rebuild's cost: 240 processes each re-parsing a module closure only to filter
# a handful of `.c.nif`s whose bytes are usually unchanged. Derive the `.c`
# path directly instead — the SAME pure computation `deps.nim.backendCFile`
# uses to DECLARE this stage's output (`getCFile` == that formula) — so an emit
# process loads nothing and the fire-all costs process-startup, not a graph load.
let cfilename =
if targetIsMain: AbsoluteFile toFullPath(g.config, mainFileIdx)
else: AbsoluteFile g.config.icBackendModule
let cfile = changeFileExt(completeCfilePath(g.config,
mangleModuleName(g.config, cfilename).AbsoluteFile), ".nim.c").string
let artifact = cfile & ".nif"
if not fileExists(artifact):
rawMessage(g.config, errGenerated,
"per-module emit: missing .c.nif artifact for suffix: " & g.config.icBackendModule)
return
let decision = readMergeDecision(getNimcacheDir(g.config).string / MergeDecisionFile)
if decision.broken:
rawMessage(g.config, errGenerated,
"per-module emit: missing or unparsable merge decision " & MergeDecisionFile)
return
var dropped = 0
let code = renderCFromArtifact(artifact, decision, extractFilename(artifact), dropped)
# Write the `.c` content-stably. `merge` re-runs on any edit and bumps the
# decision file's mtime, so nifmake re-fires every `emit` (the filter is cheap);
# but the FILTERED output is usually byte-identical for modules unaffected by
# the edit. Rewriting it unconditionally would bump every `.c`'s mtime and make
# `callCCompiler` recompile every `.o`. Writing only on a real change preserves
# the mtime, so the C compiler recompiles exactly the modules whose `.c` changed
# — the same DCE model as Nimony's. Safe here (unlike a content-stable merge
# decision): a `.c` is a per-module LEAF consumed only by the C compiler's own
# up-to-date check, not a shared prerequisite in nifmake's mtime ordering.
if not fileExists(cfile) or readFile(cfile) != code:
writeFile(cfile, code)
if isDefined(g.config, "icDceCheck"):
stderr.writeLine "[icEmit] " & extractFilename(cfile) & " dropped " &
$dropped & " bodies (" & $code.len & " bytes)"
proc generateLinkStage(g: ModuleGraph; mainFileIdx: FileIndex) =
## Per-module backend link (`--icBackendStage:link`): the `emit` stages have
## written every module's `.c`; register them and run the C compiler + linker
## once via `extccomp.callCCompiler` (which parallelizes the per-file cc and
## skips up-to-date objects itself). No codegen runs — the graph is loaded only
## so `getCFile` yields each module's emitted `.c` path.
let (modules, precompSys, _) = loadBackendModules(g, mainFileIdx)
# Load all modules in dependency order using stack traversal
# This must happen BEFORE any code generation so that hooks are loaded into loadedOps
let modules = loadModuleDependencies(g, mainFileIdx)
if modules.len == 0:
rawMessage(g.config, errGenerated,
"Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx))
return
# The per-module `cg` processes each collect their module's C compile/link
# directives (`{.passL: "-lm".}` etc.) via `replayBackendActions`, but those
# live in the cg process and never reach this separate link process. Re-collect
# every loaded module's directives here so the final `callCCompiler` sees them
# (without this, math's `-lm` is lost → undefined `floor`/`pow`/… at link).
# Set up backend modules for all modules that need code generation
for m in modules:
replayBackendActions(g, m.module, m.topLevel)
discard setupNifBackendModule(g, m.module)
# Also ensure system module is set up and generated first if it exists
if precompSys.module != nil:
replayBackendActions(g, precompSys.module, precompSys.topLevel)
let bl = BModuleList(g.backend)
var addedCFiles = initHashSet[string]()
for m in bl.mods:
discard setupNifBackendModule(g, precompSys.module)
generateCodeForModule(g, precompSys)
# Track which modules have been processed to avoid duplicates
var processed = initIntSet()
if precompSys.module != nil:
processed.incl precompSys.module.position
# Generate code for all modules (skip system since it's already processed)
for m in modules:
if not processed.containsOrIncl(m.module.position):
generateCodeForModule(g, m)
# during code generation of `main.nim` we can trigger the code generation
# of symbols in different modules so we need to finish these modules
# here later, after the above loop!
# Important: The main module must be finished LAST so that all other modules
# have registered their init procs before genMainProc uses them.
var mainModule: BModule = nil
for m in BModuleList(g.backend).mods:
if m != nil:
let cfile = getCFile(m)
# Only modules that are their own cg/emit target produced a `.c`; the rest
# (extra members of system's closure that no build rule targets) had their
# code emit-everywhere'd into the targets, so they have no file to compile.
if not fileExists(cfile.string): continue
addedCFiles.incl extractFilename(cfile.string)
var cf = Cfile(nimname: m.module.name.s, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
# `addExternalFileToCompile` (not `addFileToCompile`) gates each `.c` on its
# SHA1 footprint: an unchanged `.c` keeps its `.o` and is flagged Cached, so
# `callCCompiler` skips its compile but still links the existing object. This
# is what makes a localized edit recompile only the handful of `.c`s the
# `emit` stage actually rewrote, instead of every object every time — the
# final piece of per-module backend incrementality after the merge barrier.
addExternalFileToCompile(g.config, cf)
# deps.nim's static scanner can keep a CONDITIONALLY-imported module as a build
# node (e.g. `net`'s `when defineSsl: import openssl`, or a `when defined(os)`
# import) that the NIF-`deps` walk above never reaches because the condition is
# off. Such a node still emitted a `.c`, and it can OWN a live generic instance
# that a REACHABLE module reuses (openssl owns `toHex[uint8]`, reused by
# `strutils.escape`) — so its body must be at link or that reference is
# undefined. Link every emitted `.c` the merge decision says OWNS a LIVE symbol;
# a node that owns nothing live (a Windows-only winsock node on Linux) is
# correctly skipped.
block:
let nimcache = getNimcacheDir(g.config).string
let decision = readMergeDecision(nimcache / MergeDecisionFile)
if not decision.broken:
var liveOwners = initHashSet[string]()
for cname, owner in decision.owners:
if owner.endsWith(".c.nif") and cname in decision.live:
liveOwners.incl owner
for owner in liveOwners:
let cbase = owner[0 ..< owner.len - ".nif".len] # "@m….nim.c.nif" -> ".c"
if addedCFiles.containsOrIncl(cbase): continue
let cfile = AbsoluteFile(nimcache / cbase)
if not fileExists(cfile.string): continue
var cf = Cfile(nimname: cbase, cname: cfile,
obj: completeCfilePath(g.config, toObjFile(g.config, cfile)),
flags: {})
addExternalFileToCompile(g.config, cf)
assert m.module != nil
if sfMainModule in m.module.flags:
mainModule = m
else:
finishModule g, m
if mainModule != nil:
finishModule g, mainModule
# Write C files
cgenWriteModules(g.backend, g.config)
# Run C compiler
if g.config.cmd != cmdTcc:
extccomp.callCCompiler(g.config)
proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
## Main entry point for NIF-based C code generation.
## Traverses the module dependency graph and generates C code.
if g.config.icBackendStage == "lower":
generateLowerStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "cg":
generateCgStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "merge":
generateMergeStage(g)
return
elif g.config.icBackendStage == "emit":
generateEmitStage(g, mainFileIdx)
return
elif g.config.icBackendStage == "link":
generateLinkStage(g, mainFileIdx)
return
else:
rawMessage(g.config, errGenerated,
"the per-module NIF backend requires --icBackendStage:lower|cg|merge|emit|link")
if not g.config.hcrOn:
extccomp.writeJsonBuildInstructions(g.config, g.cachedFiles)

View File

@@ -16,7 +16,7 @@ import
import "../dist/nimony/src/lib" / nifbuilder
import "../dist/nimony/src/models" / nifler_tags
import icmodnames
import "../dist/nimony/src/gear2" / modnames
## This was copied from Nifler's bridge.nim. However, this code will evolve
## in a different direction as it needs to translate the semchecked AST which

View File

@@ -183,6 +183,12 @@ func `<`*(a: ExprIndex, b: ExprIndex): bool =
func `<=`*(a: ExprIndex, b: ExprIndex): bool =
a.int16 <= b.int16
func `>`*(a: ExprIndex, b: ExprIndex): bool =
a.int16 > b.int16
func `>=`*(a: ExprIndex, b: ExprIndex): bool =
a.int16 >= b.int16
func `==`*(a: ExprIndex, b: ExprIndex): bool =
a.int16 == b.int16

View File

@@ -14,10 +14,6 @@ define:nimPreviewAsmSemSymbol
define:nimPreviewCStringComparisons
#define:nimPreviewDuplicateModuleError
# Incompatible with Nimony's compat2.nim for now
# NOTE: `-d:virtualParRi` (jump-encoded ParLe + elided ParRi) is NOT yet enabled:
# the IC writer assembles buffers by raw token splicing (`dest.add content[i]`),
# which does not seal scopes the way `addParRi` does, so sealed `(stmts)` get
# jump=0 and serialize empty. Enabling it needs writer buffer-sealing work first.
threads:off

View File

@@ -28,13 +28,10 @@ import
commands, options, msgs, extccomp, main, idents, lineinfos, cmdlinehelper,
pathutils, modulegraphs
from ast2nif import registerNifAstTags
from icconfig import ensureIcConfig
from std/browsers import openDefaultBrowser
from nodejs import findNodeJs
when defined(tinyc): # == hasTinyCBackend; spelled out for the IC dep scanner
when hasTinyCBackend:
import tccgen
when defined(profiler) or defined(memProfiler):
@@ -99,11 +96,6 @@ proc getNimRunExe(conf: ConfigRef): string =
result = ""
proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
# NIF tag registration must not depend on module init order — the IC-built
# compiler orders module init calls differently and the top-level
# `registerTag` initializers then ran against a not-yet-initialized pool,
# corrupting every written NIF (see registerNifAstTags).
registerNifAstTags()
let self = NimProg(
supportsStdinFile: true,
processCmdLine: processCmdLine
@@ -115,14 +107,6 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
self.processCmdLineAndProjectPath(conf)
# `nim ic` driver: ensure the precompiled config exists (produced by a separate
# `nim icconfig` process, skipped when nothing changed) BEFORE config loading,
# so `loadConfigs` replays it instead of re-parsing the `nim.cfg` chain — the
# driver runs on the exact same config its children will. See icconfig.nim.
when not defined(nimKochBootstrap):
if conf.cmd in {cmdIc, cmdTrack}:
ensureIcConfig(conf)
var graph = newModuleGraph(cache, conf)
if not self.loadConfigsAndProcessCmdLine(cache, conf, graph):
return
@@ -134,7 +118,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
if conf.selectedGC == gcUnselected:
if conf.backend in {backendC, backendCpp, backendObjc} or
(conf.cmd in cmdDocLike and conf.backend != backendJs) or
conf.cmd in {cmdGendepend, cmdNifC, cmdIc, cmdM, cmdTrack}:
conf.cmd in {cmdGendepend, cmdNifC, cmdIc, cmdM}:
initOrcDefines(conf)
if conf.selectedStrings == stringSso and

View File

@@ -11,7 +11,7 @@
import
llstream, commands, msgs, lexer, ast,
options, idents, wordrecg, lineinfos, pathutils, scriptconfig, icconfig
options, idents, wordrecg, lineinfos, pathutils, scriptconfig
import std/[os, strutils, strtabs]
@@ -246,16 +246,6 @@ proc getSystemConfigPath*(conf: ConfigRef; filename: RelativeFile): AbsoluteFile
proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen: IdGenerator) =
setDefaultLibpath(conf)
# The `nim ic` driver and its `nim m`/`nim nifc` children replay the precompiled
# config (produced once by a separate `nim icconfig` process — see
# `icconfig.ensureIcConfig`, which sets `icPreparsedConfig` for the driver
# before this runs; the children get it as a forwarded `--icPreparsedConfig`
# argument) instead of re-reading the `nim.cfg` chain and re-running
# `config.nims` in the VM. A missing/format-incompatible artifact returns false:
# fall through to a normal parse (this is also the path the `nim icconfig`
# producer itself takes, since it runs with no `icPreparsedConfig`).
if conf.icPreparsedConfig.len > 0 and applyIcConfig(conf, conf.icPreparsedConfig):
return
template readConfigFile(path) =
let configPath = path
conf.currentConfigDir = configPath.splitFile.dir.string
@@ -316,7 +306,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen:
if conf.cmd == cmdNimscript:
showHintConf()
conf.configFiles.setLen 0
if not conf.ideActive and conf.cmd notin {cmdCheck, cmdDump}:
if conf.cmd notin {cmdIdeTools, cmdCheck, cmdDump}:
if conf.cmd == cmdNimscript:
runNimScriptIfExists(conf.projectFull, isMain = true)
else:

View File

@@ -29,32 +29,6 @@ const
nimEnableCovariance* = defined(nimEnableCovariance)
icFormatVersion* = "30"
## Version of the IC cache format (the sem-NIF module layout written by
## ast2nif.nim plus the iface/impl/edges side files). Bump it whenever
## that layout changes: `commandIc` wipes a nimcache whose `ic.version`
## stamp differs, instead of letting a newer reader mis-parse records
## written by an older compiler (nifmake's rebuild check is mtime-only
## and knows nothing about format changes).
## v2: iface cookie hashes routine SIGNATURES only (no inline-semantics
## body folding); body access now records a NeedsImpl edge instead. A v1
## cache mixes body-sensitive and body-insensitive cookies, so it must be
## wiped rather than warm-rebuilt.
## v3: added the `.s.deps` sidecar (real post-sem imports) and switched the
## macro-generated-import discovery from `icmissing.txt` to it.
## v4: backend C-name scheme change — the module suffix is now the trailing
## token (`name_u<disamb>__<suffix>`, was `name__<suffix>_u<disamb>`), so
## cached `.c.nif` artifacts hold incompatible names and must be wiped.
## v5: data definitions (consts, RTTI) are now wrapped in droppable `'d'`
## cdef directives with an always-present extern declaration, so the
## per-module merge stage can assign them a single owner; old `.c.nif`
## artifacts lack the wrappers.
## v6: `signatureHash`/`hashType` of a builtin type class (`object`, `tuple`,
## `proc`, ...) no longer mixes in the placeholder son's process-local type
## id, so its hash is stable across the NIF boundary (was breaking
## nim-serialization's auto-serialization lookup under IC). The sem-NIF
## macrocache entries and baked generic-instance bodies hold the old hashes.
type # please make sure we have under 32 options
# (improves code efficiency a lot!)
TOption* = enum # **keep binary compatible**
@@ -140,7 +114,6 @@ type # please make sure we have under 32 options
optDocRaw # for documentation: Don't render markdown for JSON output
optItaniumMangle # mangling follows the Itanium spec
optCompress # turn on AST compression by converting it to NIF
optGenBif # generate semantic BIF alongside ordinary code generation
optWithinConfigSystem # we still compile within the configuration system
TGlobalOptions* = set[TGlobalOption]
@@ -184,6 +157,7 @@ type
cmdCheck # semantic checking for whole project
cmdM # only compile a single
cmdParse # parse a single file (for debugging)
cmdIdeTools # ide tools (e.g. nimsuggest)
cmdNimscript # evaluate nimscript
cmdDoc0
cmdDoc # convert .nim doc comments to HTML
@@ -205,8 +179,6 @@ type
cmdCompileToNif
cmdNifC # generate C code from NIF files
cmdIc # generate .build.nif for nifmake
cmdIcConfig # `nim ic`'s precompiled-config producer (writes ic_config.cfg.nif)
cmdTrack # `nim track --def/--usages`: IC frontend build + NIF scan for IDE queries
const
cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,
@@ -290,11 +262,6 @@ type
procParamTypeBackendAliases
## Keep the old proc type compatibility rules that ignore backend
## c type aliases.
injectedSymbolRedefinition
## Allow a template to inject a symbol *definition* that is then emitted
## more than once (e.g. a `typed` argument captured by a `{.dirty.}`
## template and re-emitted). This is a redefinition and rejected by
## default; enabling this restores the old, unsound behavior. See #25693.
SymbolFilesOption* = enum
disabledSf, writeOnlySf, readOnlySf, v2Sf, stressTest
@@ -401,13 +368,6 @@ type
evalMacroCounter*: int
exitcode*: int8
cmd*: Command # raw command parsed as enum
ideActive*: bool # serving IDE tooling (nimsuggest): collect suggestions and
# keep going after errors. Decoupled from `cmd` so the IDE
# server can run under any compilation mode (cmdCheck, cmdM).
ideImportsFromNif*: bool # nimsuggest: load the unchanged import closure from
# precompiled NIF (run under cmdM) instead of recompiling it
# from source (cmdCheck). IC is opt-in: default off (cmdCheck);
# `--ideImports:nif` opts in.
cmdInput*: string # input command
projectIsCmd*: bool # whether we're compiling from a command input
implicitCmd*: bool # whether some flag triggered an implicit `command`
@@ -420,48 +380,6 @@ type
lastCmdTime*: float # when caas is enabled, we measure each command
symbolFiles*: SymbolFilesOption
ic*: bool # whether ic is enabled
icGroup*: HashSet[string] # under `nim m`: absolute paths of the modules in
# this strongly-connected import group. They are all
# compiled from source in one process (so mutual
# recursion resolves in-memory) and each gets its NIF
# written, instead of being loaded from a precompiled
# NIF. See `compiler/deps.nim` (SCC grouping).
icProject*: string # under `nim m`/`nim nifc`: absolute path of the
# ORIGINAL project file. The child's own project file
# is the module being compiled, which would make that
# module's package the "main package" and unfilter
# foreign-package diagnostics; the real project
# restores whole-program filtering semantics.
icPreparsedConfig*: string # under the `nim ic` driver and its `nim m`/`nim nifc`
# children: path of the precompiled config artifact.
# When set, `loadConfigs` replays the recorded
# config-file switches from it instead of re-reading
# the `nim.cfg` chain and re-running `config.nims`
# (which the VM makes expensive) per process. The
# artifact itself is produced by a separate
# `nim icconfig` process (see `cmdIcConfig`).
icConfigOut*: string # under `nim icconfig`: the path to write the
# precompiled config artifact to (set via `--o`).
icConfigSwitches*: seq[tuple[switch, arg: string]]
# the config-file (`passPP`) switches applied while
# loading config, in order. Recorded by every nim
# process; only the `ic` driver serialises them.
# Path-search switches are excluded — the driver
# forwards the resolved `searchPaths` as `--path`.
icBackendStage*: string # under `nim nifc`: which stage of the per-module
# backend this invocation runs — "cg" (codegen one
# module to its `.c.nif`), "merge" (global liveness
# + owner assignment across all `.c.nif`), "emit"
# (render one module's `.c` from its `.c.nif` + the
# merge decision), "link" (cc + link every emitted
# `.c`). Empty = whole-program backend (load all,
# codegen+DCE+cc+link in one process). The stages
# are wired as nifmake rules by `deps.nim`'s backend
# build file. See `compiler/nifbackend.nim`.
icBackendModule*: string # under `nim nifc` with icBackendStage in {cg,emit}:
# the NIF module suffix this invocation codegens or
# emits. The other modules are loaded only so types
# resolve; their definitions are referenced extern.
spellSuggestMax*: int # max number of spelling suggestions for typos
cppDefines*: HashSet[string] # (*)
@@ -508,12 +426,6 @@ type
lastMsgWasDot*: set[StdOrrKind] # the last compiler message was a single '.'
projectMainIdx*: FileIndex # the canonical path id of the main module
projectMainIdx2*: FileIndex # consider merging with projectMainIdx
isMainModule*: bool # `nim m`/IC only: whether the single module being
# semantically checked is the program's real entry point.
# Under IC every module is compiled via `nim m` (which sets
# `sfMainModule` so the module writes its own NIF), so
# `sfMainModule` can no longer answer `isMainModule`. The IC
# build file passes `--isMainModule:on` for the root module.
command*: string # the main command (e.g. cc, check, scan, etc)
commandArgs*: seq[string] # any arguments after the main command
commandLine*: string
@@ -670,7 +582,6 @@ proc newConfigRef*(): ConfigRef =
arcToExpand: newStringTable(modeStyleInsensitive),
m: initMsgConfig(),
cppDefines: initHashSet[string](),
icGroup: initHashSet[string](),
headerFile: "", features: {}, legacyFeatures: {},
configVars: newStringTable(modeStyleInsensitive),
symbols: newStringTable(modeStyleInsensitive),
@@ -693,7 +604,6 @@ proc newConfigRef*(): ConfigRef =
command: "", # the main command (e.g. cc, check, scan, etc)
commandArgs: @[], # any arguments after the main command
commandLine: "",
ideImportsFromNif: false, # IC opt-in; see `--ideImports`
implicitImports: @[], # modules that are to be implicitly imported
implicitIncludes: @[], # modules that are to be implicitly included
docSeeSrcUrl: "",
@@ -747,7 +657,6 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
of "x86": result = conf.target.targetCPU == cpuI386
of "itanium": result = conf.target.targetCPU == cpuIa64
of "x8664": result = conf.target.targetCPU == cpuAmd64
of "wasm": result = conf.target.targetCPU in {cpuWasm32, cpuWasm64}
of "posix", "unix":
result = conf.target.targetOS in {osLinux, osMorphos, osSkyos, osIrix, osPalmos,
osQnx, osAtari, osAix,
@@ -795,18 +704,7 @@ template quitOrRaise*(conf: ConfigRef, msg = "") =
else:
quit(msg) # quits with QuitFailure
proc icReuseSemLowering*(conf: ConfigRef): bool {.inline.} =
## When ON, the per-module `lower` backend stage REUSES the VM/CT lowering that
## sem cached in the `.s.nif` 2-way-body slot (the non-IC single-lowering
## semantics) instead of re-deriving the transform. Default OFF: the backend
## re-derives every body from the pristine semchecked body (simpler; allowed by
## the 2026-06-27 spec that VM-requested frontend transforms need not influence
## the backend). The switch exists so caching can be restored if a target (e.g.
## Nimbus) depends on the cached lowering being reused, not re-derived. See
## doc/ic_backend_simplify.md §6b.
isDefined(conf, "icReuseSemLowering")
proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.ideActive or conf.cmd in cmdDocLike
proc importantComments*(conf: ConfigRef): bool {.inline.} = conf.cmd in cmdDocLike + {cmdIdeTools}
proc usesWriteBarrier*(conf: ConfigRef): bool {.inline.} = conf.selectedGC >= gcRefc
proc usesSso*(conf: ConfigRef): bool {.inline.} = conf.selectedStrings == stringSso
@@ -932,8 +830,7 @@ proc getOsCacheDir(): string =
proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir =
proc nimcacheSuffix(conf: ConfigRef): string =
if conf.ideActive: "_nimsuggest" # dedicated cache, never shared with `nim c`
elif conf.cmd == cmdCheck: "_check"
if conf.cmd == cmdCheck: "_check"
elif isDefined(conf, "release") or isDefined(conf, "danger"): "_r"
else: "_d"

View File

@@ -54,10 +54,7 @@ import
when not defined(nimCustomAst):
import ast
when defined(nimCustomAst):
# NOTE: explicit negated `when` rather than `else:` — nifler's dep scanner
# guards `when`/`elif` imports with their condition but emits `else:` imports
# unconditionally, which would wrongly schedule this module under `nim ic`.
else:
import plugins / customast
import std/strutils
@@ -2244,17 +2241,14 @@ proc parseTypeClassParam(p: var Parser): PNode =
proc parseTypeClass(p: var Parser): PNode =
#| conceptParam = ('var' | 'out' | 'ptr' | 'ref' | 'static' | 'type')? symbol
#| conceptDecl = 'concept' (conceptParam ^* ',' (pragma)?)? ('of' typeDesc ^* ',')?
#| conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
#| &IND{>} stmt
result = newNodeP(nkTypeClassTy, p)
getTok(p)
if p.tok.tokType == tkComment:
skipComment(p, result)
if p.tok.tokType == tkOf and p.tok.indent < 0:
# new-styled `concept of A, B` on the same line as `concept`
result.add(p.emptyNode)
elif p.tok.indent < 0:
if p.tok.indent < 0:
var args = newNodeP(nkArgList, p)
result.add(args)
args.add(p.parseTypeClassParam)
@@ -2280,10 +2274,9 @@ proc parseTypeClass(p: var Parser): PNode =
result.add(p.emptyNode)
if p.tok.tokType == tkComment:
skipComment(p, result)
# an initial IND{>} HAS to follow, unless this concept inherits requirements:
# an initial IND{>} HAS to follow:
if not realInd(p):
let hasParents = result[2].kind != nkEmpty
if result.isNewStyleConcept and not hasParents:
if result.isNewStyleConcept:
parMessage(p, "routine expected, but found '$1' (empty new-styled concepts are not allowed)", p.tok)
result.add(p.emptyNode)
else:

View File

@@ -15,7 +15,7 @@ import ../dist/checksums/src/checksums/sha1
when not defined(leanCompiler):
import jsgen, docgen2
import std/[syncio, objectdollar, assertions, tables, strutils, strtabs, sets, intsets]
import std/[syncio, objectdollar, assertions, tables, strutils, strtabs]
import renderer
import ic/replayer
@@ -167,8 +167,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
s = stream
graph.interactive = stream.kind == llsStdIn
var topLevelStmts =
if {optCompress, optGenBif} * graph.config.globalOptions != {} or
graph.config.cmd == cmdM:
if optCompress in graph.config.globalOptions or graph.config.cmd == cmdM:
newNodeI(nkStmtList, module.info)
else:
nil
@@ -244,23 +243,9 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
when not defined(nimKochBootstrap):
# For cmdM: only write NIF for the main module, not for imported modules
# (imported modules should be loaded from existing NIF files). Members of the
# current strongly-connected import group (`--icGroup`) are the exception:
# they are compiled from source here, so each must write its own NIF.
let shouldWriteNif =
if graph.config.ideActive:
# nimsuggest (cmdM): persist NIF for cleanly-compiled, SAVED modules so
# later queries load them instead of recompiling. Never persist the
# actively edited buffer (it may hold unsaved/incomplete code) nor a
# module that failed to compile — that would poison the cache.
graph.config.cmd == cmdM and graph.config.errorCounter == 0 and
graph.config.m.fileInfos[module.position].dirtyFile.isEmpty
else:
({optCompress, optGenBif} * graph.config.globalOptions != {}) or
(graph.config.cmd == cmdM and
(sfMainModule in module.flags or
(graph.config.icGroup.len > 0 and
toFullPath(graph.config, module.position.FileIndex) in graph.config.icGroup)))
# (imported modules should be loaded from existing NIF files)
let shouldWriteNif = (optCompress in graph.config.globalOptions) or
(graph.config.cmd == cmdM and sfMainModule in module.flags)
if shouldWriteNif and not graph.config.isDefined("nimscript"):
topLevelStmts.add finalNode
# Collect replay actions from both pragma computations and VM state diff
@@ -274,141 +259,16 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator
if m == module:
replayActions.add n
# NeedsImpl edge recording: which modules' bodies this process consumed
# at compile time (VM/getImpl). For an --icGroup cycle every member gets
# the union; intra-group entries are filtered by the writer.
var implDeps: seq[int] = @[]
for id in graph.icImplDeps: implDeps.add id
# Generic-instance OFFERS: every instance THIS module created, so a
# consumer reuses it rather than re-instantiating in its own scope (which
# cannot see symbols visible only at the generic's definition site — e.g.
# a distinct type's `==`). See ast2nif.writeNifModule / moduleFromNifFile.
var genericOffers: seq[tuple[generic, inst: PSym;
concreteTypes: seq[PType]; genericParamsCount: int]] = @[]
for genItemId, instList in graph.procInstCache:
for inst in instList:
if inst.sym != nil and inst.sym.itemId.module == module.position and
inst.sym.instantiatedFrom != nil and inst.compilesId == 0:
# `concreteTypes` is pre-sized to `paramsLen+gp.len`; a tail slot can
# stay nil (e.g. fewer materialized params than `paramsLen`). Such an
# offer can't be serialized — skip it (the consumer re-instantiates,
# the prior behaviour) rather than emit a nil type reference.
var hasNil = false
for ct in inst.concreteTypes:
if ct == nil: hasNil = true; break
if not hasNil:
genericOffers.add (inst.sym.instantiatedFrom, inst.sym,
inst.concreteTypes, inst.genericParamsCount)
# Generic TYPE-instance OFFERS: every `tyGenericInst` THIS module created,
# so a consumer reuses its baked structure (array bounds etc.) rather than
# re-instantiating with a scope-divergent bound. See ast2nif.writeNifModule.
var typeOffers: seq[tuple[generic: PSym; inst: PType]] = @[]
for genItemId, instList in graph.typeInstCache:
for inst in instList:
if inst != nil and inst.uniqueId.module == module.position and
inst.kidsLen > 0 and inst[0] != nil and
inst[0].kind == tyGenericBody and inst[0].sym != nil:
typeOffers.add (inst[0].sym, inst)
# The module's REAL resolved direct imports (incl. macro/template-generated
# ones with no surviving syntactic node). Passed to writeNifModule so the
# NIF `deps` section is complete (the backend closure walk needs it), and
# reused below for the `.s.deps` sidecar (frontend graph re-derivation).
let resolvedImportDeps = graph.importDeps.getOrDefault(module.position.FileIndex, @[])
# The frontend's highest used itemId (max of the sym and type counters):
# the backend seeds its id minting ABOVE this so closure envs / RTTI hooks
# never share a `toId` with a frontend sym/type. See ast2nif `(unusedid)`.
let firstUnusedId = max(idgen.symId, idgen.typeId)
var expansions: seq[(PSym, TLineInfo)] = @[]
discard graph.nifExpansions.take(module.position.int32, expansions)
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog,
replayActions, implDeps, reexportedModuleSyms(graph, module),
genericOffers, typeOffers, resolvedImportDeps, firstUnusedId,
expansions)
# The module's REAL direct imports (incl. macro-generated) for `nim ic`'s
# graph re-derivation; see ast2nif.writeSemDeps / semdata.addImportFileDep.
var semDepPaths: seq[string] = @[]
for f in resolvedImportDeps:
semDepPaths.add toFullPath(graph.config, f)
writeSemDeps(graph.config, module.position.int32, semDepPaths)
writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, replayActions)
result = true
proc loadedDefSym(defs: PNode): PSym =
## The defined symbol of a let/var entry as it loads back from a NIF: the
## section child is a bare `nkSym` (the `(sd …)` reference), but be defensive
## about the from-source shapes too (`nkIdentDefs`, a pragma-wrapped name).
case defs.kind
of nkSym: result = defs.sym
of nkPragmaExpr:
result = if defs.len > 0: loadedDefSym(defs[0]) else: nil
of nkIdentDefs, nkConstDef:
result = if defs.len > 0: loadedDefSym(defs[0]) else: nil
else: result = nil
proc initLoadedCompileTimeGlobals(graph: ModuleGraph; module: PSym; topLevel: PNode) =
## Eagerly initialize the compile-time globals (`let/var {.compileTime.}`) of a
## module restored from a NIF. In a normal sem these VM slots are filled by
## `setupCompileTimeVar` (semstmts) as the section is semchecked; a NIF-loaded
## module is never semchecked, so without this a macro or compile-time proc that
## reads such a global finds a nil slot. The lazy `vmgen.genGlobalInit` fallback
## is order-fragile across proc boundaries (it emits the init at the first
## VM-gen'd reference, which need not be the first one executed), so the init has
## to happen here, once, before any of the module's code can run. The symbol's
## own `ast` is the `nkIdentDefs` (initializer included); re-wrap it in a section
## exactly as semstmts does and hand it to the same evaluator.
if topLevel == nil: return
let idgen = idGeneratorFromModule(module)
for stmt in topLevel:
if stmt.kind notin {nkLetSection, nkVarSection}: continue
for defs in stmt:
let s = loadedDefSym(defs)
if s != nil and s.kind in {skLet, skVar} and
{sfCompileTime, sfGlobal} <= s.flags and
s.ast != nil and s.ast.kind == nkIdentDefs:
var sect = newNodeI(stmt.kind, s.info)
sect.add s.ast
setupCompileTimeVar(module, idgen, graph, sect)
proc finalizeLoadedModules(graph: ModuleGraph) =
## Apply the VM-level load effects of every module just loaded from a NIF —
## direct import OR dep-of-a-dep, both collected in `graph.pendingNifInit` by the
## loader (modulegraphs.moduleFromNifFile / loadTransitiveHooks). This is the ONE
## place that knows what loading a module does to global VM state, so a
## transitively-reached module (which never passes through this proc's caller)
## gets identical treatment. Modules are in dependency order (deps before
## dependents), which is the correct macro-cache replay order.
## 1. macro-cache replay: std/macrocache put/inc/add/incl recorded in the
## module's top level (pragma replay actions are a backend concern, skipped).
## 2. eager `{.compileTime.}` global init (see initLoadedCompileTimeGlobals).
## To add a new per-load effect, extend this proc — do not add a parallel buffer.
if graph.pendingNifInit.len == 0: return
for (m, topLevel) in graph.pendingNifInit:
if topLevel == nil: continue
var replayList = newNodeI(nkStmtList, m.info)
for n in topLevel:
if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
n[0].strVal in ["put", "inc", "add", "incl"]:
replayList.add n
if replayList.len > 0:
replayStateChanges(m, graph, replayList)
initLoadedCompileTimeGlobals(graph, m, topLevel)
graph.pendingNifInit.setLen 0
proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym =
var flags = flags
if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
result = graph.getModule(fileIdx)
template processModuleAux(moduleStatus) =
when defined(icDbg):
block:
let dbgf = open("/tmp/defdbg.txt", fmAppend)
dbgf.writeLine toFullPath(graph.config, fileIdx) &
" nimStackTraceOverride=" & $isDefined(graph.config, "nimStackTraceOverride") &
" nimscript=" & $isDefined(graph.config, "nimscript") &
" optCompress=" & $(optCompress in graph.config.globalOptions) &
" cmd=" & $graph.config.cmd
dbgf.close()
onProcessing(graph, fileIdx, moduleStatus, fromModule = fromModule)
var s: PLLStream = nil
if sfMainModule in flags:
@@ -418,57 +278,27 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
if result == nil:
when not defined(nimKochBootstrap):
# For cmdM: load imports from NIF files (but compile the main module from source)
# Skip when withinSystem is true (compiling system.nim itself).
# Also skip for members of the current strongly-connected import group
# (`--icGroup`): those are mutually recursive with the main module and have
# no precompiled NIF yet, so they must be compiled from source in this same
# process (falling through below) — that resolves the cycle in-memory, the
# same way the non-incremental compiler handles recursive module imports.
# Skip when withinSystem is true (compiling system.nim itself)
if graph.config.cmd == cmdM and
sfMainModule notin flags and
not graph.withinSystem and
not graph.config.isDefined("nimscript") and
(graph.config.icGroup.len == 0 or
toFullPath(graph.config, fileIdx) notin graph.config.icGroup):
not graph.config.isDefined("nimscript"):
let precomp = moduleFromNifFile(graph, fileIdx)
if precomp.module == nil:
if graph.config.ideActive:
# nimsuggest bootstrap: this import has no precompiled NIF yet (cold
# cache, or it was invalidated). Don't error — fall through to the
# source-compile path below; the pass-close emits a fresh NIF so the
# next query loads it instead of recompiling.
discard
else:
let nifPath = toNifFilename(graph.config, fileIdx)
# Macro-generated imports (e.g. chronicles' parseStmt("import
# chronicles/textlines") driven by the chronicles_sinks define) are
# invisible to the static scanner, so this module's NIF was never
# built. The importer already recorded this import via
# addImportFileDep, so flush every module's `.s.deps`: `nim ic` reads
# it, re-derives the graph with the missing node + edge, and reruns
# the frontend. We still error — this process cannot finish sem
# without the import — but the discovery is structured data now, not
# a side-channel file.
for importer, deps in graph.importDeps.pairs:
var paths: seq[string] = @[]
for f in deps: paths.add toFullPath(graph.config, f)
writeSemDeps(graph.config, importer.int32, paths)
globalError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) &
" (expected: " & nifPath & ")")
return nil # Don't fall through to compile from source
let nifPath = toNifFilename(graph.config, fileIdx)
globalError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) &
" (expected: " & nifPath & ")")
return nil # Don't fall through to compile from source
else:
# Module successfully loaded from NIF file - use it and skip processing
result = precomp.module
if sfSystemModule in flags:
graph.systemModule = result
partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx)))
# Apply the VM-level load effects of this module AND every dep it pulled in
# (moduleFromNifFile recorded them all in graph.pendingNifInit): macro-cache
# replay (else a NIF-loaded module's macro cache is lost — e.g.
# nim-serialization flavor registration) and eager `{.compileTime.}` global
# init. Uniform for direct and transitive deps — see finalizeLoadedModules.
finalizeLoadedModules(graph)
# Replay state changes from the loaded NIF module
if result.ast != nil:
replayStateChanges(result, graph)
return result # Return early, don't process from source
let path = toFullPath(graph.config, fileIdx)
let filename = AbsoluteFile path
@@ -534,14 +364,7 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
conf.projectMainIdx2 = projectFile
var packSym = getPackage(graph, projectFile)
if graph.config.cmd in {cmdM, cmdNifC} and graph.config.icProject.len > 0:
# per-module IC children: the process' project file is the MODULE being
# compiled, which would make its package the "main package" and unfilter
# foreign-package diagnostics (a vendored package's hintAsError promotion
# then aborts builds the whole-program compilation accepts). Use the
# original project, forwarded by deps.nim via --icproject.
packSym = getPackage(graph, fileInfoIdx(graph.config, AbsoluteFile graph.config.icProject))
let packSym = getPackage(graph, projectFile)
graph.config.mainPackageId = packSym.getPackageId
graph.importStack.add projectFile
@@ -552,32 +375,16 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
elif graph.config.cmd == cmdM:
# For cmdM: load system.nim from NIF first, then compile the main module
connectPipelineCallbacks(graph)
# Record the main module so the IC loader won't materialise duplicate stubs
# for its own symbols when a dependency (e.g. system) re-exports them.
setIcMainModule(projectFile)
graph.config.m.systemFileIdx = fileInfoIdx(graph.config,
graph.config.libpath / RelativeFile"system.nim")
when not defined(nimKochBootstrap):
# Don't clobber an already-compiled system: nimsuggest's NimScript config
# evaluation compiles `system` into this same graph before we get here.
let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx)
graph.systemModule = precomp.module
if graph.systemModule == nil:
let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx)
graph.systemModule = precomp.module
if graph.systemModule == nil:
if graph.config.ideActive:
# nimsuggest bootstrap: no system NIF yet — compile it from source
# (the pass-close emits it), then continue with the main module.
graph.compilePipelineSystemModule()
else:
let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx)
localError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for system module (expected: " & nifPath & ")")
return
# Apply system's (and its deps') load effects now: the main module is
# compiled from source and never re-enters the moduleFromNifFile drain for
# system, so without this its macro-cache / CT globals would wait until the
# first NIF import is processed. See finalizeLoadedModules.
finalizeLoadedModules(graph)
let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx)
localError(graph.config, unknownLineInfo,
"nim m requires precompiled NIF for system module (expected: " & nifPath & ")")
return
discard graph.compilePipelineModule(projectFile, {sfMainModule})
else:
graph.compilePipelineSystemModule()

View File

@@ -1,4 +1,3 @@
import std/intsets
import ast, options, lineinfos, pathutils, msgs, modulegraphs, packages
proc skipCodegen*(config: ConfigRef; n: PNode): bool {.inline.} =
@@ -24,3 +23,4 @@ proc prepareConfigNotes*(graph: ModuleGraph; module: PSym) =
proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
result = true
#module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")

View File

@@ -211,7 +211,7 @@ type
cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips,
cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430,
cpuSparc64, cpuS390x, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64,
cpuEsp, cpuWasm32, cpuE2k, cpuLoongArch64, cpuWasm64
cpuEsp, cpuWasm32, cpuE2k, cpuLoongArch64
type
TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness,
@@ -249,8 +249,7 @@ const
(name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
(name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
(name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
(name: "loongarch64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
(name: "wasm64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)]
(name: "loongarch64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)]
type
Target* = object

View File

@@ -77,7 +77,7 @@ template semIdeForTemplateOrGeneric(c: PContext; n: PNode;
# templates perform some quick check whether the cursor is actually in
# the generic or template.
when defined(nimsuggest):
if c.config.ideActive and requiresCheck:
if c.config.cmd == cmdIdeTools and requiresCheck:
#if optIdeDebug in gGlobalOptions:
# echo "passing to safeSemExpr: ", renderTree(n)
discard safeSemExpr(c, n)
@@ -89,18 +89,6 @@ proc fitNodePostMatch(c: PContext, formal: PType, arg: PNode): PNode =
changeType(c, x, formal, check=true)
result = arg
result = skipHiddenSubConv(result, c.graph, c.idgen)
# Walk through nested statement-list/block expressions to find the innermost
# value node. Empty containers (e.g. `@[]`) inside `nkStmtListExpr` wrappers
# need their type resolved to match the formal type, otherwise the C codegen
# cannot map `tyEmpty` to a concrete type (fixes #25945).
var tail = result
while tail.kind in {nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr, nkPragmaBlock} and tail.len > 0:
tail = tail.lastSon
if tail.typ != nil and tail.typ.isEmptyContainer and
formal.kind notin {tyUntyped, tyBuiltInTypeClass, tyAnything}:
changeType(c, tail, formal, check=true)
# mark inserted converter as used:
var a = result
if a.kind == nkHiddenDeref: a = a[0]
@@ -117,12 +105,9 @@ proc fitNode(c: PContext, formal: PType, arg: PNode; info: TLineInfo): PNode =
result.typ = formal
elif arg.kind in nkSymChoices and formal.skipTypes(abstractInst).kind == tyEnum:
# Pick the right 'sym' from the sym choice by looking at 'formal' type:
# The choice candidates may be wrapped in `var`/`lent` when they come from
# a loop-local view, but for enum disambiguation only the underlying enum
# type matters.
result = nil
for ch in arg:
if sameType(ch.typ.skipTypes({tyVar, tyLent}), formal):
if sameType(ch.typ, formal):
return ch
typeMismatch(c.config, info, formal, arg.typ, arg)
else:
@@ -262,40 +247,12 @@ proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
if result.kind notin {kind, skTemp}:
localError(c.config, n.info, "cannot use symbol of kind '$1' as a '$2'" %
[result.kind.toHumanStr, kind.toHumanStr])
# bug #25693: a local declared inside a template/macro operand (recorded in
# `shadowDiscardedDefs`) can be captured by a `{.dirty.}` template and
# re-emitted as a definition more than once. The first emission keeps the
# original symbol (so a leaked dirty-template name still resolves); every
# later emission gets a fresh copy, so distinct emissions don't share one
# symbol - which the destructor/liveness analysis would otherwise miscompile.
# Unlike a plain redefinition check this is control-flow agnostic, so the
# common "emit a `typed` body in several mutually-exclusive branches" pattern
# keeps working. gensym'ed locals (and ones derived from a gensym name) are
# excluded: the gensym machinery already keeps their names unique, and a
# fresh copy would reuse the unique name and clash in the same scope.
if kind in {skVar, skLet, skForVar} and
{sfGenSym, sfWasGenSym} * result.flags == {} and
result.id in c.shadowDiscardedDefs:
if containsOrIncl(c.realizedDefs, result.id):
let fresh = copySym(result, c.idgen)
fresh.ast = result.ast
put(c.p, result, fresh)
c.hasSymRedefs = true
result = fresh
when false:
if sfGenSym in result.flags and result.kind notin {skTemplate, skMacro, skParam}:
# declarative context, so produce a fresh gensym:
result = copySym(result)
result.ast = n.sym.ast
put(c.p, n.sym, result)
if result.state == Sealed:
# the symbol was loaded from another module's NIF cache (e.g. a param
# symbol spliced out of an imported proc type by a `typed` macro) and is
# therefore immutable; the caller re-owns it and assigns its type/flags,
# so hand back a fresh, mutable copy owned by the current module instead.
let fresh = copySym(result, c.idgen)
fresh.ast = result.ast
result = fresh
# when there is a nested proc inside a template, semtmpl
# will assign a wrong owner during the first pass over the
# template; we must fix it here: see #909
@@ -584,12 +541,10 @@ const
proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
let info = getCallLineInfo(n)
# the callee identifier's position is the usage site tooling expects (matches
# `markUsed` below), not the whole-call `nOrig.info`.
rememberExpansion(c, info, sym)
rememberExpansion(c, nOrig.info, sym)
pushInfoContext(c.config, nOrig.info, sym.detailedInfo)
let info = getCallLineInfo(n)
markUsed(c, info, sym)
onUse(info, sym)
if sym == c.p.owner:
@@ -896,7 +851,7 @@ proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
result = hloStmt(c, result)
if c.config.cmd == cmdInteractive and not isEmptyType(result.typ):
result = buildEchoStmt(c, result)
if c.config.ideActive:
if c.config.cmd == cmdIdeTools:
appendToModule(c.module, result)
trackStmt(c, c.module, result, isTopLevel = true)
if optMultiMethods notin c.config.globalOptions and
@@ -933,7 +888,7 @@ proc semWithPContext*(c: PContext, n: PNode): PNode =
result = nil
else:
result = newNodeI(nkEmpty, n.info)
#if c.config.ideActive: findSuggest(c, n)
#if c.config.cmd == cmdIdeTools: findSuggest(c, n)
proc reportUnusedModules(c: PContext) =
if c.config.cmd == cmdM: return
@@ -942,7 +897,7 @@ proc reportUnusedModules(c: PContext) =
message(c.config, info, warnUnusedImportX, s.name.s)
proc closePContext*(graph: ModuleGraph; c: PContext, n: PNode): PNode =
if c.config.ideActive and not c.suggestionsMade:
if c.config.cmd == cmdIdeTools and not c.suggestionsMade:
suggestSentinel(c)
closeScope(c) # close module's scope
rawCloseScope(c) # imported symbols; don't check for unused ones!

View File

@@ -90,14 +90,8 @@ proc addTypeBoundSymbols(graph: ModuleGraph, arg: PType, name: PIdent,
# argument must be typed first, meaning arguments always
# matching `untyped` are ignored
let t = nominalRoot(arg)
if t != nil and t.owner.kind == skModule and
t.owner.position >= 0 and t.owner.position < graph.ifaces.len:
# search module for routines attachable to `t`.
# Under IC the nominal type may have been loaded from a NIF file, in which
# case its owner module is a stub whose `position` (a NIF-suffix file index)
# has no `ifaces` slot; such type-bound ops are reachable through normal
# imports instead, so skip the direct module scan to avoid an out-of-range
# access.
if t != nil and t.owner.kind == skModule:
# search module for routines attachable to `t`
let module = t.owner
var iter = default(ModuleIter)
var s = initModuleIter(iter, graph, module, name)
@@ -732,15 +726,6 @@ proc indexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode =
result = paramTypesMatch(m, f, a, arg, nil)
if m.genericConverter and result != nil:
instGenericConvertersArg(c, result, m)
when defined(icDbg):
if result == nil and f != nil and a != nil and f.kind == tyEnum:
echo "INDEXMISMATCH f=", typeToString(f), " itemId=", f.itemId,
" uniqueId=", f.uniqueId, " mod=", toFullPath(c.config, f.itemId.module.FileIndex),
" sym=", (if f.sym != nil: $f.sym.itemId else: "nil"), " state=", f.state
let a2 = a.skipTypes({tyRange})
echo " a=", typeToString(a), " itemId=", a2.itemId, " uniqueId=", a2.uniqueId,
" mod=", toFullPath(c.config, a2.itemId.module.FileIndex),
" sym=", (if a2.sym != nil: $a2.sym.itemId else: "nil"), " state=", a2.state
proc inferWithMetatype(c: PContext, formal: PType,
arg: PNode, coerceDistincts = false): PNode =
@@ -983,12 +968,7 @@ proc explicitGenericSym(c: PContext, n: PNode, s: PSym, errors: var CandidateErr
diagnostics: m.diagnostics))
return nil
var newInst = generateInstance(c, s, m.bindings, n.info)
# `generateInstance` may return an instance REUSED from another module's NIF
# `(offer …)` — its type is Sealed (immutable). Such an instance is already
# fully resolved (`tfUnresolved` cleared at its original instantiation), so the
# `excl` is a no-op; skip it rather than assert on a Sealed-type mutation.
if newInst.typ.state != Sealed:
newInst.typ.excl tfUnresolved
newInst.typ.excl tfUnresolved
let info = getCallLineInfo(n)
markUsed(c, info, s, isGenericInstance = false)
onUse(info, s, isGenericInstance = false)

View File

@@ -189,18 +189,6 @@ type
inTypeofContext*: int
semAsgnOpr*: proc (c: PContext; n: PNode; k: TNodeKind): PNode {.nimcall.}
shadowDiscardedDefs*: IntSet
# ids of local symbols that were declared inside a template/macro operand's
# shadow scope and then discarded; re-emitting such a symbol as a
# definition gives a fresh copy so distinct emissions don't share a symbol.
# See bug #25693 and `rememberShadowDefs`.
realizedDefs*: IntSet
# ids from `shadowDiscardedDefs` already realized once; the first emission
# keeps the original symbol (so leaked dirty-template names still resolve),
# later emissions get a fresh copy.
hasSymRedefs*: bool
# set once a redefinition mapping has been installed; makes `getGenSym`
# consult the proc-con mapping for non-gensym symbols too.
TBorrowState* = enum
bsNone, bsReturnNotMatch, bsNoDistinct, bsGeneric, bsNotSupported, bsMatch
@@ -293,10 +281,7 @@ proc get*(p: PProcCon; key: PSym): PSym =
result = p.mapping.getOrDefault(key.itemId)
proc getGenSym*(c: PContext; s: PSym): PSym =
# `c.hasSymRedefs` additionally routes ordinary (non-gensym) symbols through
# the mapping so a re-emitted definition can redirect them to its fresh copy,
# see bug #25693 and `newSymG`.
if sfGenSym notin s.flags and not c.hasSymRedefs: return s
if sfGenSym notin s.flags: return s
var it = c.p
while it != nil:
result = get(it, s)
@@ -358,8 +343,6 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
userPragmas: initStrTable(),
generics: @[],
unknownIdents: initIntSet(),
shadowDiscardedDefs: initIntSet(),
realizedDefs: initIntSet(),
cache: graph.cache,
graph: graph,
signatures: initStrTable(),
@@ -370,21 +353,11 @@ proc addIncludeFileDep*(c: PContext; f: FileIndex) =
discard
proc addImportFileDep*(c: PContext; f: FileIndex) =
# Under `nim m` (the IC frontend) record the REAL direct imports of the
# current module as sem resolves them — including imports a macro generated
# (e.g. chronicles' `parseStmt("import chronicles/textlines")`), which the
# static dependency scanner never sees. `nim ic` writes this set as the
# module's `.s.deps` sidecar and re-derives the build graph from it, so the
# discovery is structured data instead of a build-failure side channel.
if c.config.cmd == cmdM:
let importer = c.module.position.FileIndex
var deps = addr c.graph.importDeps.mgetOrPut(importer, @[])
if f notin deps[]: deps[].add f
discard
proc addPragmaComputation*(c: PContext; n: PNode) =
# Also store whenever the semchecked module is serialized to NIF/BIF.
if {optCompress, optGenBif} * c.config.globalOptions != {} or
c.config.cmd == cmdM:
# Also store for NIF-based IC (cmdM mode or optCompress)
if optCompress in c.config.globalOptions or c.config.cmd == cmdM:
addNifReplayAction(c.graph, c.module.position.int32, n)
proc inclSym(sq: var seq[PSym], s: PSym): bool =
@@ -397,18 +370,6 @@ proc addConverter*(c: PContext, conv: PSym) =
assert conv != nil
if inclSym(c.converters, conv):
add(c.graph.ifaces[c.module.position].converters, conv)
# Record for IC: the loader rebuilds Iface.converters from the NIF's
# (repconverter ...) entries (moduleFromNifFile). This must capture not only
# converters DEFINED in this module (addConverterDef) but also ones IMPORTED
# from another module here (importer.addUnnamedIt re-adds a re-exported
# module's converters via this proc). Otherwise a loaded module's
# re-exported converters were invisible to importers and implicit
# conversions silently stopped matching at a consumer that reaches the
# converter only through this module's re-export chain (e.g. faststreams'
# `InputStreamHandle -> InputStream` via ssz_serialization, breaking
# `SSZ.decode`/`encode`). `inclSym` guards against duplicate log entries.
c.graph.opsLog.add LogEntry(kind: ConverterEntry, module: c.module.position,
key: "", sym: conv)
proc addConverterDef*(c: PContext, conv: PSym) =
addConverter(c, conv)
@@ -416,13 +377,6 @@ proc addConverterDef*(c: PContext, conv: PSym) =
proc addPureEnum*(c: PContext, e: PSym) =
assert e != nil
add(c.graph.ifaces[c.module.position].pureEnums, e)
# record for IC: a NIF-loaded module rebuilds `Iface.pureEnums` from these log
# entries (moduleFromNifFile); without it a loaded module's pure enums were
# invisible to importers, so `importPureEnumFields` never offered their fields
# and unqualified pure-enum values stopped resolving. (Same pattern as
# `addConverterDef`.)
c.graph.opsLog.add LogEntry(kind: PureEnumEntry, module: c.module.position,
key: "", sym: e)
proc addPattern*(c: PContext, p: PSym) =
assert p != nil
@@ -669,15 +623,7 @@ proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) =
## ("find all usages of this template" would not work). We need special
## logic to remember macro/template expansions. This is done here and
## delegated to the "NIF" file mechanism.
##
## We only bother when a NIF file is actually going to be written (IC / `nim m`,
## `--compress`, semantic BIF output, or a running suggestion engine); a plain
## `nim c` throws the record away, so recording it would be pure overhead.
if info.fileIndex == InvalidFileIdx: return
if c.config.cmd == cmdM or
{optCompress, optGenBif} * c.config.globalOptions != {} or
c.config.ideActive:
c.graph.nifExpansions.mgetOrPut(c.module.position.int32, @[]).add (expandedSym, info)
discard "XXX To implement"
const
errVarForOutParamNeededX = "for a 'var' type a variable needs to be passed; but '$1' is immutable"

View File

@@ -26,15 +26,8 @@ const
proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
flags: TExprFlags = {}; expectedType: PType = nil): PNode =
rememberExpansion(c, n.info, s)
let info = getCallLineInfo(n)
# `info` (the callee identifier's position, not the whole call node) is what
# tooling wants to see as the usage site — matches `markUsed` below.
rememberExpansion(c, info, s)
# IC: this expands `s`'s body into the current module's sem, so the module
# depends on that body — record a NeedsImpl (strong) edge to `s`'s module.
# The iface cookie hashes only signatures now, so a template body edit moves
# only the impl cookie, and just the modules that expanded it re-sem.
recordIcImplDep(c.graph, s)
markUsed(c, info, s)
onUse(info, s)
# Note: This is n.info on purpose. It prevents template from creating an info
@@ -64,16 +57,6 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
elif {efWantStmt, efAllowStmt} * flags != {}:
result.typ = newTypeS(tyVoid, c)
else:
when defined(icDbgRefc):
echo "[icNoType] semOperand: ", renderTree(result, {renderNoComments}),
" kind=", result.kind,
(if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym:
" calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else:
$result[0].sym.typ.kind & " ret=" &
(if result[0].sym.typ.returnType == nil: "NIL"
else: $result[0].sym.typ.returnType.kind))
else: "")
echo getStackTrace()
localError(c.config, n.info, errExprXHasNoType %
renderTree(result, {renderNoComments}))
result.typ = errorType(c)
@@ -100,17 +83,6 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType
if result.typ == nil and efInTypeof in flags:
result.typ = c.voidType
elif result.typ == nil or result.typ == c.enforceVoidContext:
when defined(icDbgRefc):
echo "[icNoType] semExprWithType: ", renderTree(result, {renderNoComments}),
" kind=", result.kind,
(if result.kind in {nkCall, nkCommand} and result[0].kind == nkSym:
" callee=" & result[0].sym.name.s &
" calleeTyp=" & (if result[0].sym.typ == nil: "NIL" else:
$result[0].sym.typ.kind & " ret=" &
(if result[0].sym.typ.returnType == nil: "NIL"
else: $result[0].sym.typ.returnType.kind))
else: "")
echo getStackTrace()
localError(c.config, n.info, errExprXHasNoType %
renderTree(result, {renderNoComments}))
result.typ = errorType(c)
@@ -134,9 +106,7 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType
proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
result = semExprCheck(c, n, flags)
if result.typ == nil and efInTypeof in flags:
result.typ = c.voidType
elif result.typ == nil:
if result.typ == nil:
localError(c.config, n.info, errExprXHasNoType %
renderTree(result, {renderNoComments}))
result.typ = errorType(c)
@@ -227,29 +197,6 @@ proc semOpenSym(c: PContext, n: PNode, flags: TExprFlags, expectedType: PType,
# set symchoice node type back to None
n.typ = newTypeS(tyNone, c)
proc resolveOpenSymDotRhs(c: PContext, n: PNode): PNode =
## Resolves an `nkOpenSym` in the field position of a dot expression.
## The dot handling (`builtinFieldAccess`, `dotTransformation`) matches on
## the node kind of the RHS directly, so the wrapper cannot be left for
## `semExpr` to unwrap; without this the captured symbol degrades to a
## plain identifier that is then only looked up in the instantiation
## context. Mirrors `semOpenSym`: a symbol injected during instantiation
## under the current proc replaces the captured symbol, otherwise the
## captured node is used.
let inner = n[0]
result = inner
if inner.kind != nkSym: return
let id = newIdentNode(inner.sym.name, n.info)
c.isAmbiguous = false
let s2 = qualifiedLookUp(c, id, {})
if s2 != nil and not c.isAmbiguous and s2 != inner.sym:
# only consider symbols defined under the current proc:
var o = s2.owner
while o != nil:
if o == c.p.owner:
return id
o = o.owner
proc semSymChoice(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType = nil): PNode =
if n.kind == nkOpenSymChoice:
result = semOpenSym(c, n, flags, expectedType,
@@ -1573,13 +1520,10 @@ proc builtinFieldAccess(c: PContext; n: PNode; flags: var TExprFlags): PNode =
# here at all!
#if isSymChoice(n[1]): return
when defined(nimsuggest):
if c.config.ideActive:
if c.config.cmd == cmdIdeTools:
suggestExpr(c, n)
if exactEquals(c.config.m.trackPos, n[1].info): suggestExprNoCheck(c, n)
if n[1].kind == nkOpenSym:
n[1] = resolveOpenSymDotRhs(c, n[1])
var s = qualifiedLookUp(c, n, {checkAmbiguity, checkUndeclared, checkModule})
if s != nil:
if s.kind in OverloadableSyms:
@@ -1907,22 +1851,6 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode =
n.typ = n.typ.elementType
result.add(n)
proc markResultVarIsPtr(c: PContext, x: PNode) {.inline.} =
## Set `tfVarIsPtr` on the (result) sym node's type. Under IC that type can be a
## NIF-loaded (Sealed) and interned instance which must not be mutated in place
## (it could corrupt other users of the shared type, and the assert forbids it):
## give this result its own copy carrying the flag, exactly like a from-source
## compile has a fresh result type here.
if tfVarIsPtr in x.typ.flags: return
if x.typ.state == Sealed:
let fresh = copyType(x.typ, c.idgen, x.typ.owner)
fresh.incl tfVarIsPtr
x.typ = fresh
if x.kind == nkSym and x.sym.state != Sealed:
x.sym.typ = fresh
else:
x.typ.incl tfVarIsPtr
proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
if le.kind == nkHiddenDeref:
var x = le[0]
@@ -1930,10 +1858,10 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
if x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView):
n[0] = x # 'result[]' --> 'result'
n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent)
markResultVarIsPtr(c, x)
x.typ.incl tfVarIsPtr
#echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info
elif sfGlobal in x.sym.flags:
markResultVarIsPtr(c, x)
x.typ.incl tfVarIsPtr
proc borrowCheck(c: PContext, n, le, ri: PNode) =
const
@@ -2190,12 +2118,6 @@ proc semProcBody(c: PContext, n: PNode; expectedType: PType = nil): PNode =
if c.p.owner.kind notin {skMacro, skTemplate} and
c.p.resultSym != nil and c.p.resultSym.typ.isMetaType:
when defined(icDbgRefc):
echo "[icMetaRet] meta result type for ", c.p.owner.name.s, ": ",
typeToString(c.p.resultSym.typ), " kind=", c.p.resultSym.typ.kind,
" flags=", c.p.resultSym.typ.flags,
" uid=", c.p.resultSym.typ.uniqueId.module, ".", c.p.resultSym.typ.uniqueId.item,
" state=", c.p.resultSym.typ.state
if isEmptyType(result.typ):
# we inferred a 'void' return type:
c.p.resultSym.typ = errorType(c)
@@ -3407,7 +3329,7 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
c.config.expandNodeResult = $n
suggestQuit()
if c.config.ideActive: suggestExpr(c, n)
if c.config.cmd == cmdIdeTools: suggestExpr(c, n)
if nfSem in n.flags: return
case n.kind
of nkIdent, nkAccQuoted:

View File

@@ -476,12 +476,7 @@ proc foldArrayAccess(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNo
#localError(g.config, n.info, formatErrorIndexBound(idx, x.len-1) & $n)
of nkBracket:
idx -= toInt64(firstOrd(g.config, x.typ))
if isDefaultBroadcastArray(x, g.config):
# compact default array: any in-bounds index folds to the default element
if idx >= 0 and idx < toInt64(lengthOrd(g.config, x.typ.skipTypes(abstractInst))):
result = copyTree(x[0])
else: result = nil
elif idx >= 0 and idx < x.len: result = x[int(idx)]
if idx >= 0 and idx < x.len: result = x[int(idx)]
else:
result = nil
#localError(g.config, n.info, formatErrorIndexBound(idx, x.len-1) & $n)
@@ -615,21 +610,10 @@ proc getConstExpr(m: PSym, n: PNode; idgen: IdGenerator; g: ModuleGraph): PNode
var s = n.sym
case s.kind
of skEnumField:
when defined(icDbg):
if n.typ == nil:
echo "ENUMFIELD niltyp sym=", s.name.s, " symtyp=",
(if s.typ == nil: "nil" else: $s.typ.kind), " lazy=", nfLazyType in n.flags,
" symstate=", s.state, " symid=", s.itemId
result = newIntNodeT(toInt128(s.position), n, idgen, g)
of skConst:
case s.magic
of mIsMainModule:
# Under `nim m` (IC) `sfMainModule` is set on every module that is being
# compiled (so it writes its own NIF), so it cannot answer `isMainModule`;
# the IC build file marks the real entry point with `--isMainModule:on`.
let isMain = if g.config.cmd == cmdM: g.config.isMainModule
else: sfMainModule in m.flags
result = newIntNodeT(toInt128(ord(isMain)), n, idgen, g)
of mIsMainModule: result = newIntNodeT(toInt128(ord(sfMainModule in m.flags)), n, idgen, g)
of mCompileDate: result = newStrNodeT(getDateStr(), n, g)
of mCompileTime: result = newStrNodeT(getClockStr(), n, g)
of mCpuEndian: result = newIntNodeT(toInt128(ord(CPU[g.config.target.targetCPU].endian)), n, idgen, g)

View File

@@ -129,14 +129,7 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
result.typ = nil
onUse(n.info, s)
of skParam:
if s.owner == c.p.owner:
# Parameters of the routine currently being semchecked stay as local
# identifiers
result = n
else:
# Preserve captured outer parameters so nested generic procs can still
# see them after the generic pre-pass.
result = newSymNode(s, n.info)
result = n
onUse(n.info, s)
of skType:
if (s.typ != nil) and
@@ -273,7 +266,7 @@ proc semGenericStmt(c: PContext, n: PNode,
when defined(nimsuggest):
if withinTypeDesc in flags: inc c.inTypeContext
#if conf.ideActive: suggestStmt(c, n)
#if conf.cmd == cmdIdeTools: suggestStmt(c, n)
semIdeForTemplateOrGenericCheck(c.config, n, ctx.cursorInBody)
case n.kind

View File

@@ -119,44 +119,11 @@ proc freshGenSyms(c: PContext; n: PNode, owner, orig: PSym, symMap: var SymMappi
proc addParamOrResult(c: PContext, param: PSym, kind: TSymKind)
proc aliasLoadedTypedescParams(c: PContext, instantiated, orig: PSym): bool =
## When the generic being instantiated had its body LOADED from a NIF (only
## `nim m`/`nim nifc`, only for a generic owned by another module), that body
## re-sems from plain identifiers — ast2nif serialises locals/params as idents,
## not `nkSym`. A `T: typedesc[...]` param referenced as a type must then
## resolve `T` to the bound type, but the instantiated skParam carries the
## concrete type `instantiateProcType` typedesc-skipped it to, which an ident
## lookup cannot use as a type name. Shadow each such param with an `skType`
## alias of the same name in a fresh scope layer (the alias is exactly how Nim
## models "this name denotes a type"). In-process bodies reach the param as
## `nkSym` and never take this path, hence the command gate.
##
## Returns true iff a scope layer was opened; the caller must `closeScope`.
if c.config.cmd notin {cmdM, cmdNifC} or orig == nil or
orig.itemId.module == c.module.position or
orig.typ == nil or orig.typ.n == nil:
return false
result = false
let procParams = instantiated.typ.n
for i in 1..<min(procParams.len, orig.typ.n.len):
if orig.typ.n[i].kind != nkSym: continue
let origParamTyp = orig.typ.n[i].sym.typ
if origParamTyp != nil and origParamTyp.kind == tyTypeDesc and
tfUnresolved in origParamTyp.flags:
if not result:
openScope(c)
result = true
let p = procParams[i].sym
let alias = newSym(skType, p.name, c.idgen, instantiated, p.info)
alias.typ = p.typ
addDecl(c, alias)
proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
if n[bodyPos].kind != nkEmpty:
let procParams = result.typ.n
for i in 1..<procParams.len:
addDecl(c, procParams[i].sym)
let aliasLayer = aliasLoadedTypedescParams(c, result, orig)
maybeAddResult(c, result, result.ast)
inc c.inGenericInst
@@ -185,7 +152,6 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
excl(result, sfForward)
trackProc(c, result, result.ast[bodyPos])
dec c.inGenericInst
if aliasLayer: closeScope(c)
proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
for i in 0..<c.generics.len:
@@ -279,7 +245,7 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
let originalParams = result.n
result.n = originalParams.shallowCopy
for i in 1 ..< originalParams.len:
var resulti = originalParams[i].sym.typ
let resulti = originalParams[i].sym.typ
# twrong_field_caching requires these 'resetIdTable' calls:
if i > FirstParamAt:
resetIdTable(cl.symMap)
@@ -292,11 +258,6 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
let needsStaticSkipping = resulti.kind == tyFromExpr
let needsTypeDescSkipping = resulti.kind == tyTypeDesc and tfUnresolved in resulti.flags
if resulti.kind == tyFromExpr:
if resulti.state == Sealed:
# The generic was loaded from a NIF; do not brand the shared original.
# A tyFromExpr is a placeholder that `replaceTypeVarsT` resolves away,
# so a copy carries no identity that later comparisons could miss.
resulti = copyType(resulti, c.idgen, resulti.owner)
resulti.incl tfNonConstExpr
var paramType = replaceTypeVarsT(cl, resulti)
if needsStaticSkipping:
@@ -315,12 +276,6 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
let param = copySym(oldParam, c.idgen)
setOwner(param, prc)
param.typ = paramType
when defined(icDbgRefc):
echo "[icInst] ", prc.name.s, " param ", oldParam.name.s,
": ", typeToString(resulti), " (kind=", resulti.kind,
" uid=", resulti.uniqueId.module, ".", resulti.uniqueId.item,
" flags=", resulti.flags, ") -> ", typeToString(paramType),
" (kind=", paramType.kind, ")"
# The default value is instantiated and fitted against the final
# concrete param type. We avoid calling `replaceTypeVarsN` on the
@@ -328,9 +283,6 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable,
if oldParam.ast != nil:
var def = oldParam.ast.copyTree
if def.typ.kind == tyFromExpr:
if def.typ.state == Sealed:
# `copyTree` shares types; see the `resulti` comment above.
def.typ = copyType(def.typ, c.idgen, def.typ.owner)
def.typ.incl tfNonConstExpr
if not isIntLit(def.typ):
def = prepareNode(cl, def)
@@ -422,11 +374,6 @@ proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable,
## parameters to their concrete types within the generic instance.
# no need to instantiate generic templates/macros:
internalAssert c.config, fn.kind notin {skMacro, skTemplate}
# IC: instantiating `fn` consumes its generic body in the current module's
# sem — record a NeedsImpl (strong) edge to `fn`'s module. The iface cookie
# hashes only signatures now, so a generic body edit moves only the impl
# cookie, and just the modules that instantiated it re-sem.
recordIcImplDep(c.graph, fn)
# generates an instantiated proc
if c.instCounter > 50:
globalError(c.config, info, "generic instantiation too nested")
@@ -508,10 +455,6 @@ proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable,
# This is needed for cyclic module dependencies where generic instances
# may be created in one module but referenced from another.
logGenericInstance(c.graph, result)
# Under IC the instance's NIF name must be canonical across modules:
# derive its `disamb` from the instantiation identity (generic +
# concrete types) instead of the per-module counter.
setInstanceDisamb(c.graph, result, fn, entry.concreteTypes)
# bug #12985 bug #22913
# TODO: use the context of the declaration of generic functions instead
# TODO: consider fixing options as well

View File

@@ -43,8 +43,17 @@ proc semAddr(c: PContext; n: PNode): PNode =
result.typ = makePtrType(c, x.typ.skipTypes({tySink}))
proc semTypeOf(c: PContext; n: PNode): PNode =
let typExpr = semTypeOfImpl(c, n)
var m = BiggestInt 1 # typeOfIter
if n.len == 3:
let mode = semConstExpr(c, n[2])
if mode.kind != nkIntLit:
localError(c.config, n.info, "typeof: cannot evaluate 'mode' parameter at compile-time")
else:
m = mode.intVal
result = newNodeI(nkTypeOfExpr, n.info)
inc c.inTypeofContext
defer: dec c.inTypeofContext # compiles can raise an exception
let typExpr = semExprWithType(c, n[1], if m == 1: {efInTypeof} else: {})
result.add typExpr
if typExpr.typ.kind == tyFromExpr:
typExpr.typ.incl tfNonConstExpr
@@ -693,10 +702,5 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
if n[1].kind in {nkStmtListExpr, nkBlockExpr,
nkIfExpr, nkCaseStmt, nkTryStmt}:
localError(c.config, n.info, "Nested expressions cannot be moved: '" & $n[1] & "'")
of mMove:
result = n
if isCursor(n[1]):
localError(c.config, n.info, errFailedMove,
"cannot move cursor '" & $n[1] & "'; a cursor does not own its value")
else:
result = n

View File

@@ -93,7 +93,6 @@ type
graph: ModuleGraph
c: PContext
escapingParams: IntSet
inNimvmBranch: int
PEffects = var TEffects
const
@@ -498,33 +497,6 @@ proc addRaiseEffect(a: PEffects, e, comesFrom: PNode) =
if not isDefectException(e.typ):
throws(a.exc, e, comesFrom)
proc skipHiddenConv(n: PNode): PNode =
result = n
while true:
case result.kind
of nkHiddenStdConv, nkHiddenSubConv:
result = result[1]
else: break
proc addRaiseEffectsFromExpr(a: PEffects, e, comesFrom: PNode) =
if e.isNil:
return
case e.kind
of nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr:
if e.len > 0:
addRaiseEffectsFromExpr(a, e.lastSon.skipHiddenConv, comesFrom)
of nkIfExpr, nkIfStmt:
for branch in items(e):
if branch.len > 0:
addRaiseEffectsFromExpr(a, branch.lastSon.skipHiddenConv, comesFrom)
of nkCaseStmt:
for i in 1..<e.len:
let branch = e[i]
if branch.len > 0:
addRaiseEffectsFromExpr(a, branch.lastSon.skipHiddenConv, comesFrom)
else:
addRaiseEffect(a, e, comesFrom)
proc addTag(a: PEffects, e, comesFrom: PNode) =
var aa = a.tags
for i in 0..<aa.len:
@@ -1118,56 +1090,6 @@ proc trackCall(tracked: PEffects; n: PNode) =
#if canRaise(a):
# echo "this can raise ", tracked.config $ n.info
let op = a.typ
# A routine whose body reaches a compile-time-only magic (`macros.error`,
# `slurp`, `gorge`, `getAst`, …) can never be code-generated — the C/JS
# backends reject those magics (ccgexprs `errXMustBeCompileTime`). Such a
# routine is compile-time-only by construction; mark it `sfCompileTime` so it
# is treated uniformly as such. Non-IC pruned it by demand-driven codegen, but
# the per-module IC backend emits every owned routine (no DCE) and would
# otherwise feed the magic to codegen. Mirrors the `tfTriggersCompileTime ->
# sfCompileTime` path in `semProcAux`.
#
# GATE TO THE IC STAGES ONLY (`cmdM` sem + `cmdNifC` cg). The magic can reach a
# runtime proc's body via an INLINED TEMPLATE (not a macro/template *owner*, so
# the `insideMeta` walk below can't see it) — e.g. confutils' runtime
# `addConfigFile`/json-serialization's `inputFile` expand a serialization
# template that pastes a `getAst`/`quote` magic inline. Under plain `nim c` such
# a proc still code-generates fine (the magic folds / is demand-pruned), so
# marking it `sfCompileTime` there is a pure regression: "request to generate
# code for .compileTime proc". Only the emit-everything IC backend needs the
# mark, so restrict it to `{cmdM, cmdNifC}` (was `!= cmdNimscript`, which
# wrongly swept in `cmdCompileToC`/JS/`cmdCheck`).
if a.kind == nkSym and a.sym.magic in {mNLen..mNError, mSlurp..mQuoteAst} and
tracked.owner != nil and tracked.owner.kind in routineKinds and
tracked.config.cmd in {cmdM, cmdNifC} and tracked.inNimvmBranch == 0:
# ...but NOT under `nim e`: nimscript has no codegen backend to protect, and
# marking a routine `sfCompileTime` makes `semExpr` eagerly fold calls to it
# at sem time (emConst), where module-level globals it reads have no VM slot
# yet — distros' `detectOsWithAllCmd` reaches `gorge` and reads the plain
# global `unameRes` → "cannot evaluate at compile time: unameRes". In the
# normal nimscript run (emRepl) the module's var section runs first and the
# slot exists, so the marking is both unnecessary and harmful here.
#
# ...and NOT if the routine is — or is nested inside — a macro/template:
# those are VM-only (never code-generated), so the per-module IC backend has
# nothing to protect there, while `sfCompileTime` on a macro-internal nested
# closure breaks its captured-variable access in the VM ("cannot evaluate at
# compile time: n" — `tests/macros/tmacros1`'s `innerProc` reading the
# macro-local `n`). Walk the owner chain and bail on the first
# skMacro/skTemplate. NB mark `tracked.owner` (the routine that directly
# reaches the magic), NOT its outermost enclosing: a runtime proc may legally
# nest a compile-time helper — `tests/generics/tunique_type`'s `[]` proc
# contains a nested `buildResult` macro — and marking the proc would wrongly
# make IT compile-time ("request to generate code for .compileTime proc: []").
var encl = tracked.owner
var insideMeta = false
while encl != nil and encl.kind != skModule:
if encl.kind in {skMacro, skTemplate}:
insideMeta = true
break
encl = encl.skipGenericOwner
if not insideMeta:
incl(tracked.owner, sfCompileTime)
if n.typ != nil:
if tracked.owner.kind != skMacro and n.typ.skipTypes(abstractVar).kind != tyOpenArray:
createTypeBoundOps(tracked, n.typ, n.info)
@@ -1209,17 +1131,7 @@ proc trackCall(tracked: PEffects; n: PNode) =
else:
if laxEffects notin tracked.c.config.legacyFeatures and a.kind == nkSym and
a.sym.kind in routineKinds:
# A hook reaching here has no effect list yet, i.e. it has not been
# effect-tracked. Propagating from its (still unset) type flags would
# spuriously mark the caller GC-unsafe/side-effecting: e.g. under
# `nim ic` a concrete `=destroy` reached through a generic
# instantiation is not analyzed before the instance body is tracked
# here. Skip all such hooks (generalizes #25940, which special-cased
# `=asgn`/`=sink`/`=dup`); once analyzed they carry an effect list and
# take the branch below.
let (isHook, _) = findHookKind(a.sym.name.s)
if not isHook:
propagateEffects(tracked, n, a.sym)
propagateEffects(tracked, n, a.sym)
else:
mergeRaises(tracked, effectList[exceptionEffects], n)
mergeTags(tracked, effectList[tagEffects], n)
@@ -1396,8 +1308,6 @@ proc allowCStringConv(n: PNode): bool =
proc track(tracked: PEffects, n: PNode) =
case n.kind
of nkTypeOfExpr:
discard "typeof() never evaluates its operand; not a definite-assignment use"
of nkSym:
useVar(tracked, n)
if n.sym.typ != nil and tfHasAsgn in n.sym.typ.flags:
@@ -1414,7 +1324,7 @@ proc track(tracked: PEffects, n: PNode) =
if n[0].kind != nkEmpty:
n[0].info = n.info
#throws(tracked.exc, n[0])
addRaiseEffectsFromExpr(tracked, n[0], n)
addRaiseEffect(tracked, n[0], n)
for i in 0..<n.safeLen:
track(tracked, n[i])
createTypeBoundOps(tracked, n[0].typ, n.info)
@@ -1503,9 +1413,7 @@ proc track(tracked: PEffects, n: PNode) =
of nkCaseStmt: trackCase(tracked, n)
of nkWhen: # This should be a "when nimvm" node.
let oldState = tracked.init.len
inc tracked.inNimvmBranch
track(tracked, n[0][1])
dec tracked.inNimvmBranch
tracked.init.setLen(oldState)
track(tracked, n[1][0])
of nkIfStmt, nkIfExpr: trackIf(tracked, n)
@@ -1649,11 +1557,10 @@ proc track(tracked: PEffects, n: PNode) =
message(tracked.config, n.info, warnPtrToCstringConv,
$n[1].typ)
# Check for implicit range conversions. Compile-time constants are already
# fully known here, so only non-constant values need the downsizing warning.
# Check for implicit range conversions
if n.kind == nkHiddenStdConv and (not tracked.isArrayIndexing) and
shouldWarnRangeConversion(tracked.config, n.info, n.typ, n[1].typ) and
getConstExpr(tracked.ownerModule, n[1], tracked.c.idgen, tracked.graph) == nil:
n[1].kind notin {nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit} and
shouldWarnRangeConversion(tracked.config, n.info, n.typ, n[1].typ):
message(tracked.config, n.info, warnImplicitRangeConversion,
typeToString(n[1].typ) & " -> " & typeToString(n.typ))
@@ -1784,18 +1691,13 @@ proc setEffectsForProcType*(g: ModuleGraph; t: PType, n: PNode; s: PSym = nil) =
elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}):
effects[exceptionEffects] = newNodeI(nkArgList, effects.info)
let forbidsSpec = effectSpec(n, wForbids)
let tagsSpec = effectSpec(n, wTags)
if not isNil(tagsSpec):
effects[tagEffects] = tagsSpec
elif not isNil(forbidsSpec):
# `.forbids` without `.tags` still declares a known empty tag set.
# Leaving this as nil would mean "unknown tags", which later widens
# indirect calls to `RootEffect`.
effects[tagEffects] = newNodeI(nkArgList, effects.info)
elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}):
effects[tagEffects] = newNodeI(nkArgList, effects.info)
let forbidsSpec = effectSpec(n, wForbids)
if not isNil(forbidsSpec):
effects[forbiddenEffects] = forbidsSpec
elif s != nil and (s.magic != mNone or {sfImportc, sfExportc} * s.flags == {sfImportc}):

View File

@@ -531,7 +531,7 @@ proc semUsing(c: PContext; n: PNode): PNode =
if not isTopLevel(c): localError(c.config, n.info, errXOnlyAtModuleScope % "using")
for i in 0..<n.len:
var a = n[i]
if c.config.ideActive: suggestStmt(c, a)
if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
if a.kind == nkCommentStmt: continue
if a.kind notin {nkIdentDefs, nkVarTuple, nkConstDef}: illFormedAst(a, c.config)
checkMinSonsLen(a, 3, c.config)
@@ -838,7 +838,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
for i in 0..<n.len:
var a = n[i]
if c.config.ideActive: suggestStmt(c, a)
if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
if a.kind == nkCommentStmt: continue
if a.kind notin {nkIdentDefs, nkVarTuple}: illFormedAst(a, c.config)
checkMinSonsLen(a, 3, c.config)
@@ -994,7 +994,7 @@ proc semConst(c: PContext, n: PNode): PNode =
var b: PNode
for i in 0..<n.len:
var a = n[i]
if c.config.ideActive: suggestStmt(c, a)
if c.config.cmd == cmdIdeTools: suggestStmt(c, a)
if a.kind == nkCommentStmt: continue
if a.kind notin {nkConstDef, nkVarTuple}: illFormedAst(a, c.config)
checkMinSonsLen(a, 3, c.config)
@@ -1535,7 +1535,7 @@ proc typeSectionLeftSidePass(c: PContext, n: PNode) =
while i < n.len: # n may grow due to type pragma macros
var a = n[i]
when defined(nimsuggest):
if c.config.ideActive:
if c.config.cmd == cmdIdeTools:
inc c.inTypeContext
suggestStmt(c, a)
dec c.inTypeContext
@@ -1812,7 +1812,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
var remainingOwners = initIntSet()
for (owner, _, _) in c.forwardTypeUpdates:
remainingOwners.incl owner.id
while c.forwardTypeUpdates.len > 0:
let pending = move c.forwardTypeUpdates
var madeProgress = false
@@ -1829,7 +1829,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
c.forwardTypeUpdates.add (owner, typ, typeNode)
elif not remainingOwners.missingOrExcl(owner.id):
madeProgress = true
if not madeProgress:
# can't error here unfortunately
break
@@ -2621,47 +2621,15 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
addParams(c, proto.typ.n, proto.kind)
proto.info = s.info # more accurate line information
proto.options = s.options
# `s` (the impl symbol) is discarded in favour of `proto`. It still carries
# `s.ast == n` (set above) and stays reachable as the owner of body-local
# symbols, so under IC it would be serialized as a SECOND, body-bearing
# `proc` entry — a phantom duplicate of `proto`. The per-module backend then
# codegens that phantom, whose `result` is owned by `proto` (addResult below
# re-parents it), not by the phantom: lambdalifting's capture check
# (`result.skipGenericOwner != owner`) then wrongly classifies `result` as a
# captured outer variable → "'result' … cannot be captured". Drop the
# discarded impl's body so it can never be emitted as a routine (same leak
# class the `miscPos` adoption below guards against for generic params).
let discardedImpl = s
s = proto
n[genericParamsPos] = proto.ast[genericParamsPos]
n[paramsPos] = proto.ast[paramsPos]
n[pragmasPos] = proto.ast[pragmasPos]
# miscPos holds this definition's *original* generic-param node (kept for
# error messages, see setGenericParamsMisc / issue #1713). For an impl that
# resolves to a forward decl, that node was analysed under the now-discarded
# impl symbol and its generic-param constraint types are owned by it. Adopt
# the prototype's miscPos so the discarded impl sym is fully unreachable —
# otherwise it leaks (via `proto.ast = n` below) as a type owner and gets
# serialized as a phantom duplicate overload under IC.
n[miscPos] = proto.ast[miscPos]
if n[namePos].kind != nkSym: internalError(c.config, n.info, "semProcAux")
n[namePos].sym = proto
if importantComments(c.config) and proto.ast.comment.len > 0:
n.comment = proto.ast.comment
proto.ast = n # needed for code generation
if discardedImpl != proto:
discardedImpl.ast = nil
# The impl symbol is discarded in favour of `proto`, but it stays `Complete`
# in this module, so `ast2nif.shouldWriteSymDef` still serializes it. With
# `sfExported` it would be written importable (`x` marker) and an importer
# would load BOTH it and `proto` into the overload set: "ambiguous call;
# both foo and foo" (identical signatures). Normally a discarded impl is a
# gensym/transient that isn't reached this way, but a `{.async: (raises).}`
# forward-decl + impl reconciles HERE with both syms exported. Strip the
# export so the design's "forward declarations are never importable" holds —
# the def still serializes (other refs may resolve to it) but is invisible
# to importer overload resolution; `proto` carries the export.
excl(discardedImpl, sfExported)
popOwner(c)
pushOwner(c, s)
@@ -2674,11 +2642,6 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
elif s.name.s == "()" and callOperator notin c.features:
localError(c.config, n.info, "the overloaded " & s.name.s &
" operator has to be enabled with {.experimental: \"callOperator\".}")
elif sfImportc notin s.flags and (s.name.s == ">" or s.name.s == ">=" or s.name.s == "!="):
# ignore imported procs as these operators in backend language might have different semantics
let op1 = if s.name.s == "!=": "==" elif s.name.s == ">": "<" else: "<="
message(c.config, n.info, warnInvalidCmpOp, "define `" & op1 & "` instead of `" & s.name.s & "` to implement user defined comparison operator. " &
"it allows you to use `" & s.name.s & "` automatically.")
if sfBorrow in s.flags and c.config.cmd notin cmdDocLike:
result[bodyPos] = c.graph.emptyNode
@@ -2892,8 +2855,7 @@ proc incMod(c: PContext, n: PNode, it: PNode, includeStmtResult, resolvedIncStmt
proc evalInclude(c: PContext, n: PNode): PNode =
result = newNodeI(nkStmtList, n.info)
var resolvedIncStmt: PNode = nil
if {optCompress, optGenBif} * c.config.globalOptions != {} or
c.config.cmd == cmdM:
if optCompress in c.config.globalOptions:
# New resolve the include filenames to string literals that contain absolute paths,
# nicer for IC:
resolvedIncStmt = newNodeI(nkIncludeStmt, n.info)

View File

@@ -219,10 +219,9 @@ proc semSet(c: PContext, n: PNode, prev: PType): PType =
result = newOrPrevType(tySet, prev, c)
if n.len == 2 and n[1].kind != nkEmpty:
var base = semTypeNode(c, n[1], nil)
if base.kind == tyTypeDesc: base = base.base # unwrap from type traits like distinctBase
addSonSkipIntLit(result, base, c.idgen)
if base.kind in {tyGenericInst, tyAlias, tySink}: base = skipModifier(base)
if base.kind notin {tyGenericParam, tyGenericInvocation, tyFromExpr}:
if base.kind notin {tyGenericParam, tyGenericInvocation}:
if base.kind == tyForward:
c.forwardTypeUpdates.add (getCurrOwner(c), result, n)
elif not isOrdinalType(base, allowEnumWithHoles = true):
@@ -513,13 +512,7 @@ proc semArrayIndex(c: PContext, n: PNode): PType =
if c.inGenericContext > 0: result.incl tfUnresolved
else:
result = e.typ.skipTypes({tyTypeDesc})
if result.state != Sealed:
# For a type loaded from the IC cache we skip the flag instead of
# mutating (or copying) the type: tfImplicitStatic has no readers in
# the compiler, and a copy would get a fresh itemId, breaking enum
# identity (`sameEnumTypes` compares ids) — `arr[enumVal]` on an
# `array[LoadedEnum, T]` would no longer typecheck.
result.incl tfImplicitStatic
result.incl tfImplicitStatic
elif e.kind in (nkCallKinds + {nkBracketExpr}) and hasUnresolvedArgs(c, e):
if not isOrdinalType(e.typ.skipTypes({tyStatic, tyAlias, tyGenericInst, tySink})):
localError(c.config, n[1].info, errOrdinalTypeExpected % typeToString(e.typ, preferDesc))
@@ -1362,7 +1355,7 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode,
for i in 0..<paramType.len - 1:
if paramType[i].kind == tyStatic:
var staticCopy = paramType[i].exactReplica(c.idgen)
var staticCopy = paramType[i].exactReplica
staticCopy.incl tfInferrableStatic
result.rawAddSon staticCopy
else:
@@ -1898,12 +1891,6 @@ proc semTypeExpr(c: PContext, n: PNode; prev: PType): PType =
# by macros. Only macros can summon unnamed types
# and cast spell upon AST. Here we need to give
# it a name taken from left hand side's node
if result.state == Sealed:
# The unnamed type was loaded from a dependency's NIF and must not
# be mutated in place; attach the name to a fresh copy instead.
let orig = result
result = copyType(orig, c.idgen, getCurrOwner(c))
copyTypeProps(c.graph, c.idgen.module, result, orig)
result.sym = prev.sym
result.sym.typ = result
else:
@@ -2076,57 +2063,6 @@ proc semStaticType(c: PContext, childNode: PNode, prev: PType): PType =
result.rawAddSon(base)
result.incl tfHasStatic
proc semTypeOfImpl(c: PContext; n: PNode): PNode =
var m = BiggestInt 1 # typeOfIter
var modifierMode = BiggestInt 0 # CompatibleTypeModifiers
type
TypeOfParams = enum
topMode
topModifier
if n.len in 3 .. 4:
for i in 2 ..< n.len:
var argKind = topMode
var arg: PNode = nil
if n[i].kind == nkExprEqExpr and n[i][0].kind == nkIdent:
# named param
case n[i][0].ident.s
of "mode": argKind = topMode
of "modifierMode": argKind = topModifier
else:
localError(c.config, n.info, "typeof: got unknown parameter name")
arg = n[i][1]
else:
if i == 2:
argKind = topMode
else:
argKind = topModifier
arg = n[i]
case argKind
of topMode:
let mode = semConstExpr(c, arg)
if mode.kind != nkIntLit:
localError(c.config, n.info, "typeof: cannot evaluate 'mode' parameter at compile-time")
else:
m = mode.intVal
of topModifier:
let modMode = semConstExpr(c, arg)
if modMode.kind != nkIntLit:
localError(c.config, n.info, "typeof: cannot evaluate 'modifierMode' parameter at compile-time")
else:
modifierMode = modMode.intVal
inc c.inTypeofContext
defer: dec c.inTypeofContext # compiles can raise an exception
var typExpr = semExprNoDeref(c, n[1], if m == 1: {efInTypeof} else: {})
if modifierMode == 0:
# CompatibleTypeModifiers
typExpr.typ = typExpr.typ.skipTypes({tyVar, tyLent})
elif modifierMode == 1:
# RemoveTypeModifiers
typExpr.typ = typExpr.typ.skipTypes({tyVar, tyLent, tySink})
result = typExpr
proc semTypeOf(c: PContext; n: PNode; prev: PType): PType =
openScope(c)
inc c.inTypeofContext
@@ -2147,7 +2083,16 @@ proc semTypeOf(c: PContext; n: PNode; prev: PType): PType =
proc semTypeOf2(c: PContext; n: PNode; prev: PType): PType =
openScope(c)
let ex = semTypeOfImpl(c, n)
var m = BiggestInt 1 # typeOfIter
if n.len == 3:
let mode = semConstExpr(c, n[2])
if mode.kind != nkIntLit:
localError(c.config, n.info, "typeof: cannot evaluate 'mode' parameter at compile-time")
else:
m = mode.intVal
inc c.inTypeofContext
defer: dec c.inTypeofContext # compiles can raise an exception
let ex = semExprWithType(c, n[1], if m == 1: {efInTypeof} else: {})
closeScope(c)
result = ex.typ
if result.kind == tyFromExpr:
@@ -2191,7 +2136,7 @@ proc semTypeIdent(c: PContext, n: PNode): PSym =
localError(c.config, n.info, errTypeExpected)
return errorSym(c, n)
result = result.typ.sym.copySym(c.idgen)
result.typ = exactReplica(result.typ, c.idgen)
result.typ = exactReplica(result.typ)
result.typ.incl tfUnresolved
if result.kind == skGenericParam:
@@ -2234,7 +2179,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType =
result = nil
inc c.inTypeContext
if c.config.ideActive: suggestExpr(c, n)
if c.config.cmd == cmdIdeTools: suggestExpr(c, n)
case n.kind
of nkEmpty: result = n.typ
of nkTypeOfExpr:

View File

@@ -272,17 +272,10 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0; expectedType: PT
if n == nil: return
result = copyNode(n)
if n.typ != nil:
var nodeTyp = n.typ
if nodeTyp.kind == tyFromExpr:
if n.typ.kind == tyFromExpr:
# type of node should not be evaluated as a static value
if nodeTyp.state == Sealed:
# IC: do not brand the loaded shared original — a tyFromExpr is a
# placeholder that `replaceTypeVarsT` resolves away, so the copy
# carries no identity later comparisons could miss (mirrors
# `instantiateProcType`)
nodeTyp = copyType(nodeTyp, cl.c.idgen, nodeTyp.owner)
nodeTyp.incl tfNonConstExpr
result.typ = replaceTypeVarsT(cl, nodeTyp)
n.typ.incl tfNonConstExpr
result.typ = replaceTypeVarsT(cl, n.typ)
checkMetaInvariants(cl, result.typ)
case n.kind
of nkNone..pred(nkSym), succ(nkSym)..nkNilLit:
@@ -294,22 +287,13 @@ proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0; expectedType: PT
replaceTypeVarsS(cl, n.sym, result.typ)
else:
replaceTypeVarsS(cl, n.sym, replaceTypeVarsT(cl, n.sym.typ))
if result.sym.kind == skField and
if result.sym.kind == skField and result.sym.ast != nil and
(cl.owner == nil or result.sym.owner == cl.owner):
if result.sym.ast != nil:
# instantiate default value of object/tuple field
var n = result.sym.ast
cl.c.fitDefaultNode(cl.c, n, result.sym.typ)
result.sym.ast = n
result.sym.typ = n.typ.skipIntLit(cl.c.idgen)
elif result.typ != nil:
# The field SYM can be SHARED across the branches of an `nkRecWhen` (the
# generic body reuses one `value` PSym, so it carries the LAST branch's
# type), while the resolved field NODE carries the correct branch type.
# Sync the sym to the node so the instantiated field's sym-type and
# node-type agree (else a generic-object instance serializes a field
# whose sym-type diverges from its node-type -> loader/computeSize crash).
result.sym.typ = result.typ
# instantiate default value of object/tuple field
var n = result.sym.ast
cl.c.fitDefaultNode(cl.c, n, result.sym.typ)
result.sym.ast = n
result.sym.typ = n.typ.skipIntLit(cl.c.idgen)
# sym type can be nil if was gensym created by macro, see #24048
if result.sym.typ != nil and result.sym.typ.kind == tyVoid:
# don't add the 'void' field
@@ -403,13 +387,6 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
# don't bind `auto` return type to a previous binding of `auto`
return nil
result = cl.typeMap.lookup(t)
when defined(icDbgRefc):
if t.kind in {tyGenericParam, tyTypeDesc}:
echo "[icBind] lookup ", t.kind, " ", typeToString(t), " uid=", t.uniqueId.module, ".",
t.uniqueId.item, " itemId=", t.itemId.module, ".", t.itemId.item,
" state=", t.state, " flags=", t.flags, " -> ",
(if result != nil: typeToString(result) else: "MISS"),
" allowMeta=", cl.allowMetaTypes
if result == nil:
if cl.allowMetaTypes or tfRetType in t.flags: return
localError(cl.c.config, t.sym.info, "cannot instantiate: '" & typeToString(t) & "'")
@@ -424,7 +401,7 @@ proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
proc instCopyType*(cl: var TReplTypeVars, t: PType): PType =
# XXX: relying on allowMetaTypes is a kludge
if cl.allowMetaTypes:
result = t.exactReplica(cl.c.idgen)
result = t.exactReplica
else:
result = copyType(t, cl.c.idgen, t.owner)
copyTypeProps(cl.c.graph, cl.c.idgen.module, result, t)
@@ -469,13 +446,6 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
header[i] = x
propagateToOwner(header, x)
else:
# Under IC `t` may be a loaded dep type (Sealed/immutable); mutating it
# would assert, so propagate into a copy. For non-Sealed types keep
# devel's in-place propagation: unconditionally copying here changes
# `header != t` and with it the cached-instance lookup below, which
# regressed non-IC generic instantiations (arraymancer: a cached
# NimSeqV2 instance with stale flags was returned for a cast target).
if header == t and t.state == Sealed: header = instCopyType(cl, t)
propagateToOwner(header, x)
if header != t:
@@ -489,11 +459,7 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
else:
header = instCopyType(cl, t)
# The instantiating module owns the instance (and announces it as an offer):
# the generic body's module (`t.genericHead.owner`) has no business owning a
# type that references instantiation-site types — that is the IC parent->child
# heap leak the write-barrier surfaces.
result = newType(tyGenericInst, cl.c.idgen, cl.c.module, son = header.genericHead)
result = newType(tyGenericInst, cl.c.idgen, t.genericHead.owner, son = header.genericHead)
result.flags = header.flags
# be careful not to propagate unnecessary flags here (don't use rawAddSon)
# ugh need another pass for deeply recursive generic types (e.g. PActor)
@@ -531,14 +497,8 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
let bbody = last body
var newbody = replaceTypeVarsT(cl, bbody, isInstValue = true)
cl.skipTypedesc = oldSkipTypedesc
let newbodyFlags = newbody.flags + (t.flags + body.flags - tfInstClearedFlags)
if newbody.state != Sealed:
newbody.flags = newbodyFlags
# else: `newbody` is a type loaded from a dep module (it can even be a
# builtin like `int` when the generic's body is computed by a macro) and is
# immutable under IC. Skip the in-place flag accumulation on the shared
# type; the instance `result` still receives the flags below.
result.flags = result.flags + newbodyFlags - tfInstClearedFlags
newbody.flags = newbody.flags + (t.flags + body.flags - tfInstClearedFlags)
result.flags = result.flags + newbody.flags - tfInstClearedFlags
setToPreviousLayer(cl.typeMap)
@@ -558,11 +518,8 @@ proc handleGenericInvocation(cl: var TReplTypeVars, t: PType): PType =
# generics *when the type is constructed*:
cl.c.graph.setAttachedOp(cl.c.module.position, newbody, attachedDeepCopy,
cl.c.instTypeBoundOp(cl.c, dc, result, cl.info, attachedDeepCopy, 1))
if newbody.typeInst == nil and newbody.state != Sealed:
if newbody.typeInst == nil:
# doAssert newbody.typeInst == nil
# An IC-loaded (Sealed) `newbody` keeps whatever `typeInst` its defining
# module serialized; recording this process's first instantiation on the
# shared type is not possible (and was always first-wins anyway).
newbody.typeInst = result
if tfRefsAnonObj in newbody.flags and newbody.kind != tyGenericInst:
# can come here for tyGenericInst too, see tests/metatype/ttypeor.nim
@@ -847,21 +804,11 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType, isInstValue = false):
# trough replaceObjBranches in order to resolve any pending nkRecWhen nodes
result = t
# Slow path, we have some work to do. CRUCIAL: only ever mutate a type that
# is LOCAL to the module we are instantiating in (`uniqueId.module ==
# idgen.module`). A type loaded from another module's NIF (foreign) already
# had its object branches resolved when it was originally compiled; mutating
# it in place here is an old→new heap write that re-homes the loaded type to
# the instantiation site (its sym then looks owned by the consumer module and
# loses its `info`, colliding C type names — the libp2p `Message` bug). The
# prior `state != Sealed` guard was insufficient: a freshly-LOADED type is
# `Complete`, not `Sealed` (`Sealed` only means "already re-written to a NIF").
if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and
t.elementType.n != nil and t.elementType.uniqueId.module == cl.c.idgen.module.int:
# Slow path, we have some work to do
if t.kind == tyRef and t.hasElementType and t.elementType.kind == tyObject and t.elementType.n != nil:
discard replaceObjBranches(cl, t.elementType.n)
elif result.n != nil and t.kind == tyObject and result.state != Sealed and
result.uniqueId.module == cl.c.idgen.module.int:
elif result.n != nil and t.kind == tyObject:
# Invalidate the type size as we may alter its structure
result.size = -1
result.n = replaceObjBranches(cl, result.n)
@@ -913,10 +860,7 @@ proc recomputeFieldPositions*(t: PType; obj: PNode; currPosition: var int) =
for i in 1..<obj.len:
recomputeFieldPositions(nil, lastSon(obj[i]), currPosition)
of nkSym:
# A field loaded from the IC cache is already at its final position and must
# not be mutated; only freshly instantiated fields need (re)positioning.
if obj.sym.state != Sealed:
obj.sym.position = currPosition
obj.sym.position = currPosition
inc currPosition
else: discard "cannot happen"

View File

@@ -10,7 +10,6 @@
## Computes hash values for routine (proc, method etc) signatures.
import ast, ropes, modulegraphs, options, msgs, pathutils
from lineinfos import FileIndex
from std/hashes import Hash
import std/tables
import types
@@ -53,17 +52,7 @@ proc hashSym(c: var MD5Context, s: PSym) =
c &= ":anon"
else:
var it = s
when defined(icDbgHash):
var ownerSteps = 0
while it != nil:
when defined(icDbgHash):
inc ownerSteps
if ownerSteps >= 1000 and ownerSteps <= 1030:
echo "OWNERLOOP(hashSym) n=", ownerSteps, " sym=", it.name.s, " kind=", it.kind,
" id=", it.itemId, " flags=", it.flags, " state=", it.state,
" start=", s.name.s, " startId=", s.itemId
elif ownerSteps == 1031:
raiseAssert "owner-chain cycle detected, see OWNERLOOP dump above"
c &= it.name.s
c &= "."
it = it.owner
@@ -75,30 +64,8 @@ proc hashTypeSym(c: var MD5Context, s: PSym; conf: ConfigRef) =
c &= ":anon"
else:
var it = s
# The source file path disambiguates same-named object types from different
# modules whose owner-chain names also coincide (e.g. libp2p kademlia/protobuf
# `Message` vs rendezvous/protobuf `Message`, both modules named `protobuf`).
# A type sym that reaches the backend as a `Complete` stub never individually
# loaded carries `unknownLineInfo` (fileIndex -1), which `toFullPath` collapses
# to the `???` placeholder — so the two would hash to ONE mangled C name and the
# wrong struct gets emitted. Fall back to the sym's HOME module file (its
# per-module NIF-suffix path, stable+unique) for the path. Only fires on a -1
# fileIndex; non-IC type syms always have a real `info`, so the fast path is
# taken and the hash is unchanged (koch boot byte-equal).
let infoFi = s.info.fileIndex
let pathFi = if infoFi.int32 >= 0'i32: infoFi else: s.itemId.module.int32.FileIndex
c &= customPath(conf.toFullPath(pathFi))
when defined(icDbgHash):
var ownerSteps = 0
c &= customPath(conf.toFullPath(s.info))
while it != nil:
when defined(icDbgHash):
inc ownerSteps
if ownerSteps >= 1000 and ownerSteps <= 1030:
echo "OWNERLOOP n=", ownerSteps, " sym=", it.name.s, " kind=", it.kind,
" id=", it.itemId, " flags=", it.flags, " state=", it.state,
" start=", s.name.s, " startId=", s.itemId
elif ownerSteps == 1031:
raiseAssert "owner-chain cycle detected, see OWNERLOOP dump above"
if sfFromGeneric in it.flags and it.kind in routineKinds and
it.typ != nil:
hashType c, it.typ, {CoProc}, conf
@@ -135,44 +102,15 @@ proc hashTree(c: var MD5Context, n: PNode; flags: set[ConsiderFlag]; conf: Confi
else:
for i in 0..<n.len: hashTree(c, n[i], flags, conf)
when defined(icDbgHash):
var hashDepth = 0
var hashCalls = 0
var hashMaxDepth = 0
proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) =
if t == nil:
c &= "\254"
return
when defined(icDbgHash):
inc hashDepth
inc hashCalls
if hashDepth > hashMaxDepth: hashMaxDepth = hashDepth
if hashCalls >= 500_000_000 and hashCalls <= 500_000_300:
echo "HASHLOOP n=", hashCalls, " d=", hashDepth, " kind=", t.kind, " id=", t.itemId,
" uniq=", t.uniqueId, " sym=", (if t.sym != nil: t.sym.name.s else: "NIL"),
" state=", t.state, " owner=", (if t.owner != nil: t.owner.name.s else: "NIL")
elif hashCalls == 500_000_301:
echo "HASHLOOP maxDepth=", hashMaxDepth
raiseAssert "hashType runaway detected, see HASHLOOP dump above"
defer:
dec hashDepth
# Ensure type is fully loaded before hashing to avoid hash changing
# as properties are accessed and trigger lazy loading.
backendEnsureMutable(t)
# Bare type-class keywords used as a typedesc without arguments (e.g. `array`,
# `range`, `distinct` passed to `signatureHash`) have no children, so the
# structural branches below would index a non-existent `elementType`. Hash them
# by kind (+ sym for an extra, stable distinction) — enough for a stable,
# distinct identity. (`seq`/`openArray`/`tuple` already fall through the empty
# `else` loop unharmed; this covers the branches that index `elementType`.)
if t.kind in {tyArray, tyRange, tyDistinct} and not t.hasElementType:
c &= char(t.kind)
if t.sym != nil: c.hashSym(t.sym)
return
case t.kind
of tyGenericInvocation:
for a in t.kids:
@@ -203,10 +141,9 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
if CoConsiderOwned in flags:
c &= char(t.kind)
c.hashType t.skipModifier, flags, conf
of tyBool, tyChar, tyPointer, tyCstring, tyInt..tyUInt64:
# no canonicalization for builtin scalar-ish / pointer-like types, so
# that e.g. ``pid_t`` or an imported ``pointer`` alias keep their
# backend spelling instead of collapsing into the generic Nim builtin:
of tyBool, tyChar, tyInt..tyUInt64:
# no canonicalization for integral types, so that e.g. ``pid_t`` is
# produced instead of ``NI``:
c &= char(t.kind)
if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}:
c.hashSym(t.sym)
@@ -274,7 +211,6 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
c.hashTree(t.n, {}, conf)
of tyTuple:
c &= char(t.kind)
c &= t.len
if t.n != nil and CoType notin flags:
for i in 0..<t.n.len:
assert(t.n[i].kind == nkSym)
@@ -312,29 +248,6 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
c.hashType(param.typ, flags, conf)
c &= ','
c.hashType(t.returnType, flags, conf)
elif t.n != nil and t.n.kind == nkFormalParams:
# Under IC a loaded proc type stores its parameters only in `n`; `sons`
# holds just the return type. Hashing `t.signature` would silently drop
# every parameter, collapsing distinct proc types onto one hash, so the
# same logical type got different C struct names in different TUs
# ("incompatible type for argument" on closure args). Hash the return
# type first and then the parameter types from `n` — for from-source
# types `n`'s param types equal `sons[1..]`, so non-IC hashes are
# unchanged. (Same fix as typekeys' tyProc branch.)
c.hashType(t.returnType, flags, conf)
for i in 1..<t.n.len:
let p = t.n[i]
if p.kind == nkSym:
backendEnsureMutable(p.sym)
# The hidden closure env param: under IC, lambda lifting shares the
# routine's AST params with `typ.n`, so the lifted `:envP` leaks into
# the TYPE's params (from-source types never carry it). It is not part
# of the type's identity — `genProcParams` skips it the same way.
if t.callConv == ccClosure and p.sym.name.s == ":envP":
continue
c.hashType(p.sym.typ, flags, conf)
else:
c.hashType(p.typ, flags, conf)
else:
for a in t.signature: c.hashType(a, flags, conf)
c &= char(t.callConv)
@@ -350,21 +263,6 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi
c &= char(t.kind)
c.hashType(t.indexType, flags-{CoIgnoreRange}+{CoIgnoreRangeInArray}, conf)
c.hashType(t.elementType, flags-{CoIgnoreRange}, conf)
of tyBuiltInTypeClass:
# A builtin type class (`object`, `tuple`, `proc`, `ref`, `seq`, ...) is
# identified solely by the *kind* of its single placeholder son plus a few
# flags/callConv (see `sameType`). That son is a fresh, field-less, sym-less
# type, so the generic `else` below would recurse into it and hash its
# process-local `t.id` — unstable across the NIF boundary. nim-serialization
# keys auto-serialization on `signatureHash(object)`/`tuple`/... and missed
# under IC because the registering and consuming modules minted different
# placeholder ids. Hash the class identity that `sameType` actually compares.
c &= char(t.kind)
let elem = t.elementType
c &= char(elem.kind)
for f in eqTypeFlags * elem.flags: c &= char(ord(f))
if elem.kind == tyProc and tfExplicitCallConv in elem.flags:
c &= char(elem.callConv)
else:
c &= char(t.kind)
for a in t.kids: c.hashType(a, flags, conf)
@@ -548,3 +446,4 @@ proc idOrSig*(s: PSym, currentModule: string,
if counter != 0:
result.add "_" & rope(counter+1)
sigCollisions.inc(sig)

View File

@@ -135,11 +135,6 @@ proc put(c: var TCandidate, key, val: PType) {.inline.} =
writeStackTrace()
if c.c.module.name.s == "temp3":
echo "binding ", key, " -> ", val
when defined(icDbgRefc):
if key.kind in {tyGenericParam, tyTypeDesc}:
echo "[icBind] put ", key.kind, " ", typeToString(key), " uid=", key.uniqueId.module, ".",
key.uniqueId.item, " itemId=", key.itemId.module, ".", key.itemId.item,
" state=", key.state, " -> ", typeToString(val)
put(c.bindings, key, val.skipIntLit(c.c.idgen))
proc typeRel*(c: var TCandidate, f, aOrig: PType,
@@ -916,7 +911,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
case typ.kind
of tyStatic:
param = paramSym skConst
param.typ = typ.exactReplica(m.c.idgen)
param.typ = typ.exactReplica
#copyType(typ, c.idgen, typ.owner)
if typ.n == nil:
param.typ.incl tfInferrableStatic
@@ -924,7 +919,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
param.ast = typ.n
of tyFromExpr:
param = paramSym skVar
param.typ = typ.exactReplica(m.c.idgen)
param.typ = typ.exactReplica
#copyType(typ, c.idgen, typ.owner)
else:
param = paramSym skType
@@ -977,7 +972,7 @@ proc matchUserTypeClass*(m: var TCandidate; ff, a: PType): PType =
if ff.kind == tyUserTypeClassInst:
result = generateTypeInstance(c, m.bindings, typeClass.sym.info, ff)
else:
result = ff.exactReplica(m.c.idgen)
result = ff.exactReplica
#copyType(ff, c.idgen, ff.owner)
result.n = checkedBody
@@ -1173,10 +1168,6 @@ proc enterConceptMatch(c: var TCandidate; f,a: PType, flags: TTypeRelFlags): TTy
if concpt.kind != tyConcept:
container = concpt
concpt = container.reduceToBase
# considerPreviousT-like behavior
let prev = lookup(c.bindings, concpt)
if prev != nil:
return typeRel(c, prev, a, flags)
if trDontBind in flags:
conceptFlags.incl mfDontBind
if trCheckGeneric in flags:
@@ -1768,21 +1759,6 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
let ff = last(f)
if ff != nil:
result = typeRel(c, ff, a, flags)
if result == isNone and a.kind == tyGenericInst and trBindGenericParam in flags:
var depth = -1
# Generic-parameter constraints like `F: Future` can miss in `last(f)`
# when the actual type inherits from a concrete generic instantiation.
# Keep this fallback scoped to generic-parameter matching so typedesc
# overloads such as `type Future[T]` still prefer more specific
# descendants like `InternalRaisesFuture[T, E]`.
if isGenericSubtype(c, a, f, depth, f) and depth > 0:
var askip = skippedNone
let aobj = a.skipToObject(askip)
if aobj != nil and tfFinal notin aobj.flags:
# Keep overload ranking consistent with other inheritance-based
# matches: deeper descendants are slightly worse candidates.
inc c.inheritancePenalty, depth + int(c.inheritancePenalty < 0)
result = isGeneric
of tyGenericInvocation:
var x = a.skipGenericAlias
if x.kind == tyGenericParam and x.len > 0:
@@ -2675,7 +2651,7 @@ proc staticAwareTypeRel(m: var TCandidate, f: PType, arg: var PNode): TTypeRelat
# The ast of the type does not point to the symbol.
# Without this we will never resolve a `static proc` with overloads
let copiedNode = copyNode(arg)
copiedNode.typ = exactReplica(copiedNode.typ, m.c.idgen)
copiedNode.typ = exactReplica(copiedNode.typ)
copiedNode.typ.n = arg
arg = copiedNode
typeRel(m, f, arg.typ)
@@ -2879,7 +2855,6 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int
if m.calleeSym != nil and m.calleeSym.kind notin {skTemplate, skMacro}:
c.mergeShadowScope
else:
c.rememberShadowDefs
c.closeShadowScope
m.state = csNoMatch
m.firstMismatch.arg = a
@@ -2936,10 +2911,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int
setSon(m.call, formal.position + 1, container)
else:
incrIndexType(container.typ)
# bug #25693: like the scalar `tyUntyped` case in `paramTypesMatchAux`,
# a previous overload candidate may have sem-checked the operand in
# place; templates/macros expect the pristine AST, so use `nOrig`.
container.add nOrig[a]
container.add n[a]
elif n[a].kind == nkExprEqExpr:
# named param
m.firstMismatch.kind = kUnknownNamedParam
@@ -3038,8 +3010,7 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int
setSon(m.call, formal.position + 1, container)
else:
incrIndexType(container.typ)
# bug #25693: see the leading isVarargsUntyped branch above.
container.add nOrig[a]
container.add n[a]
else:
m.baseTypeMatch = false
m.typedescMatched = false
@@ -3091,7 +3062,6 @@ proc matchesAux(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var Int
if m.state == csMatch and not (m.calleeSym != nil and m.calleeSym.kind in {skTemplate, skMacro}):
c.mergeShadowScope
else:
c.rememberShadowDefs
c.closeShadowScope
inc a

View File

@@ -394,10 +394,9 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) =
accum.offset = 1
computeObjectOffsetsFoldFunction(conf, typ.n, false, accum)
let paddingAtEnd = int16(accum.finish())
if (typ.sym != nil and
typ.sym.flags * {sfCompilerProc, sfImportc} == {sfImportc} and
tfCompleteStruct notin typ.flags) or
tfIncompleteStruct in typ.flags:
if typ.sym != nil and
typ.sym.flags * {sfCompilerProc, sfImportc} == {sfImportc} and
tfCompleteStruct notin typ.flags:
typ.size = szUnknownSize
typ.align = szUnknownSize
typ.paddingAtEnd = szUnknownSize

View File

@@ -911,7 +911,7 @@ proc suggestDecl*(c: PContext, n: PNode; s: PSym) =
defer:
if attached: dec(c.inTypeContext)
# If user is typing out an enum field, then don't provide suggestions
if s.kind == skEnumField and c.config.ideActive and exactEquals(c.config.m.trackPos, n.info):
if s.kind == skEnumField and c.config.cmd == cmdIdeTools and exactEquals(c.config.m.trackPos, n.info):
suggestQuit()
suggestExpr(c, n)

View File

@@ -22,7 +22,7 @@ import std / tables
import
options, ast, astalgo, trees, msgs,
idents, renderer, types, semfold, magicsys, cgmeth, parampatterns,
idents, renderer, types, semfold, magicsys, cgmeth,
lowerings, liftlocals,
modulegraphs, lineinfos
@@ -90,21 +90,11 @@ proc getCurrOwner(c: PTransf): PSym =
if c.transCon != nil: result = c.transCon.owner
else: result = c.module
proc freshOwnedSym(c: PTransf; s, owner: PSym): PNode =
# We need to copy the symbol here because we might need to change its owner and
# we don't want to mess with the original symbol which might be used in other places.
# This can happen for example for iterators which are transformed multiple times when
# they are used in different contexts.
var fresh = copySym(s, c.idgen)
if fresh.kind notin routineKinds:
incl(fresh.flagsImpl, sfFromGeneric)
setOwner(fresh, owner)
result = newSymNode(fresh)
proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
let r = newSym(skTemp, getIdent(c.graph.cache, genPrefix), c.idgen, getCurrOwner(c), info)
r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias, tySink})
incl(r.flagsImpl, sfFromGeneric)
let owner = getCurrOwner(c)
result = newSymNode(r)
proc transform(c: PTransf, n: PNode, noConstFold = false): PNode
@@ -195,39 +185,11 @@ proc transformSym(c: PTransf, n: PNode): PNode =
result = transformSymAux(c, n)
proc freshVar(c: PTransf; v: PSym): PNode =
result = freshOwnedSym(c, v, getCurrOwner(c))
proc introduceNewRoutineHeaderSyms(c: PTransf; n: PNode; oldOwner, newOwner: PSym) =
# We need to introduce new symbols for the parameters and result of a routine when
# we copy it for inlining or closure generation.
# Otherwise, we would have multiple nodes referring to the same parameter symbols which
# can lead to problems when we need to change the owner of these symbols.
case n.kind
of nkSym:
if n.sym.owner == oldOwner:
c.transCon.mapping[n.sym.itemId] = freshOwnedSym(c, n.sym, newOwner)
of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit:
discard
else:
for i in 0..<n.len:
introduceNewRoutineHeaderSyms(c, n[i], oldOwner, newOwner)
proc copyRoutineTypeHeader(c: PTransf; oldProc, newProc: PSym) =
# We need to copy the routine type header to ensure that
# modifications to the newProc do not affect the oldProc.
if oldProc.typ != nil and oldProc.typ.kind == tyProc and oldProc.typ.n != nil:
newProc.typ = copyType(oldProc.typ, c.idgen, newProc)
newProc.typ.n = newNodeI(oldProc.typ.n.kind, oldProc.typ.n.info)
if oldProc.typ.n.len > 0:
newProc.typ.n.add copyTree(oldProc.typ.n[0])
for i in 1..<oldProc.typ.n.len:
let oldParam = oldProc.typ.n[i].sym
var newParam = getOrDefault(c.transCon.mapping, oldParam.itemId)
if newParam == nil:
newParam = freshOwnedSym(c, oldParam, newProc)
c.transCon.mapping[oldParam.itemId] = newParam
doAssert newParam.kind == nkSym
newProc.typ.addParam newParam.sym
let owner = getCurrOwner(c)
var newVar = copySym(v, c.idgen)
incl(newVar.flagsImpl, sfFromGeneric)
setOwner(newVar, owner)
result = newSymNode(newVar)
proc transformVarSection(c: PTransf, v: PNode): PNode =
result = newTransNode(v)
@@ -376,18 +338,11 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PNode =
return n
of nkLambdaKinds, nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef: # todo optimize nosideeffects?
result = newTransNode(n)
let oldProc = n[namePos].sym
let x = freshOwnedSym(c, oldProc, oldProc.owner)
c.transCon.mapping[oldProc.itemId] = x
introduceNewRoutineHeaderSyms(c, n[paramsPos], oldProc, x.sym)
if resultPos < n.len and n[resultPos] != nil:
introduceNewRoutineHeaderSyms(c, n[resultPos], oldProc, x.sym)
copyRoutineTypeHeader(c, oldProc, x.sym)
let x = newSymNode(copySym(n[namePos].sym, c.idgen))
c.transCon.mapping[n[namePos].sym.itemId] = x
result[namePos] = x # we have to copy proc definitions for iters
for i in 1..<n.len:
result[i] = introduceNewLocalVars(c, n[i])
if x.sym.typ != nil and x.sym.typ.kind == tyProc:
result[paramsPos] = x.sym.typ.n
result[namePos].sym.ast = result
else:
result = newTransNode(n)
@@ -720,7 +675,7 @@ type
paDirectMapping, paFastAsgn, paFastAsgnTakeTypeFromArg
paVarAsgn, paComplexOpenarray, paViaIndirection
proc putArgInto(arg: PNode, formal: PType; borrowedFirstArg = false): TPutArgInto =
proc putArgInto(arg: PNode, formal: PType): TPutArgInto =
# This analyses how to treat the mapping "formal <-> arg" in an
# inline context.
if formal.kind == tyTypeDesc: return paDirectMapping
@@ -771,13 +726,6 @@ proc putArgInto(arg: PNode, formal: PType; borrowedFirstArg = false): TPutArgInt
if skipTypes(formal, abstractInst).kind in {tyVar, tyLent}: result = paVarAsgn
else: result = paFastAsgn
if borrowedFirstArg and result == paDirectMapping and parampatterns.exprRoot(arg) == nil and
parampatterns.isAssignable(nil, arg) == arNone:
# Inline iterators like `items(array)` borrow from the first argument.
# If that argument is just a transient expression, materialize it so the
# lifted closure keeps the backing storage alive across yields.
result = paFastAsgnTakeTypeFromArg
proc findWrongOwners(c: PTransf, n: PNode) =
if n.kind == nkVarSection:
let x = n[0][0]
@@ -876,16 +824,13 @@ proc transformFor(c: PTransf, n: PNode): PNode =
if iter.kind != skIterator: return result
# generate access statements for the parameters (unless they are constant)
pushTransCon(c, newC)
let borrowedIterResult =
iter.typ != nil and iter.typ.returnType != nil and
skipTypes(iter.typ.returnType, abstractInst).kind in {tyLent, tyVar}
for i in 1..<call.len:
var arg = transform(c, call[i])
let ff = skipTypes(iter.typ, abstractInst)
# can happen for 'nim check':
if i >= ff.n.len: return result
var formal = ff.n[i].sym
let pa = putArgInto(arg, formal.typ, borrowedIterResult and i == 1)
let pa = putArgInto(arg, formal.typ)
case pa
of paDirectMapping:
newC.mapping[formal.itemId] = arg
@@ -1386,33 +1331,7 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf
result = getBody(g, prc)
else:
prc.transformedBody = newNode(nkEmpty) # protects from recursion
# Lambda-lifting a routine body while the VM compiles it (to run a macro
# under `nim ic`) mints a closure `:env` (type + obj + fields + hidden param)
# that the lift welds into the routine's serialized signature. Such an env is
# a PROCESS-LOCAL artifact (its item number is per-process-sequential), so a
# reference to it must never carry a stable cross-module identity — otherwise
# a consumer resolves it against a canonical NIF built by a different process
# that has no matching def ('symbol has no offset', e.g. Nimbus t17.275).
# Lift in the backend (process-local) id space; ast2nif then emits these as
# module-local `@bk` defs (mirrors setAttachedOp's inVMTransform handling).
var liftIdgen = idgen
if g.inVMTransform > 0 and g.config.cmd == cmdM:
if g.vmTransfIdgen == nil:
g.vmTransfIdgen = idGeneratorForBackend(g.systemModule)
liftIdgen = g.vmTransfIdgen
var c = openTransf(g, prc.getModule, "", liftIdgen, flags)
# `liftCapturedVars` rewrites captured locals to `:env.field` IN PLACE on the
# body it is handed; the env-creation prologue lands only in the returned
# wrapper. When the VM drives this transform (running a macro/CT proc), that
# in-place mutation corrupts the routine's PRE-transform `ast[bodyPos]` —
# under IC exactly the node `getBody` serializes to the module's `.s.nif`. So
# snapshot the pristine body before the VM lift and restore `ast[bodyPos]`
# afterwards: the VM still consumes the fully-lifted `result`, but `getBody`
# keeps faithfully returning the pre-transform body for serialization. The
# cg/backend path (`inVMTransform == 0`) is untouched.
let vmPristineBody =
if g.inVMTransform > 0: copyTree(getBody(g, prc))
else: nil
var c = openTransf(g, prc.getModule, "", idgen, flags)
result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen, flags)
result = processTransf(c, result, prc)
liftDefer(c, result)
@@ -1422,8 +1341,6 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; flags: Transf
result = g.transformClosureIterator(c.idgen, prc, result)
incl(result.flags, nfTransf)
if vmPristineBody != nil:
prc.ast[bodyPos] = vmPristineBody
if useCache in flags or prc.typ.callConv == ccInline:
# genProc for inline procs will be called multiple times from different modules,

View File

@@ -225,17 +225,6 @@ proc getRoot*(n: PNode): PSym =
else: result = nil
else: result = nil
proc isCursor*(n: PNode): bool =
case n.kind
of nkSym:
sfCursor in n.sym.flags
of nkDotExpr:
isCursor(n[1])
of nkCheckedFieldExpr:
isCursor(n[0])
else:
false
proc stupidStmtListExpr*(n: PNode): bool =
for i in 0..<n.len-1:
if n[i].kind notin {nkEmpty, nkCommentStmt}: return false

View File

@@ -156,7 +156,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
result = typeAllowedAux(marker, t.elementType, kind, c, flags+{taIsOpenArray})
of tySink:
# you cannot nest openArrays/sinks/etc.
if kind != skParam or taIsOpenArray in flags or t.elementType.kind in {tySink, tyLent, tyVar}:
# `sink openarray` is not allowed
if kind != skParam or taIsOpenArray in flags or t.elementType.kind in {tySink, tyLent, tyVar, tyOpenArray, tyVarargs}:
result = t
else:
result = typeAllowedAux(marker, t.elementType, kind, c, flags)

View File

@@ -10,10 +10,10 @@
## Based on sighashes.nim but works on astdef directly as we need it in ast2nif.nim.
## Also produces more readable names thanks to treemangler.
import std/[assertions, sets]
import std/assertions
import "../dist/nimony/src/lib" / [treemangler]
import icmodnames
import "../dist/nimony/src/gear2" / modnames
import astdef, idents, options, lineinfos, msgs
import ic / [enum2nif]
@@ -47,11 +47,6 @@ type
CoConsiderOwned
CoDistinct
CoHashTypeInsideNode
CoPrecise # produce a FRONTEND-faithful, unique key (for the
# stable NIF *name*, not hook dedup): keep distinctions
# sem makes that the backend identity collapses — e.g.
# an `int literal(x)` type carries its value so it does
# not merge with `int`. See `getTypeKey`/`typeKeyHook`.
TypeLoader* = proc (t: PType) {.nimcall.}
SymLoader* = proc (s: PSym) {.nimcall.}
@@ -59,9 +54,6 @@ type
m: Mangler
tl: TypeLoader
sl: SymLoader
visited: HashSet[ItemId] # anonymous object types whose fields are currently
# being hashed — a non-mutating guard against endless
# recursion when a field references the type itself.
proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef)
proc symKey(c: var Context; s: PSym; conf: ConfigRef) =
@@ -75,26 +67,14 @@ proc symKey(c: var Context; s: PSym; conf: ConfigRef) =
name.add '.'
name.addInt s.disamb
# The owner may still be an unloaded stub (kind `skStub`): force it in
# before inspecting its kind, otherwise the module suffix is silently
# dropped from the key and def-vs-use keys diverge — e.g. `Lexer`'s base
# class keyed as `TBaseLexer.0.` at nifc vs `TBaseLexer.0.nimqydn3y` at
# sem time, making `getAttachedOp` miss ("'=destroy' operator not found").
template forceLoaded(x: PSym): PSym =
let tmp = x
if tmp != nil and tmp.state == Partial and c.sl != nil: c.sl(tmp)
tmp
let owner = forceLoaded(s.ownerFieldImpl)
let it =
if s.kindImpl == skModule:
s
elif s.kindImpl in skProcKinds and sfFromGeneric in s.flagsImpl and
owner != nil and owner.kindImpl != skModule:
forceLoaded(owner.ownerFieldImpl)
elif s.kindImpl in skProcKinds and sfFromGeneric in s.flagsImpl and s.ownerFieldImpl.kindImpl != skModule:
s.ownerFieldImpl.ownerFieldImpl
else:
owner
if it != nil and it.kindImpl == skModule:
s.ownerFieldImpl
if it.kindImpl == skModule:
name.add '.'
name.add modname(it, conf)
c.m.addSymbol(name)
@@ -144,39 +124,6 @@ proc maybeImported(c: var Context; s: PSym; conf: ConfigRef) {.inline.} =
if s != nil and {sfImportc, sfExportc} * s.flagsImpl != {}:
c.symKey(s, conf)
proc emitPreciseFlags(c: var Context; t: PType; flags: set[ConsiderFlag]) {.inline.} =
## Under CoPrecise the NIF *name* must be as fine as `types.sameType`, which
## compares `eqTypeFlags * flags` (see `sameFlags`). Without this a
## `proc() {.gcsafe.}` keyed identically to `proc()`, and a `ref X not nil`
## identically to `ref X` — the loader would then merge two frontend-distinct
## types onto one NIF name. Emitted only for the naming path (CoPrecise); the
## `setAttachedOp` hook key (no CoPrecise) is unaffected, so its byte layout is
## unchanged.
if CoPrecise in flags:
let ef = eqTypeFlags * t.flagsImpl
if ef != {}:
withTree c.m, "´tflags":
for f in ef: c.m.addIntLit ord(f)
proc backendTypeName(t: PType; conf: ConfigRef): string =
## Stable cross-module identity of a backend-minted (lower-stage) type: its
## serialized `@bk` NIF name (mirrors ast2nif.nifTypeName). A closure-env
## object/ref minted by the `lower` stage has NO stable STRUCTURAL key — its
## captured-field types re-resolve to different modules in the producing vs the
## consuming process (e.g. field `x0` → `int` in the producer, → the consumer's
## alias in the consumer) — but this name (kind + item + home-module suffix) is
## identical in both, because the consumer loads the producer's name verbatim.
## Keying hooks by it makes producer `setAttachedOp` and consumer `getAttachedOp`
## agree. The trailing `@bk` (= ast2nif.BackendLocalMarker) keeps it disjoint
## from any normal type's structural key.
result = "`t"
result.addInt ord(t.kind)
result.add '.'
result.addInt t.uniqueId.item
result.add '.'
result.add modname(t.uniqueId.module, conf)
result.add "@bk"
proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef) =
if t == nil:
c.m.addEmpty()
@@ -186,25 +133,12 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
assert c.tl != nil
c.tl(t)
if t.uniqueId.isBackendMinted:
# Backend-minted (lower-stage) closure-env types key by their stable NIF name,
# never by structure (which diverges across the NIF boundary). An env `ref`
# that is itself NOT backend-minted still keys stably: it recurses here and
# reaches its `@bk` object, which short-circuits to a stable name.
c.m.addSymbol backendTypeName(t, conf)
return
case t.kind
of tyGenericInvocation:
for a in t.sonsImpl:
c.typeKey a, flags, conf
of tyDistinct:
if t.sonsImpl.len == 0:
# a bare `distinct` typeclass (e.g. `foo(distinct, ...)` matched
# against a `T: type` param) has no base type to key — it IS its kind
withTree c.m, toNifTag(t.kind):
c.m.addEmpty()
elif CoDistinct in flags:
if CoDistinct in flags:
if t.symImpl != nil: symKey(c, t.symImpl, conf)
if t.symImpl == nil or tfFromGeneric in t.flagsImpl:
c.typeKey t.sonsImpl[^1], flags, conf
@@ -213,14 +147,7 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
else:
symKey(c, t.symImpl, conf)
of tyGenericInst:
# The generic head (son[0]) may be a lazily-loaded stub under IC; ensure it
# is materialised before peeking at its symbol. A nil sym means this is not
# an imported C++ generic, so fall through to the normal `skipModifierB`.
var base = t.sonsImpl[0]
if base.state == Partial:
assert c.tl != nil
c.tl(base)
if base.symImpl != nil and sfInfixCall in base.symImpl.flagsImpl:
if sfInfixCall in t.sonsImpl[0].symImpl.flagsImpl:
# This is an imported C++ generic type.
# We cannot trust the `lastSon` to hold a properly populated and unique
# value for each instantiation, so we hash the generic parameters here:
@@ -248,10 +175,6 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
of tyInt:
withTree c.m, "i":
c.m.addIntLit -1
# An `int literal(x)` type (nImpl holds the value) must stay distinct from
# plain `int` for the NIF name, else overload resolution breaks on reload.
if CoPrecise in flags and t.nImpl != nil:
c.m.addIntLit t.nImpl.intVal
maybeImported(c, t.symImpl, conf)
of tyInt8:
withTree c.m, "i":
@@ -289,60 +212,18 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
withTree c.m, "u":
c.m.addIntLit 64
maybeImported(c, t.symImpl, conf)
of tyFloat:
withTree c.m, "f":
c.m.addIntLit -1
# A `float literal(x)` type (nImpl = nkFloatLit) must stay distinct from
# plain `float`, just like the `int literal(x)` case above.
if CoPrecise in flags and t.nImpl != nil and t.nImpl.kind in {nkFloatLit..nkFloat64Lit}:
c.m.addFloatLit t.nImpl.floatVal
maybeImported(c, t.symImpl, conf)
of tyObject, tyEnum:
if t.typeInstImpl != nil:
# prevent against infinite recursions here, see bug #8883:
let inst = t.typeInstImpl
if inst.state == Partial:
# a lazily-loaded typeInst stub has no sons until forced in
assert c.tl != nil
c.tl(inst)
t.typeInstImpl = nil # IC: spurious writes are ok since we set it back immediately
assert inst.kind == tyGenericInst
if inst.sonsImpl.len > 0:
c.typeKey inst.sonsImpl[0], flags, conf
c.typeKey inst.sonsImpl[0], flags, conf
for i in 1..<inst.sonsImpl.len-1:
# Match sighashes: generic-instantiation arguments are keyed with
# `CoDistinct` so distinct args are not collapsed to their base.
c.typeKey inst.sonsImpl[i], flags+{CoDistinct}, conf
c.typeKey inst.sonsImpl[i], flags, conf
t.typeInstImpl = inst
elif t.symImpl != nil:
c.symKey(t.symImpl, conf)
# Anonymous / gensym'd object types (e.g. closure environments and
# `ref object` ObjectTypes) share the placeholder name `´anon`, so `symKey`
# alone collapses every one of them onto the same key — which made distinct
# closure-env `=destroy`/`=sink` hooks collide. Mirror sighashes: when the
# type symbol is anonymous/gensym'd, disambiguate further by keying the
# field types and names (or `.empty` when there are none).
template hasFlag(sym: PSym): bool =
{sfAnon, sfGenSym} * sym.flagsImpl != {}
if hasFlag(t.symImpl) or
(t.kind == tyObject and t.ownerFieldImpl != nil and t.ownerFieldImpl.kindImpl == skType and
t.ownerFieldImpl.typImpl != nil and t.ownerFieldImpl.typImpl.kind == tyRef and hasFlag(t.ownerFieldImpl)):
if t.nImpl != nil and t.nImpl.len > 0:
# Guard against endless recursion when a field references this type
# itself. Unlike sighashes (which temporarily clears `sfAnon`/`sfGenSym`
# on the symbol), do NOT mutate: `typeKey` runs during sem — it is
# called unconditionally from `modulegraphs.setAttachedOp` — so a
# mutation that an assertion deeper in `treeKey` left unrestored would
# corrupt the type. `symKey` above already emitted the type's identity,
# so on a back-reference we simply stop.
if not containsOrIncl(c.visited, t.itemId):
c.treeKey(t.nImpl, flags + {CoHashTypeInsideNode}, conf)
c.visited.excl t.itemId
else:
c.m.addIdent "´empty"
# Object inheritance is part of identity: key the base class too.
if t.kind == tyObject and t.sonsImpl.len > 0 and t.sonsImpl[0] != nil:
c.typeKey t.sonsImpl[0], flags, conf
else:
c.m.addIdent "`bug"
of tyFromExpr:
@@ -357,19 +238,10 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.symKey(t.nImpl[i].sym, conf)
c.typeKey(t.nImpl[i].sym.typImpl, flags+{CoIgnoreRange}, conf)
else:
# ALL sons are tuple fields (son 0 included — unlike tyProc, where
# son 0 is the return type). Starting at 1 dropped the first field,
# collapsing e.g. `(PSym, NifIndexEntry)` and `(PType, NifIndexEntry)`
# onto one key, so hook lookup called the wrong `=destroy`/`=sink`
# (incompatible-argument C errors). Mirrors sighashes' `for a in t.kids`.
for i in 0..<t.sonsImpl.len:
for i in 1..<t.sonsImpl.len:
c.typeKey t.sonsImpl[i], flags+{CoIgnoreRange}, conf
of tyRange:
if t.sonsImpl.len == 0:
# bare `range` typeclass: no base type, key the kind alone
withTree c.m, toNifTag(t.kind):
c.m.addEmpty()
elif CoIgnoreRange notin flags:
if CoIgnoreRange notin flags:
withTree c.m, toNifTag(t.kind):
c.treeKey(t.nImpl, {}, conf)
c.typeKey(t.sonsImpl[^1], flags, conf)
@@ -382,28 +254,12 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.typeKey(t.skipModifierB, flags, conf)
of tyProc:
withTree c.m, (if tfIterator in t.flagsImpl: "itertype" else: "proctype"):
# Proc parameter *types* are part of the type's identity. Under IC the
# parameters live in `nImpl` (`sonsImpl` holds only the return type), so a
# loaded proc type has an empty `sonsImpl[1..]`; reading params from there
# would silently drop them and collide every same-return/same-callconv
# closure onto one key (e.g. `proc(cb: proc())` onto bare `proc()`),
# which made hook lookup resolve to the wrong `=copy`. Prefer `nImpl`
# (consistent in-memory and after load); hash param types only, not their
# symbols — parameter names do not affect type identity.
if t.nImpl != nil and t.nImpl.kind == nkFormalParams:
if CoProc in flags and t.nImpl != nil:
let params = t.nImpl
for i in 1..<params.len:
if params[i].kind == nkSym:
# The param sym may be a lazily-loaded stub: force it in (as `symKey`
# does) so its type is available, then hash the param *type* only —
# parameter names are not part of the type's identity. Without the
# load the type reads back nil at codegen and the key silently loses
# its parameters (collapsing distinct closure types onto one key).
let ps = params[i].sym
if ps.state == Partial and c.sl != nil: c.sl(ps)
c.typeKey(ps.typImpl, flags, conf)
else:
c.typeKey(params[i].typField, flags, conf)
let param = params[i].sym
c.symKey(param, conf)
c.typeKey(param.typImpl, flags, conf)
else:
for i in 1..<t.sonsImpl.len:
c.typeKey(t.sonsImpl[i], flags, conf)
@@ -412,38 +268,18 @@ proc typeKey(c: var Context; t: PType; flags: set[ConsiderFlag]; conf: ConfigRef
c.m.addIdent toNifTag(t.callConvImpl)
if tfVarargs in t.flagsImpl: c.m.addIdent "´varargs"
# `.gcsafe`/`.noSideEffect` (in eqTypeFlags) distinguish proc types under
# sameType, so they must distinguish the NIF name too.
emitPreciseFlags(c, t, flags)
of tyArray:
withTree c.m, toNifTag(t.kind):
if t.sonsImpl.len == 0:
# bare `array` typeclass: no element/index types
c.m.addEmpty()
else:
c.typeKey(t.sonsImpl[^1], flags-{CoIgnoreRange}, conf)
c.typeKey(t.sonsImpl[0], flags-{CoIgnoreRange}, conf)
c.typeKey(t.sonsImpl[^1], flags-{CoIgnoreRange}, conf)
c.typeKey(t.sonsImpl[0], flags-{CoIgnoreRange}, conf)
else:
withTree c.m, toNifTag(t.kind):
for i in 0..<t.sonsImpl.len:
c.typeKey t.sonsImpl[i], flags, conf
if tfNotNil in t.flagsImpl and CoType notin flags:
c.m.addIdent "´notnil"
# tfNotNil/tfVarIsPtr/tfIsOutParam (eqTypeFlags) part of sameType identity
# for ref/ptr/var/lent/sink; the hook path (no CoPrecise) keeps its layout.
emitPreciseFlags(c, t, flags)
proc typeKey*(t: PType; conf: ConfigRef; tl: TypeLoader; sl: SymLoader): string =
var c: Context = Context(m: createMangler(30, -1), tl: tl, sl: sl,
visited: initHashSet[ItemId]())
# Mirror the flags liftdestructors uses for its `canonTypes` hash
# (`hashType(skipped, {CoType, CoConsiderOwned, CoDistinct})`): hook keys must
# distinguish what hook *lifting* distinguishes. With empty flags a generic
# `distinct` instance (e.g. nilcheck's `SeqOfDistinct[T, U]`) took the bare
# `symKey` branch — the sym is the generic's and thus SHARED by all
# instances, so `SeqOfDistinct[I, PNode]` and `SeqOfDistinct[I, Nilability]`
# collided onto one key and hook lookup returned the wrong `=sink`
# ("incompatible type for argument" in the generated C). Under `CoDistinct` a
# `tfFromGeneric` distinct keys as sym + base type, keeping instances apart.
typeKey(c, t, {CoType, CoConsiderOwned, CoDistinct}, conf)
var c: Context = Context(m: createMangler(30, -1), tl: tl, sl: sl)
typeKey(c, t, {}, conf)
result = c.m.extract()

View File

@@ -633,34 +633,6 @@ proc lengthOrd*(conf: ConfigRef; t: PType): Int128 =
let first = firstOrd(conf, t)
result = last - first + One
const broadcastArrayThreshold* = 32
## `getNullValue` represents the default of an `array[N, T]` with `N` above this
## as a single *broadcast* element — a one-son `nkBracket` standing for `N`
## identical zero copies — instead of materialising `N` zero nodes. This keeps
## huge zeroed arrays (e.g. SSZ byte buffers in nimbus) compact in the IC caches
## (`.s.bif`/`.t.bif`), in the VM, and in the generated C (`{0}` zero-fills).
proc isDefaultBroadcastArray*(n: PNode; conf: ConfigRef): bool =
## True iff `n` is a broadcast default array: a single son standing for
## `lengthOrd` identical zero copies. Identified by the explicit `nfBroadcast`
## marker (set by `getNullValue`), NOT by `len == 1 < lengthOrd` — the latter
## also matches an ordinary 1-element collection that happens to be an
## `nkBracket` carrying an array type, e.g. a `@[a, b, c]` seq value shrunk to
## length 1 by `setLen`/`delete` (its VM node keeps the array-literal type).
result = n != nil and n.kind == nkBracket and nfBroadcast in n.flags
proc expandBroadcastArray*(n: PNode; conf: ConfigRef) =
## Materialise a broadcast default array (see `isDefaultBroadcastArray`) into a
## full `lengthOrd`-son `nkBracket`, each son a copy of the single default
## element. Used by VM ops that index-address, mutate, or measure such a node;
## the common read-only paths leave it compact. Clears `nfBroadcast` since the
## node is now a fully materialised literal.
if isDefaultBroadcastArray(n, conf):
let total = toInt(lengthOrd(conf, n.typ.skipTypes(abstractInst)))
let elem = n[0]
for i in 1 ..< total: n.add copyTree(elem)
n.flags.excl nfBroadcast
# -------------- type equality -----------------------------------------------
type

View File

@@ -185,9 +185,6 @@ proc root(v: var Partitions; start: int): int =
proc potentialMutation(v: var Partitions; s: PSym; level: int; info: TLineInfo) =
let id = variableId(v, s)
if id >= 0:
# mutated here => alive here: keep aliveEnd in sync so dangerousMutation catches
# mutations recorded after the var's last use (e.g. via a call arg). See #25595.
v.s[id].aliveEnd = max(v.s[id].aliveEnd, v.abstractTime)
let r = root(v, id)
let flags = if s.kind == skParam:
if isConstParam(s):
@@ -677,13 +674,9 @@ proc deps(c: var Partitions; dest, src: PNode) =
else:
let srcid = variableId(c, s)
if srcid >= 0:
if s.kind notin {skResult, skParam} and
c.s[srcid].aliveEnd < c.s[vid].aliveEnd and
c.g.config.backend != backendJs:
# you cannot borrow from a local that lives shorter than 'vid'.
# On a traced (JS/GC) target the source object stays alive as long
# as the alias references it, so this lifetime rule does not apply;
# value-semantics safety is enforced by `dangerousMutation` instead.
if s.kind notin {skResult, skParam} and (
c.s[srcid].aliveEnd < c.s[vid].aliveEnd):
# you cannot borrow from a local that lives shorter than 'vid':
when explainCursors: echo "B not a cursor ", d.sym, " ", c.s[srcid].aliveEnd, " ", c.s[vid].aliveEnd
c.s[vid].flags.incl preventCursor
elif {isReassigned, preventCursor} * c.s[srcid].flags != {}:
@@ -1007,22 +1000,13 @@ proc checkBorrowedLocations*(par: var Partitions; body: PNode; config: ConfigRef
#if par.s[rid].con.kind == isRootOf and dangerousMutation(par.graphs[par.s[rid].con.graphIndex], par.s[i]):
# cannotBorrow(config, s, par.graphs[par.s[rid].con.graphIndex])
proc jsDeepCopied(t: PType): bool =
## On the JS backend `nimCopy` deep-copies these type classes on every
## assignment, so eliding the copy for a safe alias is worthwhile even when
## the type has no C-style destructor.
t.skipTypes({tyGenericInst, tyAlias, tyDistinct, tyVar, tyLent}).kind in
{tyObject, tyTuple, tyArray, tySequence, tyString}
proc computeCursors*(s: PSym; n: PNode; g: ModuleGraph) =
let jsCursors = g.config.backend == backendJs
var par = computeGraphPartitions(s, n, g, {cursorInference})
for i in 0 ..< par.s.len:
let v = addr(par.s[i])
if v.flags * {ownsData, preventCursor, isConditionallyReassigned} == {} and
v.sym.kind notin {skParam, skResult} and
v.sym.flags * {sfThread, sfGlobal} == {} and
(hasDestructor(v.sym.typ) or (jsCursors and jsDeepCopied(v.sym.typ))) and
v.sym.flags * {sfThread, sfGlobal} == {} and hasDestructor(v.sym.typ) and
v.sym.typ.skipTypes({tyGenericInst, tyAlias}).kind != tyOwned and
(getAttachedOp(g, v.sym.typ, attachedAsgn) == nil or
sfError notin getAttachedOp(g, v.sym.typ, attachedAsgn).flags):

View File

@@ -28,7 +28,7 @@ from magicsys import getSysType
const
traceCode = defined(nimVMDebug)
when defined(nimHasLibFFI): # == hasFFI; spelled out for the IC dep scanner
when hasFFI:
import evalffi
@@ -702,10 +702,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
# A bodge, but this takes in `toOpenArray(rb, rc, rc)` and emits
# nkTupleConstr(x, y, z) into the `regs[ra]`. These can later be used for calculating the slice we have taken.
decodeBC(rkNode)
# Slicing/openArray needs the real length and per-element nodes, so a
# compact default array must be materialised first.
if isDefaultBroadcastArray(regs[ra].node, c.config):
expandBroadcastArray(regs[ra].node, c.config)
let
collection = regs[ra].node
leftInd = regs[rb].intVal
@@ -774,15 +770,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
regs[ra].node.intVal = src.strVal[idx].ord
else:
stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.strVal.len-1))
elif isDefaultBroadcastArray(src, c.config):
# `a[i]` on a compact default array yields `default(T)` directly, without
# ever materialising the (potentially huge) array — the point of the
# broadcast form. See `getNullValue`/`isDefaultBroadcastArray`.
let total = toInt(lengthOrd(c.config, src.typ.skipTypes(abstractInst)))
if idx <% total:
regs[ra].node = copyTree(src[0])
else:
stackTrace(c, tos, pc, formatErrorIndexBound(idx, total-1))
elif src.kind notin {nkEmpty..nkFloat128Lit} and idx <% src.len:
regs[ra].node = src[idx]
else:
@@ -794,9 +781,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
stackTrace(c, tos, pc, formatErrorIndexBound(regs[rc].intVal, high(int)))
let idx = regs[rc].intVal.int
let src = if regs[rb].kind == rkNode: regs[rb].node else: regs[rb].nodeAddr[]
# Taking the address of an element needs distinct, stable per-slot nodes, so
# a compact default array must be materialised first.
if isDefaultBroadcastArray(src, c.config): expandBroadcastArray(src, c.config)
case src.kind
of nkTupleConstr:
let
@@ -845,8 +829,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
let idx = regs[rb].intVal.int
assert regs[ra].kind == rkNode
let arr = regs[ra].node
# Writing a slot materialises a compact default array into a full literal.
if isDefaultBroadcastArray(arr, c.config): expandBroadcastArray(arr, c.config)
case arr.kind
of nkTupleConstr: # refer to `opcSlice`
let
@@ -1049,8 +1031,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
case node.kind
of nkTupleConstr: # refer to `of opcSlice`
regs[ra].intVal = node[2].intVal - node[1].intVal + 1 - high
elif isDefaultBroadcastArray(node, c.config):
regs[ra].intVal = toInt(lengthOrd(c.config, node.typ.skipTypes(abstractInst))) - high
else:
# safeArrLen also return string node len
# used when string is passed as openArray in VM
@@ -1330,44 +1310,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
var a = regs[rb].node
if a.kind == nkVarTy: a = a[0]
if a.kind == nkSym:
# a macro observed this symbol's implementation: NeedsImpl edge to
# its home module under IC.
recordIcImplDep(c.graph, a.sym)
if a.sym.ast.isNil:
regs[ra].node = newNode(nkNilLit)
else:
let tree = copyTree(a.sym.ast)
# A NIF-loaded routine's `ast[paramsPos]` is an `nkEmpty` placeholder:
# ast2nif strips the formal params (recoverable from `typ.n`, see
# writeNode's `skipParams`). A macro that reads `fn.getImpl[paramsPos]`
# — e.g. taskpools `spawn` reads the return type via `getImpl[3][0]` —
# needs them, so reconstruct a read-only formalParams from the proc
# type. The synthesized type-expression nodes carry the resolved
# `PType`, which is all a macro can query for a loaded routine.
if tree.kind in {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef,
nkConverterDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo} and
tree.safeLen > paramsPos and tree[paramsPos].kind == nkEmpty and
a.sym.typ != nil and a.sym.typ.n != nil and
a.sym.typ.n.kind == nkFormalParams:
let t = a.sym.typ
let fp = newNodeI(nkFormalParams, a.sym.info)
let rt = t.returnType
# `opMapTypeInstToAst` (inst=true) reproduces a source-like type
# declaration — crucially it renders an array's range bound as
# `range 0..N` (the `inst=false` form emits `range[0, N]`, which
# re-sems to "'range' expects one type parameter").
fp.add(if rt != nil: opMapTypeInstToAst(c.cache, rt, a.sym.info, c.idgen)
else: newNodeI(nkEmpty, a.sym.info))
for i in 1 ..< t.n.len:
if t.n[i].kind == nkSym:
let p = t.n[i].sym
let def = newNodeI(nkIdentDefs, p.info)
def.add newIdentNode(p.name, p.info)
def.add opMapTypeInstToAst(c.cache, p.typ, p.info, c.idgen)
def.add newNodeI(nkEmpty, p.info)
fp.add def
tree[paramsPos] = fp
regs[ra].node = tree
regs[ra].node = if a.sym.ast.isNil: newNode(nkNilLit)
else: copyTree(a.sym.ast)
regs[ra].node.flags.incl nfIsRef
else:
stackTrace(c, tos, pc, "node is not a symbol")
@@ -1375,7 +1319,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
decodeB(rkNode)
let a = regs[rb].node
if a.kind == nkSym:
recordIcImplDep(c.graph, a.sym)
regs[ra].node =
if a.sym.ast.isNil:
newNode(nkNilLit)
@@ -2008,21 +1951,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
if regs[rb].node.kind != nkSym:
stackTrace(c, tos, pc, "node is not a symbol")
else:
let shSym = regs[rb].node.sym
# When `signatureHash` is applied to a type (e.g. a `T: typedesc`/generic
# param), hash the *type* it denotes, not the parameter symbol. Hashing the
# symbol routes through `hashNonProc`, which mixes in `s.disamb` — a
# per-module instantiation counter. Under incremental compilation the
# registering module and a consuming module instantiate the surrounding
# generic separately, get different `disamb`s, and produce different
# hashes for the same type (nim-serialization's auto-serialization lookup
# missed because of this). Hashing the underlying type via `hashType` is
# type-identity based and stable across the NIF boundary.
let shTyp = shSym.typ
if shTyp != nil and shTyp.kind == tyTypeDesc and shTyp.hasElementType:
regs[ra].node.strVal = $hashType(shTyp.elementType, c.config)
else:
regs[ra].node.strVal = $sigHash(shSym, c.config)
regs[ra].node.strVal = $sigHash(regs[rb].node.sym, c.config)
of opcSlurp:
decodeB(rkNode)
createStr regs[ra]

View File

@@ -308,7 +308,7 @@ proc newCtx*(module: PSym; cache: IdentCache; g: ModuleGraph; idgen: IdGenerator
callDepth: g.config.maxCallDepthVM,
comesFromHeuristic: unknownLineInfo, callbacks: @[], callbackIndex: initTable[string, int](), errorFlag: "",
cache: cache, config: g.config, graph: g, idgen: idgen,
contstantTab: initNodeTable(true), templInstCounter: new int)
contstantTab: initNodeTable(true))
proc refresh*(c: PCtx, module: PSym; idgen: IdGenerator) =
c.module = module

View File

@@ -36,7 +36,7 @@ import
magicsys, options, lowerings, lineinfos, transf, astmsgs,
treetab
from modulegraphs import getBody, recordIcImplDep
from modulegraphs import getBody
when defined(nimCompilerStacktraceHints):
import std/stackframes
@@ -46,7 +46,7 @@ const
when debugEchoCode:
import std/private/asciitables
when defined(nimHasLibFFI): # == hasFFI; spelled out for the IC dep scanner
when hasFFI:
import evalffi
type
@@ -786,12 +786,8 @@ proc genBinaryABCD(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) =
c.freeTemp(tmp2)
c.freeTemp(tmp3)
template sizeOfLikeMsg(name, incompleteStruct): string =
block:
if incompleteStruct:
"'$1' cannot be used with '.incompleteStruct' types" % [name]
else:
"'$1' requires '.importc' types to be '.completeStruct'" % [name]
template sizeOfLikeMsg(name): string =
"'$1' requires '.importc' types to be '.completeStruct'" % [name]
proc genNarrow(c: PCtx; n: PNode; dest: TDest) =
let t = skipTypes(n.typ, abstractVar-{tyTypeDesc})
@@ -851,26 +847,14 @@ proc genBinaryStmt(c: PCtx; n: PNode; opc: TOpcode) =
c.freeTemp(tmp)
c.freeTemp(dest)
proc genMutatingValue(c: PCtx; n: PNode): TRegister =
## Loads the value of an in-place mutation target while keeping it attached to
## its original storage. Compound lvalues must be resolved through their
## address: a normal value load can return a detached copy (for example, when
## indexing a broadcast default array).
if needsAsgnPatch(n):
let address = c.genx(n, {gfNodeAddr})
result = c.getTemp(n.typ)
c.gABC(n, opcLdDeref, result, address)
c.freeTemp(address)
else:
result = c.genx(n)
proc genBinaryStmtVar(c: PCtx; n: PNode; opc: TOpcode) =
var x = n[1]
if x.kind in {nkAddr, nkHiddenAddr}: x = x[0]
let
dest = c.genMutatingValue(x)
dest = c.genx(x)
tmp = c.genx(n[2])
c.gABC(n, opc, dest, tmp, 0)
#c.genAsgnPatch(n[1], dest)
c.freeTemp(tmp)
c.freeTemp(dest)
@@ -1174,7 +1158,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}, m: TMag
of mIncl, mExcl:
unused(c, n, dest)
var d = c.genMutatingValue(n[1])
var d = c.genx(n[1])
var tmp = c.genx(n[2])
c.genSetType(n[1], d)
c.gABC(n, if m == mIncl: opcIncl else: opcExcl, d, tmp)
@@ -1492,14 +1476,11 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}, m: TMag
else:
globalError(c.config, n.info, "expandToAst requires a call expression")
of mSizeOf:
let arg = n[1].typ.skipTypes({tyTypeDesc})
globalError(c.config, n.info, sizeOfLikeMsg("sizeof", tfIncompleteStruct in arg.flags))
globalError(c.config, n.info, sizeOfLikeMsg("sizeof"))
of mAlignOf:
let arg = n[1].typ.skipTypes({tyTypeDesc})
globalError(c.config, n.info, sizeOfLikeMsg("alignof", tfIncompleteStruct in arg.flags))
globalError(c.config, n.info, sizeOfLikeMsg("alignof"))
of mOffsetOf:
let arg = n[1].typ.skipTypes({tyTypeDesc})
globalError(c.config, n.info, sizeOfLikeMsg("offsetof", tfIncompleteStruct in arg.flags))
globalError(c.config, n.info, sizeOfLikeMsg("offsetof"))
of mRunnableExamples:
discard "just ignore any call to runnableExamples"
of mDestroy, mTrace: discard "ignore calls to the default destructor"
@@ -1794,15 +1775,8 @@ proc genGlobalInit(c: PCtx; n: PNode; s: PSym) =
# This is rather hard to support, due to the laziness of the VM code
# generator. See tests/compile/tmacro2 for why this is necessary:
# var decls{.compileTime.}: seq[NimNode] = @[]
# Load the slot's ADDRESS (not its value): the lazy initializer must REPLACE
# the null slot, which `opcWrDeref` only does for an `rkNodeAddr` target
# (`nAddr[] = n` for refs). With `opcLdGlobal` the slot value is loaded and for
# a ref-typed global that value is an `nkNilLit` ("nil ref"); writing through it
# hits the VM's nil-deref guard ("attempt to access a nil address"). This path
# is reached for compile-time globals whose defining module is restored from a
# NIF under `nim ic` (so `setupCompileTimeVar` never ran to eagerly init them).
let dest = c.getTemp(s.typ)
c.gABx(n, opcLdGlobalAddr, dest, s.position)
c.gABx(n, opcLdGlobal, dest, s.position)
if s.astdef != nil:
let tmp = c.genx(s.astdef)
c.genAdditionalCopy(n, opcWrDeref, dest, 0, tmp)
@@ -1868,8 +1842,6 @@ proc genArrAccessOpcode(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
if dest < 0: dest = c.getTemp(n.typ)
if opc in {opcLdArrAddr, opcLdStrIdxAddr} and gfNodeAddr in flags:
c.gABC(n, opc, dest, a, b)
if c.prc.regInfo[a].kind >= slotTempUnknown:
c.prc.regInfo[a].kind = slotTempPerm
elif needsRegLoad():
var cc = c.getTemp(n.typ)
c.gABC(n, opc, cc, a, b)
@@ -1886,8 +1858,6 @@ proc genObjAccessAux(c: PCtx; n: PNode; a, b: int, dest: var TDest; flags: TGenF
if dest < 0: dest = c.getTemp(n.typ)
if {gfNodeAddr} * flags != {}:
c.gABC(n, opcLdObjAddr, dest, a, b)
if a < c.prc.regInfo.len and c.prc.regInfo[a].kind >= slotTempUnknown:
c.prc.regInfo[a].kind = slotTempPerm
elif needsRegLoad():
var cc = c.getTemp(n.typ)
c.gABC(n, opcLdObj, cc, a, b)
@@ -1933,11 +1903,7 @@ proc genCheckedObjAccessAux(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags
let strType = getSysType(c.graph, n.info, tyString)
var msgReg: TDest = c.getTemp(strType)
let fieldName = $accessExpr[1]
# Re-navigate the discriminant in the object type: under `nim ic` `disc.sym` is a
# field-use stub with a nil `owner`, which `genFieldDefect` dereferences. Look up the
# canonical discriminant field by name. Byte-neutral for non-IC (returns the same sym).
let dfield = lookupFieldAgain(accessExpr[0].typ, disc.sym)
let msg = genFieldDefect(c.config, fieldName, dfield)
let msg = genFieldDefect(c.config, fieldName, disc.sym)
let strLit = newStrNode(msg, accessExpr[1].info)
strLit.typ = strType
c.genLit(strLit, msgReg)
@@ -2041,20 +2007,8 @@ proc getNullValue(c: PCtx; typ: PType, info: TLineInfo; conf: ConfigRef): PNode
getNullValueAux(c, t, t.n, result, conf, currPosition)
of tyArray:
result = newNodeIT(nkBracket, info, t)
let n = toInt(lengthOrd(conf, t))
if n > 0:
for i in 0..<toInt(lengthOrd(conf, t)):
result.add getNullValue(c, elemType(t), info, conf)
# For a large array, keep a single broadcast element (the default of every
# slot is identical) instead of `n` copies; `isDefaultBroadcastArray`
# consumers expand on demand. Small arrays stay fully materialised so the
# well-trodden paths are untouched. See `broadcastArrayThreshold`.
if n <= broadcastArrayThreshold:
for i in 1..<n:
result.add getNullValue(c, elemType(t), info, conf)
else:
# Broadcast form: mark the single-son node so `isDefaultBroadcastArray`
# recognises it unambiguously (see `nfBroadcast`).
result.flags.incl nfBroadcast
of tyTuple:
result = newNodeIT(nkTupleConstr, info, t)
for a in t.kids:
@@ -2502,10 +2456,6 @@ proc optimizeJumps(c: PCtx; start: int) =
proc genProc(c: PCtx; s: PSym): VmProcInfo =
result = c.procToCodePos.getOrDefault(s.id, NoVmProcInfo)
if result.usedRegisters < 0:
# compile-time execution consumes this routine's BODY: under IC that is a
# NeedsImpl dependency on the routine's home module (iface-cookie gating
# alone would miss body-only edits, e.g. `const x = dep.foo()`).
recordIcImplDep(c.graph, s)
#if s.name.s == "outterMacro" or s.name.s == "innerProc":
# echo "GENERATING CODE FOR ", s.name.s
let last = c.code.len-1
@@ -2519,9 +2469,7 @@ proc genProc(c: PCtx; s: PSym): VmProcInfo =
c.procToCodePos[s.id] = result
# thanks to the jmp we can add top level statements easily and also nest
# procs easily:
inc c.graph.inVMTransform
let body = transformBody(c.graph, c.idgen, s, if isCompileTimeProc(s): {} else: {useCache})
dec c.graph.inVMTransform
let procStart = c.xjmp(body, opcJmp, 0)
var p = PProc(blocks: @[], sym: s)
let oldPrc = c.prc

View File

@@ -36,9 +36,7 @@ from std/osproc import nil
when defined(nimPreviewSlimSystem):
import std/syncio
when not defined(nimPreviewSlimSystem):
# explicit negated `when` rather than `else:` so nifler's dep scanner guards
# this import with its condition (it emits `else:` imports unconditionally).
else:
from std/formatfloat import addFloatRoundtrip, addFloatSprintf

View File

@@ -152,8 +152,6 @@ proc sortVTableDispatchers*(g: ModuleGraph) =
rootItemIdCount.inc(baseType.itemId)
for idx in 0..<g.methods[bucket].methods.len:
let obj = g.methods[bucket].methods[idx].typ.firstParamType.skipTypes(skipPtrs)
if obj.itemId notin itemTable:
itemTable[obj.itemId] = newSeq[PSym](methodIndexLen)
itemTable[obj.itemId][mIndex] = g.methods[bucket].methods[idx]
for baseType in rootTypeSeq:

View File

@@ -134,11 +134,11 @@ nimblepath="$home/.nimble/pkgs/"
# BSD got posix_spawn only recently, so we deactivate it for osproc:
define:useFork
@elif haiku:
gcc.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
gcc.cpp.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
clang.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
clang.cpp.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
tcc.options.linker = "-Wl,--as-needed -lnetwork -lbsd"
gcc.options.linker = "-Wl,--as-needed -lnetwork"
gcc.cpp.options.linker = "-Wl,--as-needed -lnetwork"
clang.options.linker = "-Wl,--as-needed -lnetwork"
clang.cpp.options.linker = "-Wl,--as-needed -lnetwork"
tcc.options.linker = "-Wl,--as-needed -lnetwork"
@elif not genode:
# -fopenmp
gcc.options.linker = "-ldl"
@@ -168,19 +168,6 @@ nimblepath="$home/.nimble/pkgs/"
switch_gcc.cpp.options.always = "-g -Wall -O2 -ffunction-sections -march=armv8-a -mtune=cortex-a57 -mtp=soft -fPIE -D__SWITCH__ -fno-rtti -fno-exceptions -std=gnu++11"
@end
# Emscripten toolchain for WebAssembly (wasm32, or wasm64/Memory64).
@if emscripten:
cc = clang
clang.exe = "emcc"
clang.linkerexe = "emcc"
clang.cpp.exe = "emcc"
clang.cpp.linkerexe = "emcc"
@if wasm64:
passC = "-sMEMORY64=1"
passL = "-sMEMORY64=1"
@end
@end
# Configuration for the Intel C/C++ compiler:
@if windows:
icl.options.speed = "/Ox /arch:SSE2"

View File

@@ -21,7 +21,6 @@ Advanced commands:
see also: --dump.format:json (useful with: `| jq`)
//check checks the project for syntax and semantics
(can be combined with --defusages)
//track goto-definition / find-usages via `nim ic`
Runtime checks (see -x):
--objChecks:on|off turn obj conversion checks on|off
@@ -34,8 +33,6 @@ Runtime checks (see -x):
--infChecks:on|off turn Inf checks on|off
Advanced options:
--def:FILE,LINE,COL find the definition of the symbol at the position
--usages:FILE,LINE,COL find all usages of the symbol at the position
--defusages:FILE,LINE,COL
find the definition and all usages of a symbol
-o:FILE, --out:FILE set the output filename
@@ -122,7 +119,6 @@ Advanced options:
--lineDir:on|off generation of #line directive on|off
--embedsrc:on|off embeds the original source code as comments
in the generated output
--genBif:on|off generate per-module semantic BIF metadata in nimcache
--tlsEmulation:on|off turn thread local storage emulation on|off
--implicitStatic:on|off turn implicit compile time evaluation on|off
--trmacros:on|off turn term rewriting macros on|off

View File

@@ -188,7 +188,7 @@ objectPart = IND{>} objectPart^+IND{=} DED
/ objectWhen / objectCase / 'nil' / 'discard' / declColonEquals
objectDecl = 'object' ('of' typeDesc)? COMMENT? objectPart
conceptParam = ('var' | 'out' | 'ptr' | 'ref' | 'static' | 'type')? symbol
conceptDecl = 'concept' (conceptParam ^* ',' (pragma)?)? ('of' typeDesc ^* ',')?
conceptDecl = 'concept' conceptParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
&IND{>} stmt
typeDef = identVisDot genericParamList? pragma '=' optInd typeDefValue
indAndComment?

487
doc/ic.md
View File

@@ -2,404 +2,165 @@
Incremental Compilation (IC)
======================================
The ``nim ic`` command provides incremental compilation for Nim projects. It
decomposes compilation into per-module steps whose results are cached as NIF
files, and uses the external ``nifmake`` build tool to re-run only the steps
whose inputs changed.
This document describes **how `nim ic` works today**, including the edge cases
that shaped the current design. The per-module backend rewrite that earlier
editions of this document listed as a *Plan* has **landed**: the whole-program,
reuse/redirect/def-retention backend is gone and codegen is now a set of
`nifmake`-driven per-module rules (see *The backend*).
The ``nim ic`` command provides incremental compilation support for Nim projects,
allowing faster rebuilds by reusing previously compiled intermediate representations
of modules that haven't changed.
Overview
========
The pipeline has two halves driven by one process (`nim ic`, `commandIc` in
``compiler/deps.nim``) that constructs a dependency graph, writes a build file,
and hands it to ``nifmake``:
Incremental compilation works by decomposing the compilation process into several stages:
1. **Frontend** — per module:
- ``nifler parse --deps`` turns ``.nim`` source into a parsed NIF
(``.p.nif``) plus a static dependency list (``.deps.nif``).
- ``nim m`` (the *semantic* step, `cmdM`) reads the parsed NIF + the
precompiled NIFs of the module's imports, type-checks, and writes the
**semmed NIF** (``.nif``) plus invalidation sidecars (see *Cookies*).
2. **Backend** — ``nim nifc`` (`cmdNifC`, ``compiler/nifbackend.nim``) reads the
semmed NIFs, generates C, compiles and links.
1. **Parsing** - Source files are parsed into an abstract syntax tree (AST)
2. **Semantic Analysis** - Symbols are resolved and type checking is performed
3. **Code Generation** - Platform-specific code is generated from the analyzed AST
4. **Linking** - The generated code is linked into an executable
``nifmake`` orders the steps by their input/output files: every `nim m` runs
before the `nim nifc` step that consumes its NIF, and a step re-fires only when
one of its inputs is newer than its outputs. The driver invokes ``nifmake run
--parallel`` by default, so independent steps at the same DAG depth fan out
across cores; pass ``-d:icNoParallel`` to serialize (readable child output when
debugging a build).
The IC mechanism caches the results of earlier stages in NIF files
(Nim intermediate format): ``.p.nif`` (parsed), ``.deps.nif`` (dependencies),
and ``.nif`` (semantically analyzed). When recompiling, only modules that have
changed need to be reprocessed through the semantic analysis and code generation
stages, significantly reducing compilation time for large projects.
Artifacts (the NIF zoo)
=======================
NIF File Format
===============
Semantic BIF from regular builds
--------------------------------
NIF (Nim Intermediate Format) files are text-based files that use a Lisp-like
syntax. They employ a hybrid format where byte offsets into the text are used for
efficient access, making them simultaneously human-readable and machine-efficient.
The text representation is particularly valuable for debugging and introspection.
``--genBif:on`` makes a regular compiler invocation write each semantically
checked module as ``<suffix>.s.bif`` under the build's nimcache directory. This
reuses the semantic artifact format used by IC without enabling incremental
compilation or changing how the program is generated and linked. Tools such as
language servers, debuggers, and binding generators can request these artifacts
when they need resolved symbols and types from an ordinary build.
Each ``.nim`` module produces its own ``.nif`` file during compilation.
The NIF format contains:
Per module ``<suffix>`` (a content hash of the path; see *NIF symbols* below),
under the nimcache directory:
- **Header** - Version information (e.g., `(.nif27)`)
- **Dependencies** - List of source files and dependencies
- **Interface** - Exported symbols and their indices
- **Body** - The intermediate representation of the module's code in Lisp-like syntax
| File | Producer | Purpose |
| ---- | -------- | ------- |
| ``<s>.p.nif`` | nifler | parsed AST (syntactic) |
| ``<s>.deps.nif`` | nifler | **static** import list (syntactic `import`s) |
| ``<s>.s.deps.nif`` | `nim m` | **real** post-sem imports (incl. macro-generated); see *Discovery* |
| ``<s>.nif`` | `nim m` | semmed module (symbols resolved, typed) |
| ``<s>.iface.nif`` | `nim m` | **iface cookie**: hash of the importer-visible surface |
| ``<s>.impl.nif`` | `nim m` | **impl cookie**: hash of the entire content (bodies included) |
| ``<s>.edges.nif`` | `nim m` | **NeedsImpl edges**: modules whose bodies this sem consumed |
| ``<s>.c.nif`` | `nim nifc` | the C text as a NIF, with def/ref markers for DCE & dedup |
| ``ic_config.cfg.nif`` | driver | precompiled config replayed by every child (`icconfig.nim`) |
| ``ic.version`` | driver | format stamp; a mismatch wipes the cache (`icFormatVersion`) |
The NIF format is designed specifically for Nim and allows efficient serialization
and deserialization of the compiler's intermediate representation while remaining
readable and debuggable by tools and developers.
NIF symbols and ownership
The ``nim ic`` Switch
=====================
The ``nim ic`` command initiates incremental compilation for a project.
It automatically manages the build process by:
1. Parsing all source files into ``.nif`` format (using the ``nifler`` tool)
2. Performing semantic analysis on modified modules
3. Generating code only for modules with changes or dependencies on changed modules
4. Generating a build file (in NIFMake format) that orchestrates the compilation
5. Executing the build file through ``nifmake``
Prerequisites
-------------
- **nifler** - Tool for parsing Nim source files into NIF format. The ``nim ic`` command uses ``nifler parse --deps`` to generate both parsed files (``.p.nif``) and dependency files (``.deps.nif``).
- **nifmake** - Build orchestration tool that follows dependencies and executes the build rules defined in ``.build.nif`` files.
If these tools are not available, ``nim ic`` will display instructions on how to
obtain them.
Key Modules for IC Logic
=========================
(See ``../nifspec/doc/nif-spec.md``.) A global symbol is
``<ident>.<disamb>.<moduleSuffix>``. For a **generic instantiation** the
`<disamb>` is not a counter but a *content hash* — `setInstanceDisamb`
(``modulegraphs.nim``) MD5s the generic's identity plus the `typeKey` of every
concrete type argument, masks it to 30 bits and tags it with `InstanceDisambBit`.
So the only part of the name that varies between two modules making the **same**
instantiation (`seq[Foo]`) is the `<moduleSuffix>`. Two consequences drive the
backend:
The primary modules in the compiler that handle incremental compilation logic are:
- **Instance names are content-addressed**: the same instantiation produced in
different modules yields the *same* `<ident>.<disamb>`, so a deterministic dedup
is possible by the *module-suffix-stripped* name. The cross-TU C name
(`ccgtypes.sharedInstanceCName`) and the **merge** stage's live-set/owner
decision (`nifbackend.computeMergeDecision`) both key on this stripped form.
- **The suffix names a mint-site owner.** The `<moduleSuffix>` is the module
*that minted the instance* (the instantiation site), so the same instance has a
different full name in each module that makes it. Because every `cg` process
emits the instances it demands (*emit-everywhere*), the same definition can be
produced by several translation units; the **merge** stage then deterministically
picks the single artifact allowed to embed each body (smallest claimant), which
is the cross-process replacement for the old in-process single-writer machinery.
- **deps.nim** - Dependency analysis and build file generation. Contains the
``commandIc`` procedure which is the main entry point for the ``nim ic`` command.
This module orchestrates the incremental compilation process, handling dependency
traversal (via ``nifler deps``), build rule generation, and build file creation.
The build file is written to ``nifcache/`` directory. This module also explicitly
models ``system.nim`` as a dependency of all modules.
The driver: graph construction (`commandIc`)
============================================
- **ast2nif.nim** - Core mapping between AST and NIF.
1. Stamp/wipe the cache by ``icFormatVersion``.
2. Seed the graph with the root module and **`system.nim`**. `system`'s entire
import closure is folded into one node (one `nim m` invocation) — see
*single-writer* below.
3. ``traverseDeps`` runs ``nifler`` per module and reads ``.deps.nif`` to add
import edges.
4. **SCC grouping**: strongly-connected import cycles are collapsed (Tarjan).
A singleton compiles as ``nim m <mod>``; a cycle compiles as one
``nim m <rep> --icGroup:<member>…`` that builds every member *from source* in
one process (resolving the recursion in memory) and writes each member's NIF.
Only edges *leaving* the component become build-graph inputs.
5. **Discovery fixpoint**: write the build file, run ``nifmake``; if it fails,
re-derive the graph from every module's ``.s.deps.nif`` (adding nodes/edges
for imports the static scanner missed), and retry. See *Discovery*.
6. The backend step (`nim nifc`) depends on every module's semmed NIF, so
``nifmake`` runs it last.
Invalidation: the cookie system
================================
**Code, Logic & Debugging**
===========================
A dependent must re-sem only when a dependency's relevant surface changed. Two
hashes per module (``ast2nif.nim``):
This section focuses on the compiler-side code paths, the logic you will
inspect while debugging IC, and a pragmatic manual workflow for bug hunting
using local invocations such as ``nim m --nimcache:nifcache``.
- **iface cookie** (``.iface.nif``): hashes only the *importer-visible* surface —
exported declarations' **signatures** (for *all* routine kinds: plain procs,
templates, macros, generics, `inline` procs alike), full content for
consts/types, plus import/export/replay/hook records. Routine **bodies are
excluded.** It also chains in the iface cookies of its own dependencies, so a
surface change anywhere in the import closure propagates. A `nim m` rule for a
module depends on its dependencies' iface cookies, so a body-only edit moves no
iface cookie and stops the re-sem cascade.
- **impl cookie** (``.impl.nif``): hashes the *entire* serialized content (private
defs and bodies included), with the module's own iface mixed in.
Core places to inspect
- **`compiler/deps.nim`**: generates the NIF-based build file and implements
``commandIc`` (entry point for ``nim ic``). Look for how build rules are
emitted (calls to the NIF builder) and how inputs/outputs are wired.
- **`compiler/modulegraphs.nim`** and **`compiler/pipelines.nim`**:
dependency graph and compilation pipeline integration — useful when a module
is rebuilt unexpectedly.
**NeedsImpl edges** (``.edges.nif``): if a module *consumed another module's body*
during sem — a macro expansion, a generic instantiation, a `getImpl`, or a
compile-time call run in the VM — it records a strong edge. The dependent is then
gated on that dependency's **impl** cookie instead of its iface cookie, so e.g.
`const x = dep.foo()` re-sems when `foo`'s body changes. Recording sites:
`semExprs.semTemplateExpr` (templates), `seminst.generateInstance` (generics),
`vmgen.genProc` (VM/macros/CT procs), `vm.opcGetImpl` (`getImpl`). Inline
iterators and `inline` procs are *not* tracked — they are inlined at codegen,
where the backend's NIF-mtime invalidation re-codegens their users.
Understanding the NIF text
- NIF files are human-readable; open the per-module ``.nif`` files in
``nifcache/`` to inspect parsed ASTs, dependency lists and interface tables.
- Because NIF uses textual nodes and byte offsets, tools can quickly seek to
positions in the file — but for debugging you usually only need to read the
file top-to-bottom.
Discovery of macro-generated imports
====================================
Manual bug-hunting workflow
- Prepare a clean nimcache directory (relative to your project):
The static scanner only sees syntactic `import`s. A macro can synthesize one
(chronicles does `parseStmt("import chronicles/textlines")` driven by the
`chronicles_sinks` define). Such an import is invisible until sem runs the macro.
Each `nim m` records the imports it *actually* resolved (via the
``semdata.addImportFileDep`` hook → ``graph.importDeps`` → ``ast2nif.writeSemDeps``)
into ``<s>.s.deps.nif``; a child that fails on a not-yet-built import flushes it
before erroring. The driver re-derives the graph from those sidecars — adding the
missing node + the importer→import edge — and reruns to a fixpoint. (This replaced
an earlier `icmissing.txt` side channel.)
```bash
mkdir -p nifcache
```
The backend: per-module `nifc` stages
=====================================
- Parse/semantic-check a single module and write NIF/sem artifacts:
Codegen is no longer one whole-program process. ``nim nifc`` (`cmdNifC`,
``compiler/nifbackend.nim``) is invoked once per **stage** via
``--icBackendStage:<stage>``; `commandIc` emits these as ordinary `nifmake` rules
so "which TUs rebuild" is just "which rules `nifmake` re-fires from input mtimes"
— exactly as the frontend already works. There are four stages:
```bash
nim m --nimcache:nifcache path/to/module.nim
```
1. **`cg`** (``--icBackendStage:cg --icBackendModule:<suffix>``) — generate C for
the *single* named module and write only its ``<s>.c.nif`` artifact. A non-main
target loads only its own import closure (`loadDepClosure`), so the whole
program is **not** pulled into every parallel `cg` process. Codegen is still
demand-driven and **emit-everywhere**: a `cg` process emits every entity it
demands (generic instances, hooks, RTTI), referencing nothing `extern`-only.
There is no whole-program DCE here — a liveness pass over all ~260 NIFs would
cost ~900 MB for a result the merge stage recomputes anyway. The **main**
module's `cg` is special: it loads everything (`loadBackendModules`), emits the
whole-program method dispatchers and `NimMain`, and registers every other
module's init/datInit from the `.c.nif` meta heads — so it runs *last*, after
every other ``.c.nif`` exists. Every `cg` rule always leaves a ``.c.nif`` (empty
if the module owns no code) so its nifmake output exists and the rule settles.
2. **`merge`** (``--icBackendStage:merge``) — a pure artifact pass, *no module
graph loaded*. Reads every ``.c.nif``, computes the one program-wide live set
and, for each unique definition that several `cg` processes emitted, the single
artifact allowed to embed its body; writes that to a merge-decision file
(`computeMergeDecision` / `writeMergeDecision`). This is the cross-process
replacement for the old in-process first-claimant + DCE coordination.
3. **`emit`** (``--icBackendStage:emit --icBackendModule:<suffix>``) — render the
target module's final ``.c`` from its ``.c.nif`` and the merge decision
(`renderCFromArtifact`, dropping globally-dead and non-owned bodies). No codegen
runs; the target is loaded only so `getCFile` yields the path `cg` wrote.
4. **`link`** (``--icBackendStage:link``) — register every module's emitted ``.c``
and run `extccomp.callCCompiler` once (it parallelizes per-file cc and skips
up-to-date objects). Per-module C compile/link directives (`{.passL.}` etc.) are
re-collected here via `replayBackendActions`, since the `cg` processes that
originally saw them are separate processes (without this, e.g. `math`'s `-lm`
would be lost → undefined `floor`/`pow` at link).
- ``nim m`` runs the compiler up to the semantic checking stage for the
specified module and emits intermediate cache files into ``nifcache/``.
- Use this to reproduce and isolate failures in the semantic stage.
Because each stage is a `nifmake` rule keyed on file mtimes, a body-only edit to
one module re-fires that module's `cg`+`emit` (and the `merge`/`link`), not the
whole program — and an unchanged module's `cg` does not run at all.
- Inspect the generated files for that module under ``nifcache/`` (look for
``.nif``, sem/parsed artifacts). Because NIF is text-based you can open and
grep it directly:
Edge cases (and why the machinery exists)
=========================================
```bash
sed -n '1,200p' nifcache/ModuleName.nif
grep -n "someSymbol" -n nifcache/ModuleName.nif
```
- **Single-writer.** Instance type-ids are minted in process-local order, so if
two `nim m` processes both write a module's NIF (e.g. a stdlib module pulled
into `system`'s from-source closure *and* given its own rule), the second
overwrites with different ids and every module checked against the first carries
dangling refs ("symbol has no offset"). Fixed by folding `system`'s closure into
one SCC and by **forwarding the project's defines** to every child so their
`when` bodies (hence import sets and NIF contents) match the scanner's.
- **`when … else: import`.** nifler emits `else`-branch imports unguarded, so a
dead `else: import` would be scheduled. The compiler's own sources were rewritten
to explicit negated `when`s; the vendored nifler later learned to negate prior
conditions for the `else`.
- **`nil` sons of loaded ASTs.** NIF dot-tokens load as `nil` where from-source
ASTs have `nkEmpty`; several passes gained `nil` guards.
- **Sealed loaded types.** Loaded types are `Sealed`; sem/transform mutate via
`unsealForTransform`/`exactReplica(idgen)` (the latter mints a fresh `uniqueId`
so serialized replicas don't collapse).
- **Methods/RTTI ownership.** RTTI and type-bound hooks are emit-everywhere at
`cg` and deduplicated by the `merge` stage, like generic instances; the main
module's `cg` owns the whole-program method dispatchers.
- **Config cost.** Each child re-parsing `nim.cfg` + re-running `config.nims` in
the VM was ~80 ms; replaced by a precompiled `ic_config.cfg.nif` replayed in
`loadConfigs` (`compiler/icconfig.nim`).
- **`koch bootic`** bootstraps the compiler through `nim ic` (a 3-iteration
fixed-point check). It writes its binary to ``bin/nim_ic`` and never clobbers
``bin/nim``.
- To reproduce a full incremental compilation of the project, generate the
build file and run it (``nim ic`` automates this). The build file is generated
in ``nifcache/`` directory. To debug an individual build step, run the command
that the build file would execute manually:
- Parsing step: ``nifler parse --deps input.nim`` (produces ``.p.nif`` and ``.deps.nif``)
- Semantic step: ``nim m --nimcache:nifcache input.nim`` (produces ``.nif``)
- Code generation: ``nim nifc --nimcache:nifcache input.nim`` (produces executable)
Resolved by the rewrite
-----------------------
- Force a cache invalidation for a single module by removing its NIF/sem
artifact and re-running the semantic step:
The whole-program backend's hand-rolled mini-`nifmake` — `computeModuleReuse`,
`enforceDefRetention`, `redirectToLiveModule`, the cached-defs/claim bookkeeping
and the standalone `dce.nim` — **is gone**. Reuse is now just per-rule `nifmake`
mtime checks, and the single-writer decision is the `merge` stage. The old
**cross-mm / `--force` `var not init`** hazard dissolved with it: every codegen
rule's config (including `--mm`) is a declared `nifmake` input, so a stale-config
TU is simply rebuilt rather than mixed in. `koch bootic` is green under both `orc`
and `--mm:refc`.
```bash
rm nifcache/ModuleName.nif
nim m --nimcache:nifcache path/to/ModuleName.nim
```
Known residual hack
-------------------
- When investigating incorrect replayed state (pragmas, `{.compile: ...}`):
inspect the replay actions in ``compiler/ic/replayer.nim`` and open the
module's NIF to find the ``toReplay``/action entries that will be executed
during reload.
- `deps.runNifler` still uses `setLastModificationTime` to mark its scan
up-to-date and deletes a stale parsed file to coordinate with the nifmake nifler
rule — the driver duplicating nifmake's freshness logic. It is explicitly
flagged in the source and folds away with a full frontend/nifler split.
Tips for efficient debugging
- Use ``--path:...`` flags when invoking ``nim m`` to emulate the exact
search paths used in your project, e.g. ``--path:lib --path:vendor``.
- Compare two successive ``.nif`` files with ``diff`` to see what changed and
why a module was rebuilt.
Status and performance
======================
`nim ic` self-builds the compiler (`koch bootic`'s byte-identical fixed-point
check) under both `orc` and `--mm:refc`, and passes the external-package CI set.
Cold full bootstrap on a 32-core box (`-d:release`, **no edits** — IC's worst
case, since incremental reuse is not exercised):
| | wall | notes |
| - | ---- | ----- |
| `koch boot` (classic) | ~1m00s | reference |
| `koch bootic` (`nim ic`) | ~1m39s | **~1.66×** |
This is down from ~7.5× in the whole-program-backend era. IC does modestly more
aggregate work (more processes, NIF re-parsing of imports per process), but on a
many-core box that overhead is absorbed by the parallel `nim m`/`nifc` fan-out,
and the C compile+link floor is shared with the classic backend. On few-core
machines the cold gap is correspondingly wider — IC trades single-build latency
for incremental latency.
The cold number is the *least* favourable comparison: it pays IC's full per-process
overhead while using none of its incremental machinery. **Warm rebuilds — the
actual point of IC — recompile only the modules whose inputs changed** (a body-only
edit re-fires one module's `cg`+`emit`, not the program), so an edit-driven rebuild
is a small fraction of either full build.
The strategic direction (decided 2026-06-13) is to make this NIF backend
(`cmdNifC`) the **default** code generator. The per-module pipeline above is the
realization of that direction; remaining work is *promotion + deletion* of the
classic path, not new machinery.
Design notes and open decisions
===============================
The per-module backend (above) mirrors Nimony's ``src/nimony/deps.nim``: the
backend stopped re-implementing `nifmake`; each stage is a build rule, so reuse is
just mtime checks and the merge stage is the only cross-module coordination.
Settled vs. open:
- **Ownership.** Emittable entities (generic instances, type-bound hooks, RTTI,
lifted procs) are emit-everywhere at `cg` time and deduplicated at `merge` time
(smallest claimant owns each unique body). The earlier idea of a *static*
per-suffix owner computed before codegen was not needed — content-addressed names
make the merge decision deterministic. The precise owner *rule* (minting module
vs. root-type's module) can still be tuned where it would force a downstream
package to own stdlib code.
- **Remaining cleanup.** The `runNifler` `setLastModificationTime` coordination
(above) folds away with a full frontend/nifler split; dead `when` imports could
also be pruned during the `.s.deps` re-derivation.
Validation bar (held on every change): `koch bootic` must reach its byte-identical
fixed point, and binary size must not regress (DCE parity), across the
external-package CI set.
Further possible improvements
=============================
A warm-edit profiling pass (2026-07-02, self-compiling the compiler into a
dedicated `--nimcache`, editing one private proc body — `internalErrorImpl` — in
the hub module `compiler/msgs.nim`) surfaced where a **hub-module** warm rebuild
actually spends its time. The result refines the "a body-only edit re-fires one
module" claim above: that holds for the *backend*, but the *frontend* can still
cascade.
Measured: no-op `0.05s`; hub body edit `~15s`, split **~13s frontend / ~1.6s
backend**. Editing a body in a leaf (few importers) is fast; editing a body in a
widely-imported module is not, and the cost is almost entirely frontend re-sem.
- **Frontend over-invalidation (the dominant hub-edit cost).** Editing *any* body
in a module — even a private routine that is only ever *called* — flips that
module's whole-module **impl cookie** (`writeImplCookie` hashes the entire
serialized module). Every module carrying a **NeedsImpl** edge on it then
re-sems, even though the symbol it actually consumed is unchanged (e.g. a
dependent that expanded the `internalError` *template* needs the template body,
which is untouched; it does **not** need `internalErrorImpl`'s body). In the
msgs edit this re-fires **57** `nim m` processes. A `.s.bif` mtime diff *hides*
this — `.s.bif` is content-stable, so a re-semmed-but-identical module keeps its
timestamp; count actual `nim m` PIDs to see the fan-out.
The precise fix is **per-symbol NeedsImpl gating**: record which *symbols'*
bodies a dependent consumed (the recording site `modulegraphs.recordIcImplDep`
already receives the `PSym`; it currently coarsens to `module(s.itemId)`) and
gate the dependent
on only those. The obstacle is that `nifmake` gates on file mtimes, so
per-symbol granularity needs either many cookie files or a bucketing scheme, and
"which bodies are compile-time-consumable" is entangled with `getImpl` and the
CT call graph (a macro that runs a private helper at CT *does* consume its body).
A conservative narrowing — keep template/generic/macro/`sfCompileTime` bodies
(plus `getImpl` targets) in the impl cookie but drop ordinary runtime routine
bodies — captures the common "edit a private implementation proc" case, at the
cost of proving the exclusion is complete.
- **Serial re-sem chains.** The 57 re-sems above run essentially **one at a time**
despite `--parallel`, because the core modules they belong to form a deep import
*chain* and `nifmake`'s depth-barriered scheduler runs one depth level at a time
(≈1 node per level). This is independent of the invalidation problem: even
perfect per-symbol precision leaves a serial tail whenever the re-sem set is a
chain. Mitigations live in the scheduler (content-stability already stops the
cascade at one level, but does not flatten the chain).
- **Emit stage need not load the module graph (done).** `generateEmitStage` used
to `loadDepClosure`/`loadBackendModules` — materializing a module's whole
transitive import closure as `BModule`s — solely to reach `getCFile(bmod)` for
the output path. `renderCFromArtifact` is pure text filtering over the `.c.nif`
plus the merge decision; it needs none of that. Deriving the `.c` path directly
from the suffix (the same pure computation `deps.backendCFile` uses to *declare*
the stage's output) lets an `emit` process load nothing. Under the
fire-all-every-edit `emit` barrier (see below) this halved backend CPU
(user-time `51s → 24s` on the msgs edit); wall-clock barely moved because the
frontend dominates, but the reduced CPU/RAM contention matters when an editor is
running alongside. `koch ic` stays byte-identical.
- **Do NOT make the merge decision content-stable.** A tempting frontend to the
above: `emit` re-fires for *every* live module whenever `merge` rewrites the
decision file's mtime (deliberate — a decision change must re-render every `.c`
consistently). Writing the decision `OnlyIfChanged` (with a stamp output so the
`merge` rule is not perpetually stale) makes a warm no-op instant, but a real
edit then fires `emit` only for the modules whose `.c.nif` changed — and that
produces **multiple-definition link errors** even when the decision is
byte-identical. Fire-all `emit` is a correctness invariant, not just insurance
(see the comment at `generateEmitStage`): partial `emit` leaves inconsistent
ownership across the `.c` set. This path was tried and reverted; do not retry.
Code, logic & debugging
========================
Core modules:
- **`compiler/deps.nim`** — graph construction, SCC grouping, discovery fixpoint,
build-file generation; `commandIc`.
- **`compiler/ast2nif.nim`** — AST↔NIF, the cookie hashes (`cookieSd`,
`writeIfaceCookie`, `writeImplCookie`, `writeEdgesFile`, `writeSemDeps`).
- **`compiler/nifbackend.nim`** — the per-module backend stages (`generateCgStage`,
`generateMergeStage`, `generateEmitStage`, `generateLinkStage`).
- **`compiler/cnif.nim`** — `.c.nif` artifact read/write, `computeMergeDecision`,
`renderCFromArtifact`.
- **`compiler/icconfig.nim`** — precompiled config.
- **`compiler/pipelines.nim`** / **`modulegraphs.nim`** — pipeline integration and
the graph state (`importDeps`, `icImplDeps`, `icCnifFiles`, `instDisambs`, …).
Manual workflow:
- Frontend a module: ``nim m --nimcache:nifcache path/to/mod.nim`` (writes
``.nif`` + cookies + ``.s.deps``).
- Backend is stage-based (a bare ``nim nifc main.nim`` errors — there is no
whole-program fallback). The exact per-stage commands `nifmake` runs are in the
``*.backend.build.nif`` build file; rerun one directly against an existing cache,
e.g. ``nim nifc --nimcache:nifcache --icBackendStage:cg --icBackendModule:<suffix> main.nim``
to regenerate one module's ``.c.nif``, then ``--icBackendStage:merge`` /
``:emit`` / ``:link``.
- NIF and ``.c.nif`` files are text — open/grep them directly; ``diff`` two
successive ``.nif`` to see why a module rebuilt.
- Force a re-sem: delete the module's ``.nif`` and rerun `nim m`.
- A stale-cache crash after editing the serialization layout means bumping
``icFormatVersion`` (`compiler/options.nim`).
Where to change behavior
- Cache invalidation decisions and build-rule emission are implemented in
``compiler/deps.nim``. When investigating surprising
rebuilds, instrument those modules to log the footprint/hash/comparison
outcome.
See also
========
- NIF format spec: [nifspec/doc/nif-spec.md](../nifspec/doc/nif-spec.md)
- NIFC (C-like target) spec: dist/nimony/doc/nifc-spec.md
- `nif-spec` - NIF format specification (text format and node grammar):
[nifspec/doc/nif-spec.md](../nifspec/doc/nif-spec.md)

View File

@@ -6123,48 +6123,40 @@ instantiations cross multiple different modules:
```nim
# module A
type O* = object
proc genericA*[T](x: T) =
mixin init
init(x)
```
```nim
# module C
import A
proc init*(x: O) = discard
```
```nim
import C
# module B
import A, C
proc genericB*[T](x: T) =
# Without the `bind init` statement, C's `init` proc is not
# available when `genericA` is instantiated through `genericB`
# from `module main`, which does not import C:
# Without the `bind init` statement C's init proc is
# not available when `genericB` is instantiated:
bind init
genericA(x)
```
```nim
# module main
import A, B
genericB(O())
# module C
type O = object
proc init*(x: var O) = discard
```
Because `genericA` uses `mixin init`, `init` is an open symbol that is
resolved when `genericA` is instantiated. Here `genericA` is instantiated
through `genericB`, whose final instantiation happens in `module main`.
Since `module main` does not import `module C`, `init` is not in scope at
that point, and the instantiation fails with ``undeclared identifier: 'init'``.
The `bind init` statement inside `genericB` forwards the `init` symbol that
is visible in `module B` into the instantiation of `genericA`, which makes
the example compile. This `bind`, which re-exposes a symbol to a nested
generic instantiation, is a `delegating bind`:idx:.
```nim
# module main
import B, C
genericB O()
```
In module B has an `init` proc from module C in its scope that is not
taken into account when `genericB` is instantiated which leads to the
instantiation of `genericA`. The solution is to `forward`:idx: these
symbols by a `bind` statement inside `genericB`.
Templates
@@ -8004,9 +7996,6 @@ underlying C `struct`:c: in a `sizeof` expression:
pure, incompleteStruct.} = object
```
Attempting to use `sizeof` on an `incompleteStruct` type at compile-time
will error with "'sizeof' cannot be used with '.incompleteStruct' types".
CompleteStruct pragma
---------------------

View File

@@ -50,23 +50,9 @@ cycle collector's overhead
but `--mm:orc` also produces more machine code than `--mm:arc`, so if you're on a target
where code size matters and you know that your code does not produce cycles, you can
use `--mm:arc`. Notice that the default `async`:idx: implementation produces cycles
and leaks memory with `--mm:arc`, in other words, for `async` you need to use `--mm:orc`
or `--mm:yrc`.
and leaks memory with `--mm:arc`, in other words, for `async` you need to use `--mm:orc`.
Atomic ARC/YRC
--------------
ARC/ORC are not threadsafe if `ref` or other automatically managed types are
accessed across thread boundaries.
Moving isolated subgraphs between threads is supported for ARC/ORC and the language has support
for that in the form of `isolate`. The modes `mm:atomicArc` and `mm:yrc` do offer this thread safety -- at the cost of atomic instructions. Whether that cost is acceptable depends on your program, it hard to give general guidelines. On a modern CPU the potential speedups in the form of increased multi-threading capabilities should outweigh the costs of atomic instructions by far. On an embedded device the atomics would probably only hurt though.
`mm:atomicArc` is a threadsafe variant of ARC: All the optimizations in the form of move semantics etc are still applied. `mm:yrc` is the threadsafe variant of ORC.
YRC is a novel concurrent cycle collection algorithm -- these are beasts to verify
and to get correct so there are dragons lurking here, use at your own risk.
Other MM modes
--------------
@@ -80,7 +66,7 @@ Other MM modes
Heaps are thread-local.
--mm:boehm Boehm based garbage collector, it offers a shared heap.
--mm:go Go's garbage collector, useful for interoperability with Go.
Offers a shared heap. Note that `mm:go` has seen little real world use. Use at your own risk.
Offers a shared heap.
--mm:none No memory management strategy nor a garbage collector. Allocated memory is
simply never freed. You should use `--mm:arc` instead.
@@ -90,7 +76,6 @@ Here is a comparison of the different memory management modes:
================== ======== ================= ============== ====== =================== ===================
Memory Management Heap Reference Cycles Stop-The-World Atomic Valgrind compatible Command line switch
================== ======== ================= ============== ====== =================== ===================
YRC Shared Cycle Collector No Yes Yes `--mm:yrc`
ORC Shared Cycle Collector No No Yes `--mm:orc`
ARC Shared Leak No No Yes `--mm:arc`
Atomic ARC Shared Leak No Yes Yes `--mm:atomicArc`

107
koch.nim
View File

@@ -11,16 +11,16 @@
const
# examples of possible values for repos: Head, ea82b54
NimbleStableCommit = "a399f502dec7ffcd905c1cf54b13274ad990bada" # 0.24.1
AtlasStableCommit = "aa6fb162006f3015aa84c4305e15cb4d230f5ad6" # 0.14.7
ChecksumsStableCommit = "5c132cd332cce5d64a0da9ac3e4c9664313dccb4" # 0.2.2
SatStableCommit = "9d52513b3c68bfb929dbd687d4fb2836cfee6936"
NimbleStableCommit = "aa03f886e4a111d6af9090c6a1f1271d64b66f7b" # 0.22.2
AtlasStableCommit = "ff1f4289482dce94ba9f95b3b0ae16d16e21eb3d" # 0.10.1
ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1
SatStableCommit = "e63eaea8baf00bed8bcd5a29ffd8823abb265b39"
NimonyStableCommit = "f831b953d7c21d9a4b11d0042039e7f84d7c8dc9" # unversioned \
NimonyStableCommit = "750aa47f2139fe5ad69f04b44428b752011fe873" # unversioned \
# Note that Nimony uses Nim as a git submodule but we don't want to install
# Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive
# is **required** here.
# Commit from 2026-07-10 -- stable .bif file format
# Commit from 2026-05-05
# examples of possible values for fusion: #head, #ea82b54, 1.2.3
FusionStableHash = "#562467452b32cb7a97410ea177f083e6d8405734"
@@ -76,7 +76,6 @@ Options:
--skipIntegrityCheck skips integrity check when booting the compiler
Possible Commands:
boot [options] bootstraps with given command line options
bootic [options] bootstraps via the incremental compiler (`nim ic`)
distrohelper [bindir] helper for distro packagers
tools builds Nim related tools
toolsNoExternal builds Nim related tools (except external tools,
@@ -407,55 +406,6 @@ proc boot(args: string, skipIntegrityCheck: bool) =
if not skipIntegrityCheck:
echo "[Warning] executables are still not equal"
proc bootic(args: string, skipIntegrityCheck: bool) =
## Like `boot`, but bootstraps the compiler through the NIF-based incremental
## compiler (`nim ic`) instead of `nim c`. Differences from `boot`:
## * It starts from an already-bootstrapped Nim (found via `findStartNim`): the
## csources compiler is far too old to provide the `ic` command, and the
## `-d:nimKochBootstrap` define used by `boot`'s first stage *disables*
## `commandIc`, so neither can be used here.
## * `nim ic` drives the per-module build and the final link itself (via
## `nifmake`), so there is no `--compileOnly` + `jsonscript` split.
## The 3-step fixed-point check is kept: a successful run proves the compiler
## can compile itself under IC and reproduces a stable binary.
var output = "compiler" / "nim".exe
# Deliberately NOT `bin/nim`: `bootic` must not clobber the development
# compiler (that would replace a fast release `bin/nim` with bootic's build
# and slow every later `koch`/`nim` invocation). The IC-bootstrapped binary
# lands at `bin/nim_ic` instead; `bin/nim` is only ever read (via findStartNim).
var finalDest = "bin" / "nim_ic".exe
let smartNimcache = (if "release" in args or "danger" in args: "nimcache/ric_" else: "nimcache/dic_") &
hostOS & "_" & hostCPU
bundleChecksums(false)
let nimStart = findStartNim().quoteShell()
let times = 2 - ord(skipIntegrityCheck)
# `boot` shares the `compiler/nim` output path; remove it so a fully warm
# cache still relinks and iteration 1 cannot adopt a stale foreign binary.
removeFile output
for i in 0..times:
echo "iteration: ", i+1
# Iteration 1 may build incrementally (that's the point of IC), but every
# later iteration must start from a clean cache: with a warm cache a
# no-change rerun correctly rebuilds nothing, so iteration i+1 would just
# keep iteration i's binary and the fixed-point check would be vacuous.
# The check is only meaningful if the freshly built compiler re-translates
# everything.
if i > 0: removeDir smartNimcache
let nimi = if i == 0: nimStart else: i.thVersion
exec "$# ic --nimcache:$# $# compiler" / "nim.nim" %
[nimi, smartNimcache, args]
if sameFileContent(output, i.thVersion):
copyExe(output, finalDest)
echo "executables are equal: SUCCESS! (IC-bootstrapped compiler: ", finalDest, ")"
return
copyExe(output, (i+1).thVersion)
copyExe(output, finalDest)
when not defined(windows):
if not skipIntegrityCheck:
echo "[Warning] executables are still not equal"
# -------------- clean --------------------------------------------------------
const
@@ -600,40 +550,19 @@ proc xtemp(cmd: string) =
finally:
copyExe(d / "bin" / "nim_backup".exe, d / "bin" / "nim".exe)
proc runIcTestFile(inp: string) =
## Compile a single `tests/ic` file with `nim ic`, once per `#!EDIT!#` fragment
## (each fragment is the file's source after that incremental edit). Only checks
## that `nim ic` exits 0 — the produced binary's output is not verified here.
proc icTest(args: string) =
temp("")
let inp = os.parseCmdLine(args)[0]
let content = readFile(inp)
let nimExe = getAppDir() / "bin" / "nim_temp".exe
var i = 0
for fragment in content.split("#!EDIT!#"):
let file = inp.replace(".nim", "_temp.nim")
writeFile(file, fragment)
var cmd = nimExe & " ic --hint:Conf:off --warnings:off "
cmd.add quoteShell(file)
exec(cmd)
# The `tests/ic` files that `nim ic` must keep compiling. Multi-module tests rely
# on a sibling helper (`timp` -> `myimp`, `tcompiletimeglobal` -> `mctglobal`),
# which exercises the NIF import/load path the single-file tests do not.
const icSuite = ["thallo", "tconverter", "timp", "tmiscs", "tparseutils",
"tcompiletimeglobal", "tsighashstable", "tpureenum", "tgenericoffer",
"tconverterreexport", "ttypeoffer", "ttransitiveoffer",
"tmodsymref", "tmethupref", "temit", "ttraitparam"]
proc icTest(args: string) =
temp("")
let parsed = os.parseCmdLine(args)
if parsed.len > 0 and parsed[0].len > 0:
# `koch ic <file>`: run just that file.
runIcTestFile(parsed[0])
else:
# `koch ic`: the full regression set we want to keep working — the test
# suite plus both self-host bootstraps (`bootic` and `bootic -d:release`).
for t in icSuite:
runIcTestFile("tests" / "ic" / (t & ".nim"))
bootic("", skipIntegrityCheck = false)
bootic("-d:release", skipIntegrityCheck = false)
inc i
proc buildDrNim(args: string) =
if not dirExists("dist/nimz3"):
@@ -669,16 +598,7 @@ proc runCI(cmd: string) =
# boot without -d:nimHasLibFFI to make sure this still works
# `--lib:lib` is needed for bootstrap on openbsd, for reasons described in
# https://github.com/nim-lang/Nim/pull/14291 (`getAppFilename` bugsfor older nim on openbsd).
#
# Bootstrap exactly once per platform. The refc-mm bootstrap is a
# platform-independent compiler-correctness check, so Linux uses it as its sole
# boot (and then runs the whole suite against the refc-built compiler), while
# the other platforms cover the default ORC bootstrap. `koch` is rebuilt
# per-runner, so `when defined(linux)` selects the Linux job at compile time.
when defined(linux):
kochExecFold("Boot Nim refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib")
else:
kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib")
kochExecFold("Boot Nim ORC", "boot -d:release -d:nimStrictMode --lib:lib")
when false: # debugging: when you need to run only 1 test in CI, use something like this:
execFold("debugging test", "nim r tests/stdlib/tosproc.nim")
@@ -732,6 +652,8 @@ proc runCI(cmd: string) =
execFold("build nimsuggest_testing", "nim c -o:bin/nimsuggest_testing -d:release nimsuggest/nimsuggest")
execFold("Run nimsuggest tests", "nim r nimsuggest/tester")
kochExecFold("Testing booting in refc", "boot -d:release --mm:refc -d:nimStrictMode --lib:lib")
proc testUnixInstall(cmdLineRest: string) =
csource("-d:danger" & cmdLineRest)
@@ -822,7 +744,6 @@ when isMainModule:
of cmdArgument:
case normalize(op.key)
of "boot": boot(op.cmdLineRest, skipIntegrityCheck)
of "bootic": bootic(op.cmdLineRest, skipIntegrityCheck)
of "clean": clean(op.cmdLineRest)
of "doc", "docs": buildDocs(op.cmdLineRest & " --d:nimPreviewSlimSystem " & paCode, localDocsOnly, localDocsOut)
of "doc0", "docs0":

View File

@@ -28,15 +28,6 @@ elif defined(netbsd):
EVFILT_PROC* = 4 ## attached to struct proc
EVFILT_SIGNAL* = 5 ## attached to struct proc
EVFILT_TIMER* = 6 ## timers (in ms)
elif defined(haiku):
const
EVFILT_READ* = -1
EVFILT_WRITE* = -2
EVFILT_AIO* = -3 ## attached to aio requests
EVFILT_VNODE* = -4 ## attached to vnodes
EVFILT_PROC* = -5 ## attached to struct proc
EVFILT_SIGNAL* = -6 ## attached to struct proc
EVFILT_TIMER* = -7 ## timers
when defined(macosx):
const
EVFILT_MACHPORT* = -8 ## Mach portsets

View File

@@ -269,23 +269,6 @@ proc processPendingCallbacks(p: PDispatcherBase; didSomeWork: var bool) =
cb()
didSomeWork = true
proc processTimersBeforePoll(
p: PDispatcherBase, didSomeWork: var bool
): Option[int] {.inline.} =
# Do not let an expired timeout overtake completion callbacks which are
# already pending. `adjustTimeout` makes the I/O poll non-blocking when the
# callback queue is non-empty.
if p.callbacks.len == 0:
result = processTimers(p, didSomeWork)
proc processCallbacksAndTimers(p: PDispatcherBase; didSomeWork: var bool) =
# A completed operation can take multiple queued callbacks to propagate
# through its public future. Process the whole chain before expired timers.
processPendingCallbacks(p, didSomeWork)
discard processTimers(p, didSomeWork)
# Timer futures must still propagate within this dispatcher iteration.
processPendingCallbacks(p, didSomeWork)
proc adjustTimeout(
p: PDispatcherBase, pollTimeout: int, nextTimer: Option[int]
): int {.inline.} =
@@ -416,7 +399,7 @@ when defined(windows) or defined(nimdoc):
"No handles or timers registered in dispatcher.")
result = false
let nextTimer = processTimersBeforePoll(p, result)
let nextTimer = processTimers(p, result)
let at = adjustTimeout(p, timeout, nextTimer)
var llTimeout =
if at == -1: winlean.INFINITE
@@ -467,7 +450,10 @@ when defined(windows) or defined(nimdoc):
result = false
else: raiseOSError(errCode)
processCallbacksAndTimers(p, result)
# Timer processing.
discard processTimers(p, result)
# Callback queue processing
processPendingCallbacks(p, result)
var acceptEx: WSAPROC_ACCEPTEX
@@ -1418,7 +1404,7 @@ else:
result = false
var keys: array[64, ReadyKey]
let nextTimer = processTimersBeforePoll(p, result)
let nextTimer = processTimers(p, result)
var count =
p.selector.selectInto(adjustTimeout(p, timeout, nextTimer), keys)
for i in 0..<count:
@@ -1461,7 +1447,10 @@ else:
if writeCbListCount > 0: incl(newEvents, Event.Write)
p.selector.updateHandle(SocketHandle(fd), newEvents)
processCallbacksAndTimers(p, result)
# Timer processing.
discard processTimers(p, result)
# Callback queue processing
processPendingCallbacks(p, result)
proc recv*(socket: AsyncFD, size: int,
flags = {SocketFlag.SafeDisconn}): owned(Future[string]) =

View File

@@ -526,7 +526,7 @@ func commonPrefixLen*[T](c: CritBitTree[T]): int {.inline, since((1, 3)).} =
else: c.root.byte
else: 0
proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} =
proc toCritBitTree*[T](pairs: openArray[(string, T)]): CritBitTree[T] {.since: (1, 3).} =
## Creates a new `CritBitTree` that contains the given `pairs`.
runnableExamples:
doAssert {"a": "0", "b": "1", "c": "2"}.toCritBitTree is CritBitTree[string]
@@ -534,7 +534,7 @@ proc toCritBitTree*[T](pairs: sink openArray[(string, T)]): CritBitTree[T] {.sin
for item in pairs: result.incl item[0], item[1]
proc toCritBitTree*(items: sink openArray[string]): CritBitTree[void] {.since: (1, 3).} =
proc toCritBitTree*(items: openArray[string]): CritBitTree[void] {.since: (1, 3).} =
## Creates a new `CritBitTree` that contains the given `items`.
runnableExamples:
doAssert ["a", "b", "c"].toCritBitTree is CritBitTree[void]

View File

@@ -121,13 +121,6 @@ template unCheckedInc(x) =
inc(x)
{.pop.}
template newSeqForOverwrite(T: typedesc; len: int): untyped =
## Allocates a fixed-length seq whose elements will be assigned by index.
when supportsCopyMem(T) and declared(newSeqUninit):
newSeqUninit[T](len)
else: # TODO: use `newSeqUnsafe` when that's available
newSeq[T](len)
func concat*[T](seqs: varargs[seq[T]]): seq[T] =
## Takes several sequences' items and returns them inside a new sequence.
## All sequences must be of the same type.
@@ -1112,7 +1105,7 @@ template mapIt*(s: typed, op: untyped): untyped =
evalOnceAs(s2, s, compiles((let _ = s)))
var i = 0
var result = newSeqForOverwrite(OutType, s2.len)
var result = newSeq[OutType](s2.len)
for it {.inject.} in s2:
result[i] = op
i += 1
@@ -1178,7 +1171,10 @@ template newSeqWith*(len: int, init: untyped): untyped =
assert seqRand[0] != seqRand[1]
type T = typeof(init)
let newLen = len
var result = newSeqForOverwrite(T, newLen)
when supportsCopyMem(T) and declared(newSeqUninit):
var result = newSeqUninit[T](newLen)
else: # TODO: use `newSeqUnsafe` when that's available
var result = newSeq[T](newLen)
for i in 0 ..< newLen:
result[i] = init
move(result) # refs bug #7295

View File

@@ -246,7 +246,7 @@ proc toHashSet*[A](keys: openArray[A]): HashSet[A] =
result = initHashSet[A](keys.len)
for key in items(keys): result.incl(key)
iterator items*[A](s: HashSet[A]): lent A =
iterator items*[A](s: HashSet[A]): A =
## Iterates over elements of the set `s`.
##
## If you need a sequence with the elements you can use `sequtils.toSeq
@@ -891,7 +891,7 @@ proc `$`*[A](s: OrderedSet[A]): string =
## ```
dollarImpl()
iterator items*[A](s: OrderedSet[A]): lent A =
iterator items*[A](s: OrderedSet[A]): A =
## Iterates over keys in the ordered set `s` in insertion order.
##
## If you need a sequence with the elements you can use `sequtils.toSeq

View File

@@ -15,16 +15,12 @@
## It also provides some fast iterators over lines in text files (or
## other "line-like", variable length, delimited records).
const
nimUseFallBack = defined(nintendoswitch) or defined(nimMemfileFallback)
when defined(windows):
import std/winlean
when defined(nimPreviewSlimSystem):
import std/widestrs
elif defined(posix):
when not nimUseFallBack:
import std/posix
import std/posix
else:
{.error: "the memfiles module is not supported on your operating system!".}
@@ -33,48 +29,45 @@ import std/oserrors
when defined(nimPreviewSlimSystem):
import std/[syncio, assertions]
elif nimUseFallBack:
import std/syncio
from system/ansi_c import c_memchr
proc newEIO(msg: string): ref IOError =
result = (ref IOError)(msg: msg)
when not nimUseFallBack:
proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode =
## Set the size of open file pointed to by `fh` to `newFileSize` if != -1,
## allocating | freeing space from the file system. This routine returns the
## last OSErrorCode found rather than raising to support old rollback/clean-up
## code style. [ Should maybe move to std/osfiles. ]
result = OSErrorCode(0)
if newFileSize < 0 or newFileSize == oldSize:
return result
when defined(windows):
var sizeHigh = int32(newFileSize shr 32)
let sizeLow = int32(newFileSize and 0xffffffff)
let status = setFilePointer(Handle fh, sizeLow, addr(sizeHigh), FILE_BEGIN)
let lastErr = osLastError()
if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or
setEndOfFile(Handle fh) == 0:
result = lastErr
else:
if newFileSize > oldSize: # grow the file
var e: cint = cint(0) # posix_fallocate truncates up when needed.
when declared(posix_fallocate):
while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR):
discard
if e == EINVAL or e == EOPNOTSUPP or e == ENOSYS:
# fallback arguable; Most portable BUT allows SEGV
if ftruncate(fh, newFileSize) == -1:
result = osLastError()
else:
discard
elif e != 0:
result = osLastError()
else: # shrink the file
if ftruncate(fh.cint, newFileSize) == -1:
proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode =
## Set the size of open file pointed to by `fh` to `newFileSize` if != -1,
## allocating | freeing space from the file system. This routine returns the
## last OSErrorCode found rather than raising to support old rollback/clean-up
## code style. [ Should maybe move to std/osfiles. ]
result = OSErrorCode(0)
if newFileSize < 0 or newFileSize == oldSize:
return result
when defined(windows):
var sizeHigh = int32(newFileSize shr 32)
let sizeLow = int32(newFileSize and 0xffffffff)
let status = setFilePointer(Handle fh, sizeLow, addr(sizeHigh), FILE_BEGIN)
let lastErr = osLastError()
if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or
setEndOfFile(Handle fh) == 0:
result = lastErr
else:
if newFileSize > oldSize: # grow the file
var e: cint = cint(0) # posix_fallocate truncates up when needed.
when declared(posix_fallocate):
while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR):
discard
if e == EINVAL or e == EOPNOTSUPP or e == ENOSYS:
# fallback arguable; Most portable BUT allows SEGV
if ftruncate(fh, newFileSize) == -1:
result = osLastError()
else:
discard
elif e != 0:
result = osLastError()
else: # shrink the file
if ftruncate(fh.cint, newFileSize) == -1:
result = osLastError()
type
MemFile* = object ## represents a memory mapped file
@@ -91,89 +84,6 @@ type
else:
handle*: cint ## **Caution**: Posix specific public field.
flags: cint ## **Caution**: Platform specific private field.
when nimUseFallBack:
backing: string
path: string
readonly: bool
allowRemap: bool
when nimUseFallBack:
proc fallbackMappedSize(backingLen, mappedSize, offset: int): int =
if mappedSize < -1:
raise newEIO("mappedSize cannot be less than -1")
if offset < 0 or offset > backingLen:
raise newEIO("offset out of bounds")
if mappedSize == -1:
result = backingLen - offset
else:
result = min(mappedSize, backingLen - offset)
proc setFallbackView(m: var MemFile, mappedSize, offset: int) =
m.size = fallbackMappedSize(m.backing.len, mappedSize, offset)
if m.size > 0:
m.mem = cast[pointer](addr m.backing[offset])
else:
m.mem = nil
proc openFallbackMemFile(filename: string, mode: FileMode, mappedSize,
offset, newFileSize: int,
allowRemap: bool): MemFile =
result = MemFile(
handle: -1,
flags: 0,
path: filename,
readonly: mode == fmRead,
allowRemap: allowRemap
)
if newFileSize != -1:
result.backing = newString(newFileSize)
else:
result.backing = readFile(filename)
setFallbackView(result, mappedSize, offset)
proc mapMemFallback(m: var MemFile, mode: FileMode,
mappedSize, offset: int): pointer =
if not m.allowRemap:
raise newException(IOError,
"Cannot remap MemFile opened with allowRemap=false")
if mode != fmRead and m.readonly:
raise newEIO("cannot write to read-only mapping")
let size = fallbackMappedSize(m.backing.len, mappedSize, offset)
if size > 0:
result = cast[pointer](addr m.backing[offset])
else:
result = nil
proc flushFallback(m: var MemFile) =
if m.readonly or m.path.len == 0:
return
writeFile(m.path, m.backing)
proc resizeFallback(m: var MemFile, newFileSize: int) =
if m.readonly:
raise newException(IOError, "Cannot resize read-only MemFile")
if not m.allowRemap:
raise newException(IOError,
"Cannot resize MemFile opened with allowRemap=false")
if m.size != m.backing.len:
raise newException(IOError, "Cannot resize partial MemFile")
let oldLen = m.backing.len
m.backing.setLen(newFileSize)
for i in oldLen ..< newFileSize:
m.backing[i] = '\0'
setFallbackView(m, newFileSize, 0)
proc closeFallback(m: var MemFile) =
if not m.readonly:
flushFallback(m)
m.mem = nil
m.size = 0
m.handle = -1
m.flags = 0
m.backing = ""
m.path = ""
m.readonly = false
m.allowRemap = false
proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
mappedSize = -1, offset = 0, mapFlags = cint(-1)): pointer =
@@ -184,7 +94,7 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
if mode == fmAppend:
raise newEIO("The append mode is not supported.")
let readonly = mode == fmRead
var readonly = mode == fmRead
when defined(windows):
result = mapViewOfFileEx(
m.mapHandle,
@@ -195,8 +105,6 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
nil)
if result == nil:
raiseOSError(osLastError())
elif nimUseFallBack:
result = mapMemFallback(m, mode, mappedSize, offset)
else:
assert mappedSize > 0
@@ -224,8 +132,6 @@ proc unmapMem*(f: var MemFile, p: pointer, size: int) =
## via `mapMem`.
when defined(windows):
if unmapViewOfFile(p) == 0: raiseOSError(osLastError())
elif nimUseFallBack:
discard
else:
if munmap(p, size) != 0: raiseOSError(osLastError())
@@ -272,7 +178,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
raise newEIO("The append mode is not supported.")
assert newFileSize == -1 or mode != fmRead
let readonly = mode == fmRead
var readonly = mode == fmRead
template rollback =
result.mem = nil
@@ -346,10 +252,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
if closeHandle(result.fHandle) != 0:
result.fHandle = INVALID_HANDLE_VALUE
elif nimUseFallBack:
result = openFallbackMemFile(filename, mode, mappedSize, offset,
newFileSize, allowRemap)
elif defined(posix):
else:
template fail(errCode: OSErrorCode, msg: string) =
rollback()
if result.handle != -1: discard close(result.handle)
@@ -406,8 +309,6 @@ proc flush*(f: var MemFile; attempts: Natural = 3) =
lastErr = osLastError()
if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode:
raiseOSError(lastErr)
elif nimUseFallBack:
flushFallback(f)
else:
for i in 1..attempts:
res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0
@@ -417,71 +318,59 @@ proc flush*(f: var MemFile; attempts: Natural = 3) =
if lastErr != EBUSY.OSErrorCode:
raiseOSError(lastErr, "error flushing mapping")
when nimUseFallBack:
proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError].} =
## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS
## supports it, file space is reserved to ensure room for new virtual pages.
## Caller should wait often enough for `flush` to finish to limit use of
## system RAM for write buffering, perhaps just prior to this call.
## **Note**: this assumes the entire file is mapped read-write at offset 0.
## Also, the value of `.mem` will probably change.
if newFileSize < 1: # Q: include system/bitmasks & use PageSize ?
raise newException(IOError, "Cannot resize MemFile to < 1 byte")
resizeFallback(f, newFileSize)
else:
proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} =
## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS
## supports it, file space is reserved to ensure room for new virtual pages.
## Caller should wait often enough for `flush` to finish to limit use of
## system RAM for write buffering, perhaps just prior to this call.
## **Note**: this assumes the entire file is mapped read-write at offset 0.
## Also, the value of `.mem` will probably change.
if newFileSize < 1: # Q: include system/bitmasks & use PageSize ?
raise newException(IOError, "Cannot resize MemFile to < 1 byte")
when defined(windows):
if not f.wasOpened:
raise newException(IOError, "Cannot resize unopened MemFile")
if f.fHandle == INVALID_HANDLE_VALUE:
raise newException(IOError,
"Cannot resize MemFile opened with allowRemap=false")
if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map
raiseOSError(osLastError())
if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated.
if (let e = setFileSize(f.fHandle.FileHandle, newFileSize);
e != 0.OSErrorCode): raiseOSError(e)
f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil)
if f.mapHandle == 0: # Re-do map
raiseOSError(osLastError())
let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE,
0, 0, WinSizeT(newFileSize), nil)
if m != nil:
f.mem = m
f.size = newFileSize
else:
raiseOSError(osLastError())
elif defined(posix):
if f.handle == -1:
raise newException(IOError,
"Cannot resize MemFile opened with allowRemap=false")
if newFileSize != f.size:
let e = setFileSize(f.handle.FileHandle, newFileSize, f.size)
if e != 0.OSErrorCode: raiseOSError(e)
when defined(linux): #Maybe NetBSD, too?
# On Linux this can be over 100 times faster than a munmap,mmap cycle.
proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint):
pointer {.importc: "mremap", header: "<sys/mman.h>".}
let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint)
if newAddr == cast[pointer](MAP_FAILED):
raiseOSError(osLastError())
else:
if munmap(f.mem, f.size) != 0:
raiseOSError(osLastError())
let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE,
f.flags, f.handle, 0)
if newAddr == cast[pointer](MAP_FAILED):
raiseOSError(osLastError())
f.mem = newAddr
proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} =
## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS
## supports it, file space is reserved to ensure room for new virtual pages.
## Caller should wait often enough for `flush` to finish to limit use of
## system RAM for write buffering, perhaps just prior to this call.
## **Note**: this assumes the entire file is mapped read-write at offset 0.
## Also, the value of `.mem` will probably change.
if newFileSize < 1: # Q: include system/bitmasks & use PageSize ?
raise newException(IOError, "Cannot resize MemFile to < 1 byte")
when defined(windows):
if not f.wasOpened:
raise newException(IOError, "Cannot resize unopened MemFile")
if f.fHandle == INVALID_HANDLE_VALUE:
raise newException(IOError,
"Cannot resize MemFile opened with allowRemap=false")
if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map
raiseOSError(osLastError())
if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated.
if (let e = setFileSize(f.fHandle.FileHandle, newFileSize);
e != 0.OSErrorCode): raiseOSError(e)
f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil)
if f.mapHandle == 0: # Re-do map
raiseOSError(osLastError())
let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE,
0, 0, WinSizeT(newFileSize), nil)
if m != nil:
f.mem = m
f.size = newFileSize
else:
raiseOSError(osLastError())
elif defined(posix):
if f.handle == -1:
raise newException(IOError,
"Cannot resize MemFile opened with allowRemap=false")
if newFileSize != f.size:
let e = setFileSize(f.handle.FileHandle, newFileSize, f.size)
if e != 0.OSErrorCode: raiseOSError(e)
when defined(linux): #Maybe NetBSD, too?
# On Linux this can be over 100 times faster than a munmap,mmap cycle.
proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint):
pointer {.importc: "mremap", header: "<sys/mman.h>".}
let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint)
if newAddr == cast[pointer](MAP_FAILED):
raiseOSError(osLastError())
else:
if munmap(f.mem, f.size) != 0:
raiseOSError(osLastError())
let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE,
f.flags, f.handle, 0)
if newAddr == cast[pointer](MAP_FAILED):
raiseOSError(osLastError())
f.mem = newAddr
f.size = newFileSize
proc close*(f: var MemFile) =
## closes the memory mapped file `f`. All changes are written back to the
@@ -500,8 +389,6 @@ proc close*(f: var MemFile) =
f.fHandle = INVALID_HANDLE_VALUE
if error:
lastErr = osLastError()
elif nimUseFallBack:
closeFallback(f)
else:
error = munmap(f.mem, f.size) != 0
lastErr = osLastError()

View File

@@ -487,18 +487,11 @@ func `/`*(x: Uri, path: string): Uri =
func `?`*(u: Uri, query: openArray[(string, string)]): Uri =
## Concatenates the query parameters to the specified URI object.
## If the URI already has a query string, the new parameters are appended.
runnableExamples:
let foo = parseUri("https://example.com") / "foo" ? {"bar": "qux"}
assert $foo == "https://example.com/foo?bar=qux"
let bar = parseUri("https://example.com/foo?existing=1") ? {"bar": "qux"}
assert $bar == "https://example.com/foo?existing=1&bar=qux"
result = u
let newQuery = encodeQuery(query)
if newQuery.len > 0:
if result.query.len > 0:
result.query.add('&')
result.query.add(newQuery)
result.query = encodeQuery(query)
func `$`*(u: Uri): string =
## Returns the string representation of the specified URI object.

View File

@@ -913,14 +913,17 @@ proc findAll*(n: XmlNode, tag: string, caseInsensitive = false): seq[XmlNode] =
proc xmlConstructor(a: NimNode): NimNode =
if a.kind == nnkCall:
result = newCall("newXmlTree", newStrLitNode($a[0]))
result = newCall("newXmlTree", toStrLit(a[0]))
var attrs = newNimNode(nnkBracket, a)
var newStringTabCall = newCall(bindSym"newStringTable", attrs,
bindSym"modeCaseSensitive")
var elements = newNimNode(nnkBracket, a)
for i in 1..a.len-1:
if a[i].kind == nnkExprEqExpr:
attrs.add(newStrLitNode($a[i][0]))
# In order to support attributes like `data-lang` we have to
# replace whitespace because `toStrLit` gives `data - lang`.
let attrName = toStrLit(a[i][0]).strVal.replace(" ", "")
attrs.add(newStrLitNode(attrName))
attrs.add(a[i][1])
#echo repr(attrs)
else:

View File

@@ -11,119 +11,17 @@ when defined(nimPreviewSlimSystem):
when weirdTarget:
discard
elif defined(windows):
import std/winlean
from std/strutils import toHex, toLowerAscii
const
reparseHeaderSize = 8
substituteNameOffsetField = 8
substituteNameLengthField = 10
symlinkFlagsField = 16
mountPointPathBufferOffset = 16
symlinkPathBufferOffset = 20
type
ReparseBuffer = array[MAXIMUM_REPARSE_DATA_BUFFER_SIZE, byte]
ReparseLinkInfo = object
tag: int32
flags: int32
pathBufOffset: int
flagsField: int
substituteNameOffset: int
substituteNameLength: int
template readU16(buf: ReparseBuffer; off: int): uint16 =
uint16(buf[off]) or (uint16(buf[off + 1]) shl 8)
template readI32(buf: ReparseBuffer; off: int): int32 =
cast[int32](
uint32(buf[off]) or (uint32(buf[off + 1]) shl 8) or
(uint32(buf[off + 2]) shl 16) or (uint32(buf[off + 3]) shl 24))
func startsWithAsciiIgnoreCase(wide: openArray[Utf16Char]; prefix: openArray[char]): bool =
## Matches an ASCII prefix against UTF-16 code units.
##
## This is only correct for ASCII prefixes.
## It is not a valid general case-insensitive Unicode comparison
## and must not be used for arbitrary UTF-16 text.
if prefix.len > wide.len:
return false
var i = 0
while i < prefix.len:
let rune = ord(wide[i])
if rune > 0x7F or toLowerAscii(char(rune)) != toLowerAscii(prefix[i]):
return false
inc i
true
proc decodeWinTarget(wide: openArray[Utf16Char]): string =
if wide.startsWithAsciiIgnoreCase(r"\??\unc\"):
r"\\" & $(wide.toOpenArray(8, wide.len - 1))
elif wide.startsWithAsciiIgnoreCase(r"\??\"):
$(wide.toOpenArray(4, wide.len - 1))
else:
$wide
template invalidReparseData(path, details: string) =
raise newException(OSError,
"expandSymlink: invalid reparse data for " & path & " (" & details & ")")
proc parseReparseLinkInfo(buf: ReparseBuffer; bytesReturned: int;
symlinkPath: string): ReparseLinkInfo =
if bytesReturned < reparseHeaderSize:
invalidReparseData(symlinkPath, "truncated header")
let
reparseDataLen = int(readU16(buf, 4))
wholeDataLen = reparseHeaderSize + reparseDataLen
if wholeDataLen > bytesReturned:
invalidReparseData(symlinkPath, "payload exceeds returned size")
result.tag = readI32(buf, 0)
case result.tag
of IO_REPARSE_TAG_SYMLINK:
result.pathBufOffset = symlinkPathBufferOffset
result.flagsField = symlinkFlagsField
of IO_REPARSE_TAG_MOUNT_POINT:
result.pathBufOffset = mountPointPathBufferOffset
result.flagsField = -1
else:
raise newException(OSError,
"expandSymlink: unsupported reparse tag for " & symlinkPath &
" (ReparseTag=0x" & toHex(result.tag) & ")")
if result.pathBufOffset > wholeDataLen:
invalidReparseData(symlinkPath, "missing path buffer")
result.substituteNameOffset = int(readU16(buf, substituteNameOffsetField))
result.substituteNameLength = int(readU16(buf, substituteNameLengthField))
if result.substituteNameLength <= 0:
invalidReparseData(symlinkPath, "empty substitute name")
if (result.substituteNameOffset and 1) != 0 or
(result.substituteNameLength and 1) != 0:
invalidReparseData(symlinkPath, "unaligned UTF-16 substitute name")
let startByte = result.pathBufOffset + result.substituteNameOffset
let endByte = startByte + result.substituteNameLength
if startByte < result.pathBufOffset or endByte < startByte or
endByte > wholeDataLen:
invalidReparseData(symlinkPath, "substitute name out of bounds")
result.flags =
if result.flagsField >= 0:
readI32(buf, result.flagsField)
else:
0
import std/[winlean, times]
elif defined(posix):
import std/posix
when weirdTarget:
{.pragma: noWeirdTarget, error: "this proc is not available on the NimScript/js target".}
else:
{.pragma: noWeirdTarget.}
when defined(nimscript):
# for procs already defined in scriptconfig.nim
template noNimJs(body): untyped = discard
@@ -158,65 +56,13 @@ proc createSymlink*(src, dest: string) {.noWeirdTarget.} =
raiseOSError(osLastError(), $(src, dest))
proc expandSymlink*(symlinkPath: string): string {.noWeirdTarget.} =
## Returns the stored target of the symbolic link `symlinkPath`.
## Returns a string representing the path to which the symbolic link points.
##
## This expands exactly one level of indirection, like POSIX `readlink`.
## If the target is itself a symbolic link, it is returned as-is rather than
## being expanded further.
##
## On POSIX, raises `OSError` if `symlinkPath` is not a symbolic link or if
## the target cannot be read.
##
## On Windows, this supports symbolic links and junctions by reading the
## reparse point payload directly. Unsupported reparse tags raise `OSError`.
##
## On Nintendo Switch this is currently a noop: `symlinkPath` is simply
## returned, without checking whether it is actually a symbolic link.
## On Windows this is a noop, `symlinkPath` is simply returned.
##
## See also:
## * `createSymlink proc`_
when defined(windows):
let handle = createFileW(
newWideCString(symlinkPath),
0'i32,
FILE_SHARE_READ or FILE_SHARE_WRITE or FILE_SHARE_DELETE,
nil,
OPEN_EXISTING,
FILE_FLAG_OPEN_REPARSE_POINT or FILE_FLAG_BACKUP_SEMANTICS,
Handle(0)
)
if handle == INVALID_HANDLE_VALUE:
raiseOSError(osLastError(), "expandSymlink: cannot open " & symlinkPath)
defer:
discard closeHandle(handle)
var buf: ReparseBuffer
var bytesReturned: DWORD
if deviceIoControl(
handle,
FSCTL_GET_REPARSE_POINT,
nil, 0'i32,
addr buf[0], DWORD(buf.len),
bytesReturned,
nil
) == 0:
raiseOSError(osLastError(),
"expandSymlink: DeviceIoControl failed for " & symlinkPath)
let
info = parseReparseLinkInfo(buf, int(bytesReturned), symlinkPath)
startByte = info.pathBufOffset + info.substituteNameOffset
runeLen = info.substituteNameLength shr 1
wideSlicePtr = cast[ptr UncheckedArray[Utf16Char]](addr buf[startByte])
if info.tag == IO_REPARSE_TAG_SYMLINK and
(info.flags and SYMLINK_FLAG_RELATIVE) != 0:
return $(wideSlicePtr.toOpenArray(0, runeLen - 1))
decodeWinTarget(wideSlicePtr.toOpenArray(0, runeLen - 1))
elif defined(nintendoswitch):
when defined(windows) or defined(nintendoswitch):
result = symlinkPath
else:
var bufLen = 1024

View File

@@ -24,20 +24,9 @@ proc createSymlink*(src, dest: Path) {.inline.} =
createSymlink(src.string, dest.string)
proc expandSymlink*(symlinkPath: Path): Path {.inline.} =
## Returns the stored target of the symbolic link `symlinkPath`.
## Returns a string representing the path to which the symbolic link points.
##
## This expands exactly one level of indirection, like POSIX `readlink`.
## If the target is itself a symbolic link, it is returned as-is rather than
## being expanded further.
##
## On POSIX, raises `OSError` if `symlinkPath` is not a symbolic link or if
## the target cannot be read.
##
## On Windows, this supports symbolic links and junctions by reading the
## reparse point payload directly. Unsupported reparse tags raise `OSError`.
##
## On Nintendo Switch this is currently a noop: `symlinkPath` is simply
## returned, without checking whether it is actually a symbolic link.
## On Windows this is a noop, `symlinkPath` is simply returned.
##
## See also:
## * `createSymlink proc`_

View File

@@ -185,88 +185,47 @@ when not (defined(cpu16) or defined(cpu8)):
proc newWideCString*(s: string): WideCStringObj =
result = newWideCString(cstring s, s.len)
iterator decodeUtf16(w: WideCString; replacement: int): int =
## Looks for a terminating NUL for length
proc `$`*(w: WideCString, estimate: int, replacement: int = 0xFFFD): string =
result = newStringOfCap(estimate + estimate shr 2)
var i = 0
while w[i].int16 != 0'i16:
var ch = ord(w[i])
inc i
if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
# If the 16 bits following the high surrogate are NOT in the source...
if w[i].int16 == 0'i16:
ch = replacement #invalid UTF-16
# If the 16 bits following the high surrogate are in the source buffer...
let ch2 = ord(w[i])
# If it's a low surrogate, convert to UTF32:
if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
else:
let ch2 = ord(w[i])
# If it's a low surrogate, convert to UTF32:
if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
else:
ch = replacement #invalid UTF-16
#invalid UTF-16
ch = replacement
elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
ch = replacement #invalid UTF-16
yield ch
#invalid UTF-16
ch = replacement
iterator decodeUtf16(w: openArray[Utf16Char]; replacement: int): int =
## Doesn't look for terminating NUL for length, trusts `w.len`
var i = 0
while i < w.len:
var ch = ord(w[i])
inc i
if ch >= UNI_SUR_HIGH_START and ch <= UNI_SUR_HIGH_END:
# If the 16 bits following the high surrogate are NOT in the source...
if i >= w.len:
ch = replacement #invalid UTF-16
else:
let ch2 = ord(w[i])
# If it's a low surrogate, convert to UTF32:
if ch2 >= UNI_SUR_LOW_START and ch2 <= UNI_SUR_LOW_END:
ch = (((ch and halfMask) shl halfShift) + (ch2 and halfMask)) + halfBase
inc i
else:
ch = replacement #invalid UTF-16
elif ch >= UNI_SUR_LOW_START and ch <= UNI_SUR_LOW_END:
ch = replacement #invalid UTF-16
yield ch
proc addUtf8(dest: var string; rune: int) =
if rune < 0x80:
dest.add chr(rune)
elif rune < 0x800:
dest.add chr((rune shr 6) or 0xc0)
dest.add chr((rune and 0x3f) or 0x80)
elif rune < 0x10000:
dest.add chr((rune shr 12) or 0xe0)
dest.add chr(((rune shr 6) and 0x3f) or 0x80)
dest.add chr((rune and 0x3f) or 0x80)
elif rune <= 0x10FFFF:
dest.add chr((rune shr 18) or 0xf0)
dest.add chr(((rune shr 12) and 0x3f) or 0x80)
dest.add chr(((rune shr 6) and 0x3f) or 0x80)
dest.add chr((rune and 0x3f) or 0x80)
else:
# replacement char (in case user give very large number):
dest.add chr(0xFFFD shr 12 or 0b1110_0000)
dest.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00)
dest.add chr(0xFFFD and ones(6) or 0b10_0000_00)
proc `$`*(w: openArray[Utf16Char]; replacement: int = 0xFFFD): string =
## Decodes a length-delimited UTF-16 slice to UTF-8.
##
## Unlike the `WideCString` overloads, this preserves the provided length
## and does not search for a terminating NUL.
if w.len == 0:
result = ""
else:
result = newStringOfCap(w.len + w.len shr 2)
for rune in w.decodeUtf16(replacement):
result.addUtf8(rune)
proc `$`*(w: WideCString; estimate: int; replacement: int = 0xFFFD): string =
result = newStringOfCap(estimate + estimate shr 2)
for rune in w.decodeUtf16(replacement):
result.addUtf8(rune)
if ch < 0x80:
result.add chr(ch)
elif ch < 0x800:
result.add chr((ch shr 6) or 0xc0)
result.add chr((ch and 0x3f) or 0x80)
elif ch < 0x10000:
result.add chr((ch shr 12) or 0xe0)
result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr((ch and 0x3f) or 0x80)
elif ch <= 0x10FFFF:
result.add chr((ch shr 18) or 0xf0)
result.add chr(((ch shr 12) and 0x3f) or 0x80)
result.add chr(((ch shr 6) and 0x3f) or 0x80)
result.add chr((ch and 0x3f) or 0x80)
else:
# replacement char(in case user give very large number):
result.add chr(0xFFFD shr 12 or 0b1110_0000)
result.add chr(0xFFFD shr 6 and ones(6) or 0b10_0000_00)
result.add chr(0xFFFD and ones(6) or 0b10_0000_00)
proc `$`*(s: WideCString): string =
result = s $ 80

Some files were not shown because too many files have changed in this diff Show More