diff --git a/compiler/ast.nim b/compiler/ast.nim index eebe6f257e..67f111c984 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -549,12 +549,19 @@ proc add*(father, son: PType) = proc addAllowNil*(father, son: PType) {.inline.} = father.sonsImpl.add son -template `[]`*(n: PType, i: int): PType = n.sonsImpl[i] +template `[]`*(n: PType, i: int): PType = + if n.state == Partial: loadType(n) + n.sonsImpl[i] template `[]=`*(n: PType, i: int; x: PType) = + if n.state == Partial: loadType(n) n.sonsImpl[i] = x -template `[]`*(n: PType, i: BackwardsIndex): PType = n[n.len - i.int] -template `[]=`*(n: PType, i: BackwardsIndex; x: PType) = n[n.len - i.int] = x +template `[]`*(n: PType, i: BackwardsIndex): PType = + if n.state == Partial: loadType(n) + n[n.len - i.int] +template `[]=`*(n: PType, i: BackwardsIndex; x: PType) = + if n.state == Partial: loadType(n) + n[n.len - i.int] = x proc getDeclPragma*(n: PNode): PNode = ## return the `nkPragma` node for declaration `n`, or `nil` if no pragma was found. @@ -791,29 +798,57 @@ proc replaceFirstSon*(n, newson: PNode) {.inline.} = proc replaceSon*(n: PNode; i: int; newson: PNode) {.inline.} = n.sons[i] = newson -proc last*(n: PType): PType {.inline.} = n.sonsImpl[^1] +proc last*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[^1] -proc elementType*(n: PType): PType {.inline.} = n.sonsImpl[^1] -proc skipModifier*(n: PType): PType {.inline.} = n.sonsImpl[^1] +proc elementType*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[^1] -proc indexType*(n: PType): PType {.inline.} = n.sonsImpl[0] -proc baseClass*(n: PType): PType {.inline.} = n.sonsImpl[0] +proc skipModifier*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[^1] + +proc indexType*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[0] + +proc baseClass*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[0] proc base*(t: PType): PType {.inline.} = + if t.state == Partial: loadType(t) result = t.sonsImpl[0] -proc returnType*(n: PType): PType {.inline.} = n.sonsImpl[0] +proc returnType*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[0] + proc setReturnType*(n, r: PType) {.inline.} = + if n.state == Partial: loadType(n) n.sonsImpl[0] = r + proc setIndexType*(n, idx: PType) {.inline.} = + if n.state == Partial: loadType(n) n.sonsImpl[0] = idx -proc firstParamType*(n: PType): PType {.inline.} = n.sonsImpl[1] -proc firstGenericParam*(n: PType): PType {.inline.} = n.sonsImpl[1] +proc firstParamType*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[1] -proc typeBodyImpl*(n: PType): PType {.inline.} = n.sonsImpl[^1] +proc firstGenericParam*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[1] -proc genericHead*(n: PType): PType {.inline.} = n.sonsImpl[0] +proc typeBodyImpl*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[^1] + +proc genericHead*(n: PType): PType {.inline.} = + if n.state == Partial: loadType(n) + n.sonsImpl[0] proc skipTypes*(t: PType, kinds: TTypeKinds): PType = ## Used throughout the compiler code to test whether a type tree contains or @@ -854,14 +889,6 @@ proc newFloatNode*(kind: TNodeKind, floatVal: BiggestFloat): PNode = result = newNode(kind) result.floatVal = floatVal -proc newStrNode*(kind: TNodeKind, strVal: string): PNode = - result = newNode(kind) - result.strVal = strVal - -proc newStrNode*(strVal: string; info: TLineInfo): PNode = - result = newNodeI(nkStrLit, info) - result.strVal = strVal - proc newProcNode*(kind: TNodeKind, info: TLineInfo, body: PNode, params, name, pattern, genericParams, diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index cac2743a1e..b5f229d94a 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -11,15 +11,93 @@ import std / [assertions, tables, sets] from std / strutils import startsWith +from std / os import fileExists import astdef, idents, msgs, options import lineinfos as astli import pathutils #, modulegraphs import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos, nifindexes, nifreader] import "../dist/nimony/src/gear2" / modnames +import "../dist/nimony/src/models" / nifindex_tags import ic / [enum2nif] +# Re-export types needed for hook, converter, and method handling +export nifindexes.AttachedOp, nifindexes.HookIndexEntry, nifindexes.HooksPerType +export nifindexes.ClassIndexEntry, nifindexes.MethodIndexEntry + +proc toAttachedOp*(op: TTypeAttachedOp): AttachedOp = + ## Maps Nim compiler's TTypeAttachedOp to nimony's AttachedOp. + ## Returns attachedDestroy for attachedDeepCopy (caller should skip it). + case op + of attachedDestructor: attachedDestroy + of attachedAsgn: attachedCopy + of attachedWasMoved: nifindexes.attachedWasMoved + of attachedDup: nifindexes.attachedDup + of attachedSink: nifindexes.attachedSink + of attachedTrace: nifindexes.attachedTrace + of attachedDeepCopy: attachedDestroy # Not supported, caller should skip + +proc toTTypeAttachedOp*(op: AttachedOp): TTypeAttachedOp = + ## Maps nimony's AttachedOp back to Nim compiler's TTypeAttachedOp. + case op + of attachedDestroy: attachedDestructor + of attachedCopy: attachedAsgn + of nifindexes.attachedWasMoved: astdef.attachedWasMoved + of nifindexes.attachedDup: astdef.attachedDup + of nifindexes.attachedSink: astdef.attachedSink + of nifindexes.attachedTrace: astdef.attachedTrace + + +proc cachedModuleSuffix*(config: ConfigRef; fileIdx: FileIndex): string = + ## Gets or computes the module suffix for a FileIndex. + ## For NIF modules, the suffix is already stored in the file info. + ## For source files, computes it from the path. + let fullPath = toFullPath(config, fileIdx) + if fileInfoKind(config, fileIdx) == fikNifModule: + result = fullPath # Already a suffix + else: + result = moduleSuffix(fullPath, cast[seq[string]](config.searchPaths)) + +proc toHookIndexEntry*(config: ConfigRef; typeId: ItemId; hookSym: PSym): HookIndexEntry = + ## Converts a type ItemId and hook symbol to a HookIndexEntry for the NIF index. + let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex) + let hookSymName = hookSym.name.s & "." & $hookSym.disamb & "." & cachedModuleSuffix(config, hookSym.itemId.module.FileIndex) + let typSymId = pool.syms.getOrIncl(typeSymName) + let hookSymId = pool.syms.getOrIncl(hookSymName) + # Check if it's a generic hook (has non-empty generic params) + let isGeneric = hookSym.astImpl != nil and hookSym.astImpl.len > genericParamsPos and + hookSym.astImpl[genericParamsPos].kind != nkEmpty + result = HookIndexEntry(typ: typSymId, hook: hookSymId, isGeneric: isGeneric) + +proc toConverterIndexEntry*(config: ConfigRef; converterSym: PSym): (nifstreams.SymId, nifstreams.SymId) = + ## Converts a converter symbol to an index entry (destType, converterSym). + ## Returns the destination type's SymId and the converter's SymId. + # Get the return type of the converter (destination type) + let retType = converterSym.typImpl + if retType != nil and retType.sonsImpl.len > 0: + let destType = retType.sonsImpl[0] # Return type is first son + if destType != nil: + let destTypeSymName = "`t" & $destType.itemId.item & "." & cachedModuleSuffix(config, destType.itemId.module.FileIndex) + let convSymName = converterSym.name.s & "." & $converterSym.disamb & "." & cachedModuleSuffix(config, converterSym.itemId.module.FileIndex) + result = (pool.syms.getOrIncl(destTypeSymName), pool.syms.getOrIncl(convSymName)) + return + # Fallback: return empty entry + result = (nifstreams.SymId(0), nifstreams.SymId(0)) + +proc toMethodIndexEntry*(config: ConfigRef; methodSym: PSym; signature: string): MethodIndexEntry = + ## Converts a method symbol to a MethodIndexEntry. + let methodSymName = methodSym.name.s & "." & $methodSym.disamb & "." & cachedModuleSuffix(config, methodSym.itemId.module.FileIndex) + result = MethodIndexEntry( + fn: pool.syms.getOrIncl(methodSymName), + signature: pool.strings.getOrIncl(signature) + ) + +proc toClassSymId*(config: ConfigRef; typeId: ItemId): nifstreams.SymId = + ## Converts a type ItemId to its SymId for the class index. + let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex) + result = pool.syms.getOrIncl(typeSymName) + # ---------------- Line info handling ----------------------------------------- type @@ -51,13 +129,14 @@ proc nifLineInfo(w: var LineInfoWriter; info: TLineInfo): PackedLineInfo = result = NoLineInfo else: let fid = get(w, info.fileIndex) - result = pack(w.man, fid, info.line.int32, info.col) + # Must use pool.man since toString uses pool.man to unpack + result = pack(pool.man, fid, info.line.int32, info.col) proc oldLineInfo(w: var LineInfoWriter; info: PackedLineInfo): TLineInfo = if info == NoLineInfo: result = unknownLineInfo else: - var x = unpack(w.man, info) + var x = unpack(pool.man, info) var fileIdx: FileIndex if w.fileV == x.file: fileIdx = w.fileK @@ -73,19 +152,12 @@ proc oldLineInfo(w: var LineInfoWriter; info: PackedLineInfo): TLineInfo = # -------------- Module name handling -------------------------------------------- -proc modname(moduleToNifSuffix: var Table[FileIndex, string]; module: int; conf: ConfigRef): string = - let idx = module.FileIndex - # copied from ../nifgen.nim - result = moduleToNifSuffix.getOrDefault(idx) - if result.len == 0: - let fp = toFullPath(conf, idx) - result = moduleSuffix(fp, cast[seq[string]](conf.searchPaths)) - moduleToNifSuffix[idx] = result - #echo result, " -> ", fp +proc modname(module: int; conf: ConfigRef): string = + cachedModuleSuffix(conf, module.FileIndex) -proc modname(moduleToNifSuffix: var Table[FileIndex, string]; module: PSym; conf: ConfigRef): string = +proc modname(module: PSym; conf: ConfigRef): string = assert module.kindImpl == skModule - result = modname(moduleToNifSuffix, module.positionImpl, conf) + modname(module.positionImpl, conf) @@ -123,11 +195,15 @@ type infos: LineInfoWriter currentModule: int32 decodedFileIndices: HashSet[FileIndex] - moduleToNifSuffix: Table[FileIndex, string] locals: HashSet[ItemId] # track proc-local symbols inProc: int writtenTypes: seq[PType] # types written in this module, to be unloaded later writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later + exports: Table[FileIndex, HashSet[string]] # module -> specific symbol names (empty = all) + +const + # Symbol kinds that are always local to a proc and should never have module suffix + skLocalSymKinds = {skParam, skGenericParam, skForVar, skResult, skTemp} proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, @@ -135,11 +211,11 @@ proc toNifSymName(w: var Writer; sym: PSym): string = result = sym.name.s result.add '.' result.addInt sym.disamb - if sym.itemId notin w.locals: + if sym.kindImpl notin skLocalSymKinds and sym.itemId notin w.locals: # Global symbol: ident.disamb.moduleSuffix let module = sym.itemId.module result.add '.' - result.add modname(w.moduleToNifSuffix, module, w.infos.config) + result.add modname(module, w.infos.config) type ParsedSymName* = object @@ -191,7 +267,7 @@ proc writeFlags[E](dest: var TokenBuf; flags: set[E]) = proc trLineInfo(w: var Writer; info: TLineInfo): PackedLineInfo {.inline.} = result = nifLineInfo(w.infos, info) -proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode) +proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) @@ -201,7 +277,7 @@ proc typeToNifSym(w: var Writer; typ: PType): string = result.add '.' result.addInt typ.uniqueId.item result.add '.' - result.add modname(w.moduleToNifSuffix, typ.uniqueId.module, w.infos.config) + result.add modname(typ.uniqueId.module, w.infos.config) proc writeLoc(w: var Writer; dest: var TokenBuf; loc: TLoc) = dest.addIdent toNifTag(loc.k) @@ -259,6 +335,26 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = dest.addStrLit lib.name writeNode w, dest, lib.path +proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) # forward declaration + +proc collectGenericParams(w: var Writer; n: PNode) = + ## Pre-collect generic param symbols into w.locals before writing the type. + ## This ensures generic params get consistent short names, and their sdefs + ## are written in the type (where lazy loading can find them). + if n == nil: return + case n.kind + of nkSym: + if n.sym != nil and w.inProc > 0: + w.locals.incl(n.sym.itemId) + of nkIdentDefs, nkVarTuple: + for i in 0 ..< max(0, n.len - 2): + collectGenericParams(w, n[i]) + of nkGenericParams: + for child in n: + collectGenericParams(w, child) + else: + discard + proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addParLe sdefTag, trLineInfo(w, sym.infoImpl) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo @@ -266,13 +362,6 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addIdent "x" else: dest.addDotToken - if sym.magicImpl == mNone: - dest.addDotToken - else: - dest.addIdent toNifTag(sym.magicImpl) - writeFlags(dest, sym.flagsImpl) - writeFlags(dest, sym.optionsImpl) - dest.addIntLit sym.offsetImpl # field `disamb` made part of the name, so do not store it here dest.buildTree sym.kindImpl.toNifTag: case sym.kindImpl @@ -282,28 +371,70 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addIntLit sym.alignmentImpl else: discard + + if sym.magicImpl == mNone: + dest.addDotToken + else: + dest.addIdent toNifTag(sym.magicImpl) + writeFlags(dest, sym.flagsImpl) + writeFlags(dest, sym.optionsImpl) + dest.addIntLit sym.offsetImpl + if sym.kindImpl == skModule: dest.addDotToken() # position will be set by the loader! else: dest.addIntLit sym.positionImpl + + # For routine symbols, pre-collect generic params into w.locals before writing + # the type. This ensures they get consistent short names, and their sdefs are + # written in the type where lazy loading can find them via extractLocalSymsFromTree. + if sym.kindImpl in routineKinds and sym.astImpl != nil and sym.astImpl.len > genericParamsPos: + inc w.inProc + collectGenericParams(w, sym.astImpl[genericParamsPos]) + dec w.inProc + writeType(w, dest, sym.typImpl) writeSym(w, dest, sym.ownerFieldImpl) - # We do not store `sym.ast` here but instead set it in the deserializer - #writeNode(w, sym.ast) + # Store the AST for routine symbols (procs, funcs, etc.) + if sym.kindImpl in routineKinds: + writeNode(w, dest, sym.astImpl, forAst = true) + else: + dest.addDotToken writeLoc w, dest, sym.locImpl writeNode(w, dest, sym.constraintImpl) writeSym(w, dest, sym.instantiatedFromImpl) dest.addParRi # Collect for later unloading after entire module is written - if sym.kindImpl notin {skModule, skPackage}: + if sym.kindImpl notin {skPackage}: # do not unload modules w.writtenSyms.add sym +proc shouldWriteSymDef(w: Writer; sym: PSym): bool {.inline.} = + # Don't write module/package symbols - they don't have NIF files + if sym.kindImpl in {skPackage}: + return false + # Already written - don't write again + if sym.state == Sealed: + return false + # If the symbol belongs to current module and would be written WITHOUT module suffix + # (due to being in w.locals or being in skLocalSymKinds), it MUST have an sdef. + # Otherwise it gets written as a bare SymUse and can't be found when loading. + if sym.itemId.module == w.currentModule: + if sym.itemId in w.locals or sym.kindImpl in skLocalSymKinds: + return true # Would be written without module suffix, needs sdef + if sym.state == Complete: + return true # Normal case for global symbols + return false + proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = if sym == nil: dest.addDotToken() - elif sym.itemId.module == w.currentModule and sym.state == Complete: + elif sym.kindImpl in {skPackage}: + # Write module/package symbols as dots - they're resolved differently + # (by position/FileIndex, not by NIF lookup) + dest.addDotToken() + elif shouldWriteSymDef(w, sym): sym.state = Sealed writeSymDef(w, dest, sym) else: @@ -314,7 +445,7 @@ proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = if sym == nil: dest.addDotToken() - elif sym.itemId.module == w.currentModule and sym.state == Complete: + elif shouldWriteSymDef(w, sym): sym.state = Sealed if n.typField != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): @@ -357,24 +488,29 @@ proc addLocalSyms(w: var Writer; n: PNode) = elif n.kind == nkSym: addLocalSym(w, n) + proc trInclude(w: var Writer; n: PNode) = w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) + w.deps.addDotToken # flags + w.deps.addDotToken # type for child in n: assert child.kind == nkStrLit - w.deps.addStrLit child.strVal + w.deps.addStrLit child.strVal # raw string literal, no wrapper needed w.deps.addParRi proc trImport(w: var Writer; n: PNode) = for child in n: if child.kind == nkSym: w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) + w.deps.addDotToken # flags + w.deps.addDotToken # type let s = child.sym assert s.kindImpl == skModule let fp = toFullPath(w.infos.config, s.positionImpl.FileIndex) - w.deps.addStrLit fp + w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi -proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode) = +proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: dest.addDotToken else: @@ -413,94 +549,168 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode) = for child in n: addLocalSyms w, child # Process the child node - writeNode(w, dest, child) + writeNode(w, dest, child, forAst) of nkForStmt, nkTypeDef: # Track for loop variable (first child is the loop variable) w.withNode dest, n: if n.len > 0: addLocalSyms(w, n[0]) for i in 0 ..< n.len: - writeNode(w, dest, n[i]) + writeNode(w, dest, n[i], forAst) of nkFormalParams: # Track parameters (first child is return type, rest are parameters) + inc w.inProc w.withNode dest, n: for i in 0 ..< n.len: if i > 0: # Skip return type addLocalSyms(w, n[i]) - writeNode(w, dest, n[i]) - of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkLambda, nkDo, nkMacroDef: + writeNode(w, dest, n[i], forAst) + dec w.inProc + of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef: + # For top-level named routines (not forAst), just write the symbol. + # The full AST will be stored in the symbol's sdef. + if not forAst and n[namePos].kind == nkSym: + writeSym(w, dest, n[namePos].sym) + else: + # Writing AST inside sdef or anonymous proc: write full structure + inc w.inProc + var ast = n + if n[namePos].kind == nkSym: + ast = n[namePos].sym.astImpl + if ast == nil: ast = n + w.withNode dest, ast: + for i in 0 ..< ast.len: + writeNode(w, dest, ast[i], forAst) + dec w.inProc + of nkLambda, nkDo: + # Lambdas are expressions, always write full structure inc w.inProc - # Entering a proc/function body - parameters are local var ast = n if n[namePos].kind == nkSym: ast = n[namePos].sym.astImpl if ast == nil: ast = n w.withNode dest, ast: - # Process body and other parts for i in 0 ..< ast.len: - writeNode(w, dest, ast[i]) + writeNode(w, dest, ast[i], forAst) dec w.inProc of nkImportStmt: # this has been transformed for us, see `importer.nim` to contain a list of module syms: trImport w, n of nkIncludeStmt: trInclude w, n + of nkExportStmt, nkExportExceptStmt: + # Collect export information for the index + # nkExportStmt children are nkSym nodes + # When exporting a module (export dollars), the module symbol is a child + # followed by all symbols from that module - we use empty set to mean "export all" + # When exporting specific symbols (export foo, bar), we collect their names + # Note: nkExportExceptStmt is transformed to nkExportStmt by semExportExcept, + # but we handle both just in case + var exportAllModules = initHashSet[FileIndex]() + for child in n: + if child.kind == nkSym: + let s = child.sym + if s.kindImpl == skModule: + # Export all from this module - use empty set + let modIdx = s.positionImpl.FileIndex + exportAllModules.incl modIdx + if modIdx notin w.exports: + w.exports[modIdx] = initHashSet[string]() # empty means "export all" + else: + # Export specific symbol, but only if we're not already exporting all from this module + let modIdx = s.itemId.module.FileIndex + if modIdx notin exportAllModules: + if modIdx notin w.exports: + w.exports[modIdx] = initHashSet[string]() + w.exports[modIdx].incl s.name.s + # Write the export statement as a regular node + w.withNode dest, n: + for i in 0 ..< n.len: + writeNode(w, dest, n[i], forAst) else: w.withNode dest, n: for i in 0 ..< n.len: - writeNode(w, dest, n[i]) + writeNode(w, dest, n[i], forAst) -proc writeToplevelNode(w: var Writer; outer, inner: var TokenBuf; n: PNode) = +proc writeToplevelNode(w: var Writer; dest: var TokenBuf; n: PNode) = case n.kind of nkStmtList, nkStmtListExpr: - for son in n: writeToplevelNode(w, outer, inner, son) - of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkLambda, nkDo, nkMacroDef: - # Delegate to `w.topLevel`! - writeNode w, inner, n - of nkConstSection, nkTypeSection, nkTypeDef: - writeNode w, inner, n + for son in n: writeToplevelNode(w, dest, son) else: - writeNode w, outer, n + writeNode w, dest, n proc createStmtList(buf: var TokenBuf; info: PackedLineInfo) {.inline.} = buf.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), info buf.addDotToken # flags buf.addDotToken # type -proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = +proc buildExportBuf(w: var Writer): TokenBuf = + ## Build the export section for the NIF index from collected exports + result = createTokenBuf(32) + for modIdx, names in w.exports: + let path = toFullPath(w.infos.config, modIdx) + if names.len == 0: + # Export all from this module + result.addParLe(TagId(ExportIdx), NoLineInfo) + result.add strToken(pool.strings.getOrIncl(path), NoLineInfo) + result.addParRi() + else: + # Export specific symbols + result.addParLe(TagId(FromexportIdx), NoLineInfo) + result.add strToken(pool.strings.getOrIncl(path), NoLineInfo) + for name in names: + result.add identToken(pool.strings.getOrIncl(name), NoLineInfo) + result.addParRi() + +let replayTag = registerTag("replay") + +proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; + hooks: array[AttachedOp, seq[HookIndexEntry]]; + converters: seq[(nifstreams.SymId, nifstreams.SymId)]; + classes: seq[ClassIndexEntry]; + replayActions: seq[PNode] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) - var outer = createTokenBuf(300) - var inner = createTokenBuf(300) + var content = createTokenBuf(300) let rootInfo = trLineInfo(w, n.info) - createStmtList(outer, rootInfo) - createStmtList(inner, rootInfo) + createStmtList(content, rootInfo) - w.writeToplevelNode outer, inner, n + # Write replay actions first, wrapped in a (replay ...) node + if replayActions.len > 0: + content.addParLe replayTag, rootInfo + for action in replayActions: + writeNode(w, content, action) + content.addParRi() - outer.addParRi() - inner.addParRi() + w.writeToplevelNode content, n - let m = modname(w.moduleToNifSuffix, w.currentModule, w.infos.config) + content.addParRi() + + let m = modname(w.currentModule, w.infos.config) let nifFilename = AbsoluteFile(m).changeFileExt(".nif") let d = completeGeneratedFilePath(config, nifFilename).string var dest = createTokenBuf(600) createStmtList(dest, rootInfo) dest.add w.deps - dest.add outer - dest.add inner + dest.add content dest.addParRi() writeFile(dest, d) - createIndex(d, false, dest[0].info) - # Unload all written types and symbols from memory after the entire module is written - # This handles cyclic references correctly since everything is written before unloading - for typ in w.writtenTypes: - forcePartial(typ) - for sym in w.writtenSyms: - forcePartial(sym) + # Build index with export, hook, converter, and method information + let exportBuf = buildExportBuf(w) + createIndex(d, dest[0].info, false, + IndexSections(hooks: hooks, converters: converters, classes: classes, exportBuf: exportBuf)) + + # Don't unload symbols/types yet - they may be needed by other modules that haven't + # had their NIF files written. For recursive module dependencies (like system.nim), + # we need all NIFs to exist before we can safely unload and reload. + # TODO: Implement deferred unloading at end of compilation for memory savings. + #for typ in w.writtenTypes: + # forcePartial(typ) + #for sym in w.writtenSyms: + # forcePartial(sym) # --------------------------- Loader (lazy!) ----------------------------------------------- @@ -557,19 +767,19 @@ proc loadBool(n: var Cursor): bool = raiseAssert "(true)/(false) expected" type - NifModule = object + NifModule = ref object stream: nifstreams.Stream symCounter: int32 index: NifIndex + suffix: string DecodeContext* = object infos: LineInfoWriter #moduleIds: Table[string, int32] types: Table[ItemId, (PType, NifIndexEntry)] syms: Table[ItemId, (PSym, NifIndexEntry)] - mods: seq[NifModule] + mods: Table[FileIndex, NifModule] cache: IdentCache - moduleToNifSuffix: Table[FileIndex, string] proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable @@ -577,9 +787,8 @@ proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry; buf: var TokenBuf): Cursor = - let s = addr c.mods[module.int32].stream + let s = addr c.mods[module].stream s.r.jumpTo entry.offset - var buf = createTokenBuf(30) nifcursors.parse(s[], buf, entry.info) result = cursorAt(buf, 0) @@ -589,19 +798,22 @@ proc moduleId(c: var DecodeContext; suffix: string): FileIndex = if not isKnownFile: let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.idx.nif")).string - if result.int >= c.mods.len: - c.mods.setLen(result.int + 1) - c.mods[result.int] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile)) + if not fileExists(modFile): + raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile & + ". This can happen when loading a module from NIF that references another module " & + "whose NIF file hasn't been written yet." + c.mods[result] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile), suffix: suffix) proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = - let ii = addr c.mods[module.int32].index + let ii = addr c.mods[module].index result = ii.public.getOrDefault(nifName) if result.offset == 0: result = ii.private.getOrDefault(nifName) if result.offset == 0: raiseAssert "symbol has no offset: " & nifName -proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string): PNode +proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]): PNode proc loadTypeStub(c: var DecodeContext; t: SymId): PType = let name = pool.syms[t] @@ -625,6 +837,41 @@ proc loadTypeStub(c: var DecodeContext; t: SymId): PType = result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) c.types[id] = (result, offs) +proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]) = + ## Scan a tree for local symbol definitions (sdef tags) and add them to localSyms. + ## This doesn't fully load the symbols, just pre-registers them so references + ## can find them. After this proc returns, n is positioned AFTER the tree. + # Handle atoms (non-compound nodes) - just skip them + if n.kind != ParLe: + inc n + return + var depth = 0 + while true: + if n.kind == ParLe: + if n.tagId == sdefTag: + # Found an sdef - check if it's local + let name = n.firstSon + if name.kind == SymbolDef: + let symName = pool.syms[name.symId] + let sn = parseSymName(symName) + if sn.module.len == 0 and symName notin localSyms: + # Local symbol - create a stub entry in localSyms + let module = moduleId(c, thisModule) + let val = addr c.mods[module].symCounter + inc val[] + let id = ItemId(module: module.int32, item: val[]) + let sym = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), + disamb: sn.count.int32, state: Complete) + localSyms[symName] = sym + inc depth + elif n.kind == ParRi: + dec depth + if depth == 0: + inc n # Move PAST the closing ) + break + inc n + proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = if n.kind == DotToken: result = nil @@ -640,11 +887,40 @@ proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = else: raiseAssert "type expected but got " & $n.kind -proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string): PSym = +proc loadTypeStubWithLocalSyms(c: var DecodeContext; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]): PType = + ## Like loadTypeStub but also extracts local symbols from inline type definitions + if n.kind == DotToken: + result = nil + inc n + elif n.kind == Symbol: + let s = n.symId + result = loadTypeStub(c, s) + inc n + elif n.kind == ParLe and n.tagId == tdefTag: + # First extract local symbols from the inline type + let s = n.firstSon.symId + extractLocalSymsFromTree(c, n, thisModule, localSyms) + result = loadTypeStub(c, s) + else: + raiseAssert "type expected but got " & $n.kind + +proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string; + localSyms: var Table[string, PSym]): PSym = let symAsStr = pool.syms[t] let sn = parseSymName(symAsStr) - let module = moduleId(c, if sn.module.len > 0: sn.module else: thisModule) - let val = addr c.mods[module.int32].symCounter + # For local symbols (no module suffix), they MUST be in localSyms. + # Local symbols are not in the index - they're defined inline in the NIF file. + # If not found, it's a bug in how we populate localSyms. + if sn.module.len == 0: + result = localSyms.getOrDefault(symAsStr) + if result != nil: + return result + else: + raiseAssert "local symbol '" & symAsStr & "' not found in localSyms." + # Global symbol - look up in index for lazy loading + let module = moduleId(c, sn.module) + let val = addr c.mods[module].symCounter inc val[] let id = ItemId(module: module.int32, item: val[]) @@ -653,20 +929,20 @@ proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string): PSym = let offs = c.getOffset(module, symAsStr) result = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[id] = (result, offs) - c.moduleToNifSuffix[module] = (if sn.module.len > 0: sn.module else: thisModule) -proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string): PSym = +proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]): PSym = if n.kind == DotToken: result = nil inc n elif n.kind == Symbol: let s = n.symId - result = loadSymStub(c, s, thisModule) + result = loadSymStub(c, s, thisModule, localSyms) inc n elif n.kind == ParLe and n.tagId == sdefTag: let s = n.firstSon.symId skip n - result = loadSymStub(c, s, thisModule) + result = loadSymStub(c, s, thisModule, localSyms) else: raiseAssert "sym expected but got " & $n.kind @@ -719,10 +995,18 @@ proc loadType*(c: var DecodeContext; t: PType) = expect n, ParLe if n.tagId != tdefTag: raiseAssert "(td) expected" - inc n + + # Pre-scan the ENTIRE type definition for local symbol definitions (sdefs). + # We need to do this before loading any fields, because local symbols may be + # defined anywhere in the type and referenced anywhere else. + var localSyms = initTable[string, PSym]() + var scanCursor = n # copy cursor at start of type + let typesModule = parseSymName(pool.syms[n.firstSon.symId]).module + extractLocalSymsFromTree(c, scanCursor, typesModule, localSyms) + + inc n # move past (td expect n, SymbolDef # ignore the type's name, we have already used it to create this PType's itemId! - let typesModule = parseSymName(pool.syms[n.symId]).module inc n #loadField t.kind loadField t.flagsImpl @@ -733,9 +1017,9 @@ proc loadType*(c: var DecodeContext; t: PType) = loadField t.itemId.item # nonUniqueId t.typeInstImpl = loadTypeStub(c, n) - t.nImpl = loadNode(c, n, typesModule) - t.ownerFieldImpl = loadSymStub(c, n, typesModule) - t.symImpl = loadSymStub(c, n, typesModule) + t.nImpl = loadNode(c, n, typesModule, localSyms) + t.ownerFieldImpl = loadSymStub(c, n, typesModule, localSyms) + t.symImpl = loadSymStub(c, n, typesModule, localSyms) loadLoc c, n, t.locImpl while n.kind != ParRi: @@ -743,7 +1027,7 @@ proc loadType*(c: var DecodeContext; t: PType) = skipParRi n -proc loadAnnex(c: var DecodeContext; n: var Cursor; thisModule: string): PLib = +proc loadAnnex(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PLib = if n.kind == DotToken: result = nil inc n @@ -755,22 +1039,15 @@ proc loadAnnex(c: var DecodeContext; n: var Cursor; thisModule: string): PLib = expect n, StringLit result.name = pool.strings[n.litId] inc n - result.path = loadNode(c, n, thisModule) + result.path = loadNode(c, n, thisModule, localSyms) skipParRi n else: raiseAssert "`lib/annex` information expected" -proc loadSym*(c: var DecodeContext; s: PSym) = - if s.state != Partial: return - s.state = Sealed - var buf = createTokenBuf(30) - let symsModule = s.itemId.module.FileIndex - var n = cursorFromIndexEntry(c, symsModule, c.syms[s.itemId][1], buf) - - expect n, ParLe - if n.tagId != sdefTag: - raiseAssert "(sd) expected" - inc n +proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]) = + ## Loads a symbol definition from the current cursor position. + ## The cursor should be positioned after the opening (sd tag. expect n, SymbolDef # ignore the symbol's name, we have already used it to create this PSym instance! inc n @@ -785,38 +1062,69 @@ proc loadSym*(c: var DecodeContext; s: PSym) = else: raiseAssert "expected `x` or '.' but got " & $n.kind - loadField s.magicImpl - loadField s.flagsImpl - loadField s.optionsImpl - loadField s.offsetImpl - expect n, ParLe - s.kindImpl = parse(TSymKind, pool.tags[n.tagId]) + {.cast(uncheckedAssign).}: + s.kindImpl = parse(TSymKind, pool.tags[n.tagId]) inc n case s.kindImpl of skLet, skVar, skField, skForVar: - s.guardImpl = loadSymStub(c, n, c.moduleToNifSuffix[symsModule]) + s.guardImpl = loadSymStub(c, n, thisModule, localSyms) loadField s.bitsizeImpl loadField s.alignmentImpl else: discard skipParRi n + loadField s.magicImpl + loadField s.flagsImpl + loadField s.optionsImpl + loadField s.offsetImpl + if s.kindImpl == skModule: expect n, DotToken inc n + var isKnownFile = false + s.positionImpl = int c.infos.config.registerNifSuffix(thisModule, isKnownFile) else: loadField s.positionImpl - s.typImpl = loadTypeStub(c, n) - s.ownerFieldImpl = loadSymStub(c, n, c.moduleToNifSuffix[symsModule]) - # We do not store `sym.ast` here but instead set it in the deserializer - #writeNode(w, sym.ast) + + # For routine symbols, pre-scan the type to find local symbol definitions + # (generic params, params). These sdefs are written inline in the type. + if s.kindImpl in routineKinds: + s.typImpl = loadTypeStubWithLocalSyms(c, n, thisModule, localSyms) + else: + s.typImpl = loadTypeStub(c, n) + s.ownerFieldImpl = loadSymStub(c, n, thisModule, localSyms) + # Load the AST for routine symbols (procs, funcs, etc.) + if s.kindImpl in routineKinds: + s.astImpl = loadNode(c, n, thisModule, localSyms) + elif n.kind == DotToken: + inc n + else: + raiseAssert "expected '.' for non-routine symbol AST but got " & $n.kind loadLoc c, n, s.locImpl - s.constraintImpl = loadNode(c, n, c.moduleToNifSuffix[symsModule]) - s.instantiatedFromImpl = loadSymStub(c, n, c.moduleToNifSuffix[symsModule]) + s.constraintImpl = loadNode(c, n, thisModule, localSyms) + s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms) skipParRi n +proc loadSym*(c: var DecodeContext; s: PSym) = + if s.state != Partial: return + s.state = Sealed + var buf = createTokenBuf(30) + let symsModule = s.itemId.module.FileIndex + var n = cursorFromIndexEntry(c, symsModule, c.syms[s.itemId][1], buf) + + expect n, ParLe + if n.tagId != sdefTag: + raiseAssert "(sd) expected" + # Extract line info from the sdef tag before moving past it + s.infoImpl = c.infos.oldLineInfo(n.info) + inc n + # Create localSyms for any local symbols encountered in the AST + var localSyms = initTable[string, PSym]() + loadSymFromCursor(c, s, n, c.mods[symsModule].suffix, localSyms) + template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) @@ -828,15 +1136,26 @@ template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNod body skipParRi n -proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string): PNode = +proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]): PNode = result = nil case n.kind of Symbol: let info = c.infos.oldLineInfo(n.info) - result = newSymNode(c.loadSymStub(n, thisModule), info) + let symName = pool.syms[n.symId] + # Check local symbols first + let localSym = localSyms.getOrDefault(symName) + if localSym != nil: + result = newSymNode(localSym, info) + inc n + else: + result = newSymNode(c.loadSymStub(n, thisModule, localSyms), info) of DotToken: result = nil inc n + of StringLit: + result = newStrNode(pool.strings[n.litId], c.infos.oldLineInfo(n.info)) + inc n of ParLe: let kind = n.nodeKind case kind @@ -847,14 +1166,39 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string): PNode = inc n let typ = c.loadTypeStub n let info = c.infos.oldLineInfo(n.info) - result = newSymNode(c.loadSymStub(n, thisModule), info) + result = newSymNode(c.loadSymStub(n, thisModule, localSyms), info) result.typField = typ skipParRi n of symDefTagName: + let info = c.infos.oldLineInfo(n.info) let name = n.firstSon assert name.kind == SymbolDef - result = newSymNode(c.loadSymStub(name.symId, thisModule), c.infos.oldLineInfo(n.info)) - skip n + let symName = pool.syms[name.symId] + # Check if this is a local symbol (no module suffix in name) + let sn = parseSymName(symName) + let isLocal = sn.module.len == 0 + var sym: PSym + if isLocal: + # Local symbol - not in the index, defined inline in NIF. + # Check if we already have a stub from extractLocalSymsFromType + sym = localSyms.getOrDefault(symName) + if sym == nil: + # First time seeing this local symbol - create it + let module = moduleId(c, thisModule) + let val = addr c.mods[module].symCounter + inc val[] + let id = ItemId(module: module.int32, item: val[]) + sym = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), + disamb: sn.count.int32, state: Complete) + localSyms[symName] = sym # register for later references + # Now fully load the symbol from the sdef + inc n # skip `sd` tag + loadSymFromCursor(c, sym, n, thisModule, localSyms) + sym.state = Sealed # mark as fully loaded + else: + sym = c.loadSymStub(name.symId, thisModule, localSyms) + skip n # skip the entire sdef for indexed symbols + result = newSymNode(sym, info) of typeDefTagName: raiseAssert "`td` tag in invalid context" of "none": @@ -928,27 +1272,50 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string): PNode = else: c.withNode n, result, kind: while n.kind != ParRi: - result.sons.add c.loadNode(n, thisModule) + result.sons.add c.loadNode(n, thisModule, localSyms) else: - raiseAssert "Not yet implemented " & $n.kind + raiseAssert "expected string literal but got " & $n.kind proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = - moduleSuffix(toFullPath(conf, f), cast[seq[string]](conf.searchPaths)) + cachedModuleSuffix(conf, f) proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; nifName: string; entry: NifIndexEntry; thisModule: string): PSym = - ## Loads a symbol from the NIF index entry. - ## Creates a symbol stub and loads its full definition. - result = loadSymStub(c, pool.syms.getOrIncl nifName, thisModule) + ## Loads a symbol from the NIF index entry using the entry directly. + ## Creates a symbol stub without looking up in the index (since the index may be moved out). + let symAsStr = nifName + let sn = parseSymName(symAsStr) + let symModule = moduleId(c, if sn.module.len > 0: sn.module else: thisModule) + let val = addr c.mods[symModule].symCounter + inc val[] + + let id = ItemId(module: symModule.int32, item: val[]) + result = c.syms.getOrDefault(id)[0] + if result == nil: + # Use the entry directly instead of looking it up in the index + result = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) + c.syms[id] = (result, entry) + +proc extractBasename(nifName: string): string = + ## Extract the base name from a NIF name (ident.disamb.module -> ident) + result = "" + for c in nifName: + if c == '.': break + result.add c proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; interf, interfHidden: var TStrTable; thisModule: string) = ## Populates interface tables from the NIF index structure. ## Uses the index's public/private tables instead of traversing AST. - let idx = addr c.mods[module.int32].index + + # Move the public table and exports list out to avoid iterator invalidation + # (moduleId can add to c.mods which would invalidate Table iterators) + # We move them back after iteration. + var publicTab = move c.mods[module].index.public + var exportsList = move c.mods[module].index.exports # Add all public symbols to interf (exported interface) and interfHidden - for nifName, entry in idx.public: + for nifName, entry in publicTab: if not nifName.startsWith("`t"): # do not load types, they are not part of an interface but an implementation detail! #echo "LOADING SYM ", nifName, " ", entry.offset @@ -957,6 +1324,49 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; strTableAdd(interf, sym) strTableAdd(interfHidden, sym) + # Move public table back + c.mods[module].index.public = move publicTab + + # Process exports (re-exports from other modules) + for exp in exportsList: + let (path, kind, names) = exp + # Convert path to module suffix + let expSuffix = moduleSuffix(path, cast[seq[string]](c.infos.config.searchPaths)) + # Load the exported module's index + let expModule = moduleId(c, expSuffix) + + # Move the exported module's public table out to avoid iterator invalidation + var expPublicTab = move c.mods[expModule].index.public + + # Build a set of names for filtering + var nameSet = initHashSet[string]() + for nameId in names: + nameSet.incl pool.strings[nameId] + + # Add symbols based on export kind + for nifName, entry in expPublicTab: + if nifName.startsWith("`t"): + continue # skip types + + let basename = extractBasename(nifName) + let shouldInclude = case kind + of ExportIdx: true # export all + of FromexportIdx: basename in nameSet # only specific names + of ExportexceptIdx: basename notin nameSet # all except specific names + else: false + + if shouldInclude: + let sym = loadSymFromIndexEntry(c, expModule, nifName, entry, expSuffix) + if sym != nil: + strTableAdd(interf, sym) + strTableAdd(interfHidden, sym) + + # Move exported module's public table back + c.mods[expModule].index.public = move expPublicTab + + # Move exports list back + c.mods[module].index.exports = move exportsList + when false: # Add private symbols to interfHidden only for nifName, entry in idx.private: @@ -972,27 +1382,149 @@ proc toNifIndexFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.idx.nif").string -proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable): PNode = +proc parseTypeSymIdToItemId*(c: var DecodeContext; symId: nifstreams.SymId): ItemId = + ## Parses a type SymId (format: "`tN.modulesuffix") to extract ItemId. + let s = pool.syms[symId] + if not s.startsWith("`t"): + return ItemId(module: -1, item: 0) + var i = 2 # skip "`t" + var item = 0'i32 + while i < s.len and s[i] in {'0'..'9'}: + item = item * 10 + int32(ord(s[i]) - ord('0')) + inc i + if i < s.len and s[i] == '.': + inc i + let suffix = s.substr(i) + let module = moduleId(c, suffix) + result = ItemId(module: int32(module), item: item) + else: + result = ItemId(module: -1, item: item) + +proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym = + ## Resolves a hook SymId to PSym. + let symAsStr = pool.syms[symId] + let sn = parseSymName(symAsStr) + if sn.module.len == 0: + return nil # Local symbols shouldn't be hooks + let module = moduleId(c, sn.module) + # Look up the symbol in the module's index + let offs = c.mods[module].index.public.getOrDefault(symAsStr) + if offs.offset == 0: + return nil + # Create a stub symbol + let val = addr c.mods[module].symCounter + inc val[] + let id = ItemId(module: int32(module), item: val[]) + result = c.syms.getOrDefault(id)[0] + if result == nil: + result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(sn.name), + disamb: sn.count.int32, state: Partial) + c.syms[id] = (result, offs) + +proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym = + ## Tries to resolve a compiler proc from a module by checking the NIF index. + ## Returns nil if the symbol doesn't exist. + let suffix = moduleSuffix(c.infos.config, moduleFileIdx) + let symName = name & ".0." & suffix + + # Check if module index is loaded, if not load it + let module = moduleId(c, suffix) + + # Check if symbol exists in the index (check both public and private) + var offs = c.mods[module].index.public.getOrDefault(symName) + if offs.offset == 0: + offs = c.mods[module].index.private.getOrDefault(symName) + if offs.offset == 0: + return nil + + # Create a stub symbol + let val = addr c.mods[module].symCounter + inc val[] + let id = ItemId(module: int32(module), item: val[]) + result = c.syms.getOrDefault(id)[0] + if result == nil: + result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(name), + disamb: 0, state: Partial) + c.syms[id] = (result, offs) + +proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; + hooks: var Table[nifstreams.SymId, HooksPerType]; + converters: var seq[(string, string)]; + classes: var seq[ClassIndexEntry]; + loadFullAst: bool = false): PNode = let suffix = moduleSuffix(c.infos.config, f) - let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(suffix), ".nif").string # Ensure module index is loaded - moduleId returns the FileIndex for this suffix let module = moduleId(c, suffix) # Populate interface tables from the NIF index structure - # Use the FileIndex returned by moduleId to ensure we access the correct index + # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym populateInterfaceTablesFromIndex(c, module, interf, interfHidden, suffix) - var buf = createTokenBuf(300) - var s = nifstreams.open(modFile) + # Return hooks from the index + hooks = move c.mods[module].index.hooks + # Return converters from the index + converters = move c.mods[module].index.converters + # Return classes/methods from the index + classes = move c.mods[module].index.classes + + # Load the module AST (or just replay actions if loadFullAst is false) + result = newNode(nkStmtList) + let s = addr c.mods[module].stream + s.r.jumpTo 0 # Start from beginning discard processDirectives(s.r) - # XXX We can optimize this here and only load the top level entries! - try: - nifcursors.parse(s, buf, NoLineInfo) - finally: - nifstreams.close(s) - var n = cursorAt(buf, 0) - result = loadNode(c, n, suffix) + var localSyms = initTable[string, PSym]() + var t = next(s[]) + if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): + t = next(s[]) # skip flags + t = next(s[]) # skip type + # Process all top-level statements + while t.kind != ParRi and t.kind != EofToken: + if t.kind == ParLe: + let tag = pool.tags[t.tagId] + if tag == "replay": + # Always load replay actions (macro cache operations) + t = next(s[]) # move past (replay + while t.kind != ParRi and t.kind != EofToken: + if t.kind == ParLe: + var buf = createTokenBuf(50) + nifcursors.parse(s[], buf, t.info) + var cursor = cursorAt(buf, 0) + let replayNode = loadNode(c, cursor, suffix, localSyms) + if replayNode != nil: + result.sons.add replayNode + t = next(s[]) + elif loadFullAst: + # Parse the full statement + var buf = createTokenBuf(50) + buf.add t # Add the ParLe token we already read + var nested = 1 + while nested > 0: + t = next(s[]) + buf.add t + if t.kind == ParLe: + inc nested + elif t.kind == ParRi: + dec nested + elif t.kind == EofToken: + break + var cursor = cursorAt(buf, 0) + let stmtNode = loadNode(c, cursor, suffix, localSyms) + if stmtNode != nil: + result.sons.add stmtNode + else: + # Skip over the statement by counting parentheses + var nested = 1 + while nested > 0: + t = next(s[]) + if t.kind == ParLe: + inc nested + elif t.kind == ParRi: + dec nested + elif t.kind == EofToken: + break + else: + t = next(s[]) when isMainModule: import std / syncio diff --git a/compiler/astdef.nim b/compiler/astdef.nim index cc0c4c49a6..ffd02f3a96 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -981,6 +981,14 @@ proc newSymNode*(sym: PSym, info: TLineInfo): PNode = result.typField = sym.typImpl result.info = info +proc newStrNode*(kind: TNodeKind, strVal: string): PNode = + result = newNode(kind) + result.strVal = strVal + +proc newStrNode*(strVal: string; info: TLineInfo): PNode = + result = newNodeI(nkStrLit, info) + result.strVal = strVal + proc forcePartial*(s: PSym) = ## Resets all impl-fields to their default values and sets state to Partial. ## This is useful for creating a stub symbol that can be lazily loaded later. diff --git a/compiler/ccgliterals.nim b/compiler/ccgliterals.nim index 84f72017cf..069ed48df7 100644 --- a/compiler/ccgliterals.nim +++ b/compiler/ccgliterals.nim @@ -16,13 +16,10 @@ ## implementation. template detectVersion(field, corename) = - if m.g.field == 0: - let core = getCompilerProc(m.g.graph, corename) - if core == nil or core.kind != skConst: - m.g.field = 1 - else: - m.g.field = toInt(ast.getInt(core.astdef)) - result = m.g.field + if m.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc, gcHooks}: + result = 2 + else: + result = 1 proc detectStrVersion(m: BModule): int = detectVersion(strVersion, "nimStrVersion") diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 76669d41ba..a2b5e32cb8 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -721,7 +721,7 @@ proc genRecordFieldsAux(m: BModule; n: PNode, # have to recurse via 'getTypeDescAux'. And not doing so prevents problems # with heavily templatized C++ code: if not isImportedCppType(rectype): - let fieldType = field.loc.lode.typ.skipTypes(abstractInst) + let fieldType = field.loc.t.skipTypes(abstractInst) var typ: Rope = "" var isFlexArray = false var initializer = "" diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 946a144321..59bddeae19 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -1307,16 +1307,15 @@ proc countStateOccurences(ctx: var Ctx, n: PNode, stateOccurences: var openArray proc replaceDeletedStates(ctx: var Ctx, n: PNode): PNode = result = n - for i in 0 ..< n.safeLen: - let c = n[i] - if c.kind == nkIntLit: - let idx = c.intVal - if idx >= 0 and idx < ctx.states.len and ctx.states[idx].label == c and ctx.states[idx].deletable: - let gt = ctx.replaceDeletedStates(skipStmtList(ctx.states[idx].body)) - assert(gt.kind == nkGotoState) - n[i] = gt[0] - else: - n[i] = ctx.replaceDeletedStates(c) + if n.kind == nkIntLit: + let idx = n.intVal + if idx >= 0 and idx < ctx.states.len and ctx.states[idx].label == n and ctx.states[idx].deletable: + let gt = ctx.replaceDeletedStates(skipStmtList(ctx.states[idx].body)) + assert(gt.kind == nkGotoState) + result = gt[0] + else: + for i in 0 ..< n.safeLen: + n[i] = ctx.replaceDeletedStates(n[i]) proc replaceInlinedStates(ctx: var Ctx, n: PNode): PNode = ## Find all nkGotoState(stateIdx) nodes that do not follow nkYield. @@ -1347,6 +1346,7 @@ proc optimizeStates(ctx: var Ctx) = # Replace deletable state labels to labels of respective non-empty states for i in 0 .. ctx.states.high: ctx.states[i].body = ctx.replaceDeletedStates(ctx.states[i].body) + ctx.states[i].excLandingState = ctx.replaceDeletedStates(ctx.states[i].excLandingState) # Remove deletable states var i = 0 diff --git a/compiler/commands.nim b/compiler/commands.nim index 415fe6b352..f782c6dc3d 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -498,6 +498,8 @@ proc parseCommand*(command: string): Command = of "secret": cmdInteractive of "nop", "help": cmdNop of "jsonscript": cmdJsonscript + of "nifc": cmdNifC # generate C from NIF files + of "deps": cmdDeps # generate .build.nif for nifmake else: cmdUnknown proc setCmd*(conf: ConfigRef, cmd: Command) = @@ -510,6 +512,12 @@ proc setCmd*(conf: ConfigRef, cmd: Command) = of cmdCompileToOC: conf.backend = backendObjc of cmdCompileToJS: conf.backend = backendJs of cmdCompileToNif: conf.backend = backendNif + of cmdNifC: + conf.backend = backendC # NIF to C compilation + conf.globalOptions.incl optCompress # enable NIF loading + of cmdM: + # cmdM requires optCompress for proper IC handling (include files, etc.) + conf.globalOptions.incl optCompress else: discard proc setCommandEarly*(conf: ConfigRef, command: string) = diff --git a/compiler/deps.nim b/compiler/deps.nim new file mode 100644 index 0000000000..255cd3e80f --- /dev/null +++ b/compiler/deps.nim @@ -0,0 +1,340 @@ +# +# +# The Nim Compiler +# (c) Copyright 2025 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Generate a .build.nif file for nifmake from a Nim project. +## This enables incremental and parallel compilation using the `m` switch. + +import std / [os, tables, sets, times, osproc, strutils] +import options, msgs, pathutils, lineinfos + +import "../dist/nimony/src/lib" / [nifstreams, nifcursors, bitabs, nifreader, nifbuilder] +import "../dist/nimony/src/gear2" / modnames + +type + FilePair = object + nimFile: string + modname: string + + Node = ref object + files: seq[FilePair] # main file + includes + deps: seq[int] # indices into DepContext.nodes + id: int + + DepContext = object + config: ConfigRef + nifler: string + nodes: seq[Node] + processedModules: Table[string, int] # modname -> node index + includeStack: seq[string] + +proc toPair(c: DepContext; f: string): FilePair = + FilePair(nimFile: f, modname: moduleSuffix(f, cast[seq[string]](c.config.searchPaths))) + +proc depsFile(c: DepContext; f: FilePair): string = + getNimcacheDir(c.config).string / f.modname & ".deps.nif" + +proc parsedFile(c: DepContext; f: FilePair): string = + getNimcacheDir(c.config).string / f.modname & ".p.nif" + +proc semmedFile(c: DepContext; f: FilePair): string = + getNimcacheDir(c.config).string / f.modname & ".nif" + +proc findNifler(): string = + # Look for nifler in common locations + result = findExe("nifler") + if result.len == 0: + # Try relative to nim executable + let nimDir = getAppDir() + result = nimDir / "nifler" + if not fileExists(result): + result = nimDir / ".." / "nimony" / "bin" / "nifler" + if not fileExists(result): + result = "" + +proc runNifler(c: DepContext; nimFile: string): bool = + ## Run nifler deps on a file if needed. Returns true on success. + let pair = c.toPair(nimFile) + let depsPath = c.depsFile(pair) + + # Check if deps file is up-to-date + if fileExists(depsPath) and fileExists(nimFile): + if getLastModificationTime(depsPath) > getLastModificationTime(nimFile): + return true # Already up-to-date + + # Create output directory if needed + createDir(parentDir(depsPath)) + + # Run nifler deps + let cmd = quoteShell(c.nifler) & " deps " & quoteShell(nimFile) & " " & quoteShell(depsPath) + let exitCode = execShellCmd(cmd) + result = exitCode == 0 + +proc resolveFile(c: DepContext; origin, toResolve: string): string = + ## Resolve an import path relative to origin file + # Handle std/ prefix + var path = toResolve + if path.startsWith("std/"): + path = path.substr(4) + + # Try relative to origin first + let originDir = parentDir(origin) + result = originDir / path.addFileExt("nim") + if fileExists(result): + return result + + # Try search paths + for searchPath in c.config.searchPaths: + result = searchPath.string / path.addFileExt("nim") + if fileExists(result): + return result + + result = "" + +proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) + +proc processInclude(c: var DepContext; includePath: string; current: Node) = + let resolved = resolveFile(c, current.files[current.files.len - 1].nimFile, includePath) + if resolved.len == 0 or not fileExists(resolved): + return + + # Check for recursive includes + for s in c.includeStack: + if s == resolved: + return # Skip recursive include + + c.includeStack.add resolved + current.files.add c.toPair(resolved) + traverseDeps(c, c.toPair(resolved), current) + discard c.includeStack.pop() + +proc processImport(c: var DepContext; importPath: string; current: Node) = + let resolved = resolveFile(c, current.files[0].nimFile, importPath) + if resolved.len == 0 or not fileExists(resolved): + return + + let pair = c.toPair(resolved) + let existingIdx = c.processedModules.getOrDefault(pair.modname, -1) + + if existingIdx == -1: + # New module - create node and process it + let newNode = Node(files: @[pair], id: c.nodes.len) + current.deps.add newNode.id + c.processedModules[pair.modname] = newNode.id + c.nodes.add newNode + traverseDeps(c, pair, newNode) + else: + # Already processed - just add dependency + if existingIdx notin current.deps: + current.deps.add existingIdx + +proc readDepsFile(c: var DepContext; pair: FilePair; current: Node) = + ## Read a .deps.nif file and process imports/includes + let depsPath = c.depsFile(pair) + if not fileExists(depsPath): + return + + var s = nifstreams.open(depsPath) + defer: nifstreams.close(s) + discard processDirectives(s.r) + + var t = next(s) + if t.kind != ParLe: + return + + # Skip to content (past stmts tag) + t = next(s) + + while t.kind != EofToken: + if t.kind == ParLe: + let tag = pool.tags[t.tagId] + case tag + of "import", "fromimport": + # Read import path + t = next(s) + # Check for "when" marker (conditional import) + if t.kind == Ident and pool.strings[t.litId] == "when": + t = next(s) # skip it, still process the import + # Handle path expression (could be ident, string, or infix like std/foo) + var importPath = "" + if t.kind == Ident: + importPath = pool.strings[t.litId] + elif t.kind == StringLit: + importPath = pool.strings[t.litId] + elif t.kind == ParLe and pool.tags[t.tagId] == "infix": + # Handle std / foo style imports + t = next(s) # skip infix tag + if t.kind == Ident: # operator (/) + t = next(s) + if t.kind == Ident: # first part (std) + importPath = pool.strings[t.litId] + t = next(s) + if t.kind == Ident: # second part (foo) + importPath = importPath & "/" & pool.strings[t.litId] + if importPath.len > 0: + processImport(c, importPath, current) + # Skip to end of import node + var depth = 1 + while depth > 0: + t = next(s) + if t.kind == ParLe: inc depth + elif t.kind == ParRi: dec depth + of "include": + # Read include path + t = next(s) + if t.kind == Ident and pool.strings[t.litId] == "when": + t = next(s) # skip conditional marker + var includePath = "" + if t.kind == Ident: + includePath = pool.strings[t.litId] + elif t.kind == StringLit: + includePath = pool.strings[t.litId] + if includePath.len > 0: + processInclude(c, includePath, current) + # Skip to end + var depth = 1 + while depth > 0: + t = next(s) + if t.kind == ParLe: inc depth + elif t.kind == ParRi: dec depth + else: + # Skip unknown node + var depth = 1 + while depth > 0: + t = next(s) + if t.kind == ParLe: inc depth + elif t.kind == ParRi: dec depth + t = next(s) + +proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) = + ## Process a module: run nifler and read deps + if not runNifler(c, pair.nimFile): + rawMessage(c.config, errGenerated, "nifler failed for: " & pair.nimFile) + return + readDepsFile(c, pair, current) + +proc generateBuildFile(c: DepContext): string = + ## Generate the .build.nif file for nifmake + result = getNimcacheDir(c.config).string / c.nodes[0].files[0].modname & ".build.nif" + + var b = nifbuilder.open(result) + defer: b.close() + + b.addHeader("nim deps", "nifmake") + b.addTree "stmts" + + # Define nifler command + b.addTree "cmd" + b.addSymbolDef "nifler" + b.addStrLit c.nifler + b.addStrLit "parse" + b.addStrLit "--deps" + b.addTree "input" + b.endTree() + b.addTree "output" + b.endTree() + b.endTree() + + # Define nim m command + b.addTree "cmd" + b.addSymbolDef "nim_m" + b.addStrLit getAppFilename() + b.addStrLit "m" + # Add search paths + for p in c.config.searchPaths: + b.addStrLit "--path:" & p.string + b.addTree "input" + b.addIntLit 0 + b.endTree() + b.endTree() + + # Build rules for parsing (nifler) + var seenFiles = initHashSet[string]() + for node in c.nodes: + for pair in node.files: + let parsed = c.parsedFile(pair) + if not seenFiles.containsOrIncl(parsed): + b.addTree "do" + b.addIdent "nifler" + b.addTree "input" + b.addStrLit pair.nimFile + b.endTree() + b.addTree "output" + b.addStrLit parsed + b.endTree() + b.addTree "output" + b.addStrLit c.depsFile(pair) + b.endTree() + b.endTree() + + # Build rules for semantic checking (nim m) + for i in countdown(c.nodes.len - 1, 0): + let node = c.nodes[i] + let pair = node.files[0] + b.addTree "do" + b.addIdent "nim_m" + # Input: all parsed files for this module + for f in node.files: + b.addTree "input" + b.addStrLit c.parsedFile(f) + b.endTree() + # Also depend on semmed files of dependencies + for depIdx in node.deps: + b.addTree "input" + b.addStrLit c.semmedFile(c.nodes[depIdx].files[0]) + b.endTree() + # Output: semmed file + b.addTree "output" + b.addStrLit c.semmedFile(pair) + b.endTree() + b.addTree "args" + b.addStrLit pair.nimFile + b.endTree() + b.endTree() + + b.endTree() # stmts + +proc commandDeps*(conf: ConfigRef) = + ## Main entry point for `nim deps` + when not defined(nimKochBootstrap): + let nifler = findNifler() + if nifler.len == 0: + rawMessage(conf, errGenerated, "nifler tool not found. Install nimony or add nifler to PATH.") + return + + let projectFile = conf.projectFull.string + if not fileExists(projectFile): + rawMessage(conf, errGenerated, "project file not found: " & projectFile) + return + + # Create nimcache directory + createDir(getNimcacheDir(conf).string) + + var c = DepContext( + config: conf, + nifler: nifler, + nodes: @[], + processedModules: initTable[string, int](), + includeStack: @[] + ) + + # Create root node for main project file + let rootPair = c.toPair(projectFile) + let rootNode = Node(files: @[rootPair], id: 0) + c.nodes.add rootNode + c.processedModules[rootPair.modname] = 0 + + # Process dependencies + traverseDeps(c, rootPair, rootNode) + + # Generate build file + let buildFile = generateBuildFile(c) + rawMessage(conf, hintSuccess, "generated: " & buildFile) + rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile) + else: + rawMessage(conf, errGenerated, "nim deps not available in bootstrap build") diff --git a/compiler/liftdestructors.nim b/compiler/liftdestructors.nim index 92655cf49c..ef0920d180 100644 --- a/compiler/liftdestructors.nim +++ b/compiler/liftdestructors.nim @@ -1212,8 +1212,10 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; result.ast[bodyPos].add newAsgnStmt(d, src) else: var tk: TTypeKind + var skipped: PType = nil if g.config.selectedGC in {gcArc, gcOrc, gcHooks, gcAtomicArc}: - tk = skipTypes(typ, {tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyAlias, tySink}).kind + skipped = skipTypes(typ, {tyOrdinal, tyRange, tyInferred, tyGenericInst, tyStatic, tyAlias, tySink}) + tk = skipped.kind else: tk = tyNone # no special casing for strings and seqs case tk @@ -1223,7 +1225,7 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp; fillStrOp(a, typ, result.ast[bodyPos], d, src) else: fillBody(a, typ, result.ast[bodyPos], d, src) - if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy, attachedDup} and not isObjLackingTypeField(typ): + if tk == tyObject and a.kind in {attachedAsgn, attachedSink, attachedDeepCopy, attachedDup} and not isObjLackingTypeField(skipped): # bug #19205: Do not forget to also copy the hidden type field: genTypeFieldCopy(a, typ, result.ast[bodyPos], d, src) diff --git a/compiler/lineinfos.nim b/compiler/lineinfos.nim index 292a02e60e..397d407077 100644 --- a/compiler/lineinfos.nim +++ b/compiler/lineinfos.nim @@ -273,6 +273,10 @@ const errFloatToString* = "cannot convert '$1' to '$2'" type + FileInfoKind* = enum + fikSource, ## A real source file path + fikNifModule ## A NIF module suffix (not a real path) + TFileInfo* = object fullPath*: AbsoluteFile # This is a canonical full filesystem path projPath*: RelativeFile # This is relative to the project's root @@ -291,6 +295,7 @@ type # for 'nimsuggest' hash*: string # the checksum of the file dirty*: bool # for 'nimpretty' like tooling + kind*: FileInfoKind # distinguishes real files from NIF suffixes when defined(nimpretty): fullContent*: string FileIndex* = distinct int32 diff --git a/compiler/main.nim b/compiler/main.nim index 377c85b6e1..d63c7d3fbf 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -32,6 +32,10 @@ import ../dist/checksums/src/checksums/sha1 import pipelines +when not defined(nimKochBootstrap): + import nifbackend + import deps + when not defined(leanCompiler): import docgen @@ -133,6 +137,22 @@ proc commandCompileToNif(graph: ModuleGraph) = setPipeLinePass(graph, NifgenPass) compilePipelineProject(graph) +proc commandNifC(graph: ModuleGraph) = + ## Generate C code from precompiled NIF files. + ## This is the new IC approach: compile modules to NIF first with `nim m`, + ## then generate C code from the entry.nif file with whole-program DCE. + when not defined(nimKochBootstrap): + let conf = graph.config + extccomp.initVars(conf) + + if not extccomp.ccHasSaneOverflow(conf): + conf.symbols.defineSymbol("nimEmulateOverflowChecks") + + # Use the NIF backend to generate C code + nifbackend.generateCode(graph, conf.projectMainIdx) + else: + rawMessage(graph.config, errGenerated, "NIF backend not available during bootstrap build") + proc commandCompileToC(graph: ModuleGraph) = let conf = graph.config extccomp.initVars(conf) @@ -420,9 +440,22 @@ proc mainCommand*(graph: ModuleGraph) = of cmdCheck: commandCheck(graph) of cmdM: - graph.config.symbolFiles = v2Sf - setUseIc(graph.config.symbolFiles != disabledSf) + # cmdM uses NIF files, not ROD files + graph.config.symbolFiles = disabledSf + setUseIc(false) commandCheck(graph) + of cmdNifC: + # Generate C code from NIF files + wantMainModule(conf) + setOutFile(conf) + commandNifC(graph) + of cmdDeps: + # Generate .build.nif for nifmake + wantMainModule(conf) + when not defined(nimKochBootstrap): + commandDeps(conf) + else: + rawMessage(conf, errGenerated, "nim deps not available in bootstrap build") of cmdParse: wantMainModule(conf) discard parseFile(conf.projectMainIdx, cache, conf) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 408acd3ed3..b52ee9f4f0 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -18,6 +18,7 @@ import ic / [packed_ast, ic] when not defined(nimKochBootstrap): import ast2nif + import "../dist/nimony/src/lib" / [nifstreams, bitabs] when defined(nimPreviewSlimSystem): import std/assertions @@ -140,6 +141,7 @@ type cachedFiles*: StringTableRef procGlobals*: seq[PNode] + nifReplayActions*: Table[int32, seq[PNode]] # module position -> replay actions for NIF TPassContext* = object of RootObj # the pass's context idgen*: IdGenerator @@ -366,6 +368,10 @@ proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; ## we also need to record this to the packed module. g.attachedOps[op][t.itemId] = LazySym(sym: value) +proc setAttachedOp*(g: ModuleGraph; module: int; typeId: ItemId; op: TTypeAttachedOp; value: PSym) = + ## Overload that takes ItemId directly, useful for registering hooks from NIF index. + g.attachedOps[op][typeId] = LazySym(sym: value) + proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = ## we also need to record this to the packed module. g.attachedOps[op][t.itemId] = LazySym(sym: value) @@ -393,6 +399,10 @@ proc setMethodsPerType*(g: ModuleGraph; id: ItemId, methods: seq[LazySym]) = # TODO: add it for packed modules g.methodsPerType[id] = methods +proc addNifReplayAction*(g: ModuleGraph; module: int32; n: PNode) = + ## Stores a replay action for NIF-based incremental compilation. + g.nifReplayActions.mgetOrPut(module, @[]).add n + iterator getMethodsPerType*(g: ModuleGraph; t: PType): PSym = if g.methodsPerType.contains(t.itemId): for it in mitems g.methodsPerType[t.itemId]: @@ -425,7 +435,30 @@ proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) = proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = result = nil - if g.config.symbolFiles == disabledSf: return nil + if g.config.symbolFiles == disabledSf: + # For NIF-based compilation, search in loaded NIF modules + when not defined(nimKochBootstrap): + # Only try to resolve from NIF if we're actually using NIF files (cmdNifC) + if g.config.cmd == cmdNifC: + # First try system module (most compilerprocs are there) + let systemFileIdx = g.config.m.systemFileIdx + if systemFileIdx != InvalidFileIdx: + result = tryResolveCompilerProc(ast.program, name, systemFileIdx) + if result != nil: + strTableAdd(g.compilerprocs, result) + return result + + # Try threadpool module (some compilerprocs like FlowVar are there) + # Find threadpool module by searching loaded modules + for moduleIdx in 0..= 0: + for op in AttachedOp: + let (hookSymId, isGeneric) = hooksPerType.a[op] + if hookSymId != nifstreams.SymId(0): + let hookSym = resolveHookSym(ast.program, hookSymId) + if hookSym != nil: + setAttachedOp(g, int(fileIdx), typeItemId, toTTypeAttachedOp(op), hookSym) + # Register converters from NIF index with the module's interface + for (destType, convSym) in converters: + let symId = pool.syms.getOrIncl(convSym) + let convPSym = resolveHookSym(ast.program, symId) # reuse hook resolution + if convPSym != nil: + g.ifaces[fileIdx.int].converters.add LazySym(sym: convPSym) + # Register methods per type from NIF index + for classEntry in classes: + let typeItemId = parseTypeSymIdToItemId(ast.program, classEntry.cls) + if typeItemId.module >= 0: + var methodSyms: seq[LazySym] = @[] + for methodEntry in classEntry.methods: + let methodSym = resolveHookSym(ast.program, methodEntry.fn) + if methodSym != nil: + methodSyms.add LazySym(sym: methodSym) + if methodSyms.len > 0: + setMethodsPerType(g, typeItemId, methodSyms) cachedModules.add fileIdx proc configComplete*(g: ModuleGraph) = diff --git a/compiler/msgs.nim b/compiler/msgs.nim index f0e7419f68..5c52c10d01 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -60,11 +60,12 @@ proc makeCString*(s: string): Rope = toCChar(s[i], result) result.add('\"') -proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo = +proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile; kind = fikSource): TFileInfo = result = TFileInfo(fullPath: fullPath, projPath: projPath, shortName: fullPath.extractFilename, quotedFullName: fullPath.string.makeCString, - lines: @[] + lines: @[], + kind: kind ) result.quotedName = result.shortName.makeCString when defined(nimpretty): @@ -138,11 +139,18 @@ proc registerNifSuffix*(conf: ConfigRef; suffix: string; isKnownFile: var bool): if result == InvalidFileIdx: isKnownFile = false result = conf.m.fileInfos.len.FileIndex - conf.m.fileInfos.add(newFileInfo(AbsoluteFile suffix, RelativeFile suffix)) + conf.m.fileInfos.add(newFileInfo(AbsoluteFile suffix, RelativeFile suffix, fikNifModule)) conf.m.filenameToIndexTbl[suffix] = result else: isKnownFile = true +proc fileInfoKind*(conf: ConfigRef; fileIdx: FileIndex): FileInfoKind = + ## Returns the kind of a FileIndex (source file or NIF module suffix). + if fileIdx.int >= 0 and fileIdx.int < conf.m.fileInfos.len: + result = conf.m.fileInfos[fileIdx.int].kind + else: + result = fikSource # Default to source for unknown indices + proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo = result = TLineInfo(fileIndex: fileInfoIdx) if line < int high(uint16): diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim new file mode 100644 index 0000000000..7cd8b1f580 --- /dev/null +++ b/compiler/nifbackend.nim @@ -0,0 +1,121 @@ +# +# +# The Nim Compiler +# (c) Copyright 2025 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## NIF-based C/C++ code generator backend. +## +## This module implements C code generation from precompiled NIF files. +## It traverses the module dependency graph starting from the main module +## and generates C code for all reachable modules. +## +## Usage: +## 1. Compile modules to NIF: nim m mymodule.nim +## 2. Generate C from NIF: nim nifc myproject.nim + +import std/[intsets, tables, sets, os] + +when defined(nimPreviewSlimSystem): + import std/assertions + +import ast, options, lineinfos, modulegraphs, cgendata, cgen, + pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif + +proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[PSym] = + ## Traverse the module dependency graph using a stack. + ## Returns all modules that need code generation, in dependency order. + var visited = initIntSet() + var stack: seq[FileIndex] = @[mainFileIdx] + result = @[] + var cachedModules: seq[FileIndex] = @[] + + while stack.len > 0: + let fileIdx = stack.pop() + + if not visited.containsOrIncl(int(fileIdx)): + # Only load full AST for main module; others are loaded lazily by codegen + let isMainModule = fileIdx == mainFileIdx + let module = moduleFromNifFile(g, fileIdx, cachedModules, loadFullAst=isMainModule) + if module != nil: + result.add module + # Add dependencies to stack (they come from cachedModules) + for dep in cachedModules: + if not visited.contains(int(dep)): + stack.add dep + cachedModules.setLen(0) + +proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule = + ## Set up a BModule for code generation from a NIF module. + if g.backend == nil: + g.backend = cgendata.newModuleList(g) + result = cgen.newModule(BModuleList(g.backend), module, g.config) + +proc generateCodeForModule(g: ModuleGraph; module: PSym) = + ## Generate C code for a single module. + let moduleId = module.position + var bmod = BModuleList(g.backend).modules[moduleId] + if bmod == nil: + bmod = setupNifBackendModule(g, module) + + # Generate code for the module's top-level statements + if module.ast != nil: + cgen.genTopLevelStmt(bmod, module.ast) + + # Finalize the module (this adds it to modulesClosed) + # Create an empty stmt list as the init body - genInitCode in writeModule will set it up properly + let initStmt = newNodeI(nkStmtList, module.info) + finalCodegenActions(g, bmod, initStmt) + + # Generate dispatcher methods + for disp in getDispatchers(g): + genProcAux(bmod, disp) + +proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = + ## Main entry point for NIF-based C code generation. + ## Traverses the module dependency graph and generates C code. + + # Reset backend state + resetForBackend(g) + let mainModule = g.getModule(mainFileIdx) + + # Also ensure system module is set up and generated if it exists + if g.systemModule != nil and g.systemModule != mainModule: + let systemBmod = BModuleList(g.backend).modules[g.systemModule.position] + if systemBmod == nil: + discard setupNifBackendModule(g, g.systemModule) + generateCodeForModule(g, g.systemModule) + + # Load all modules in dependency order using stack traversal + let modules = loadModuleDependencies(g, mainFileIdx) + if modules.len == 0: + rawMessage(g.config, errGenerated, + "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx)) + return + + # Set up backend modules for all modules that need code generation + for module in modules: + discard setupNifBackendModule(g, module) + + # Generate code for all modules except main (main goes last) + # This ensures all modules are added to modulesClosed + for module in modules: + if module != mainModule: + generateCodeForModule(g, module) + + # Generate main module last (so all init procs are registered) + if mainModule != nil: + generateCodeForModule(g, mainModule) + + # Write C files + if g.backend != nil: + cgenWriteModules(g.backend, g.config) + + # Run C compiler + if g.config.cmd != cmdTcc: + extccomp.callCCompiler(g.config) + if not g.config.hcrOn: + extccomp.writeJsonBuildInstructions(g.config, g.cachedFiles) diff --git a/compiler/nim.nim b/compiler/nim.nim index 005f11a580..72302a186e 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -118,7 +118,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if conf.selectedGC == gcUnselected: if conf.backend in {backendC, backendCpp, backendObjc} or (conf.cmd in cmdDocLike and conf.backend != backendJs) or - conf.cmd == cmdGendepend: + conf.cmd in {cmdGendepend, cmdNifC, cmdDeps, cmdM}: initOrcDefines(conf) mainCommand(graph) diff --git a/compiler/options.nim b/compiler/options.nim index 142080cc8f..80b6cd729c 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -175,6 +175,8 @@ type cmdJsonscript # compile a .json build file # old unused: cmdInterpret, cmdDef: def feature (find definition for IDEs) cmdCompileToNif + cmdNifC # generate C code from NIF files + cmdDeps # generate .build.nif for nifmake const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index f00d0a3196..58f77b6533 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -1,10 +1,11 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs, lineinfos, reorder, options, semdata, cgendata, modules, pathutils, - packages, syntaxes, depends, vm, pragmas, idents, lookups, wordrecg, + packages, syntaxes, depends, vm, vmdef, pragmas, idents, lookups, wordrecg, liftdestructors, nifgen when not defined(nimKochBootstrap): import ast2nif + import "../dist/nimony/src/lib" / [nifstreams, bitabs] import pipelineutils @@ -38,7 +39,12 @@ proc processPipeline(graph: ModuleGraph; semNode: PNode; bModule: PPassContext): of GenDependPass: result = addDotDependency(bModule, semNode) of SemPass: - result = graph.emptyNode + # Return the semantic node for cmdM (NIF generation needs it) + # For regular check, we don't need the result + if graph.config.cmd == cmdM: + result = semNode + else: + result = graph.emptyNode of Docgen2Pass, Docgen2TexPass: when not defined(leanCompiler): result = processNode(bModule, semNode) @@ -160,7 +166,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator s = stream graph.interactive = stream.kind == llsStdIn var topLevelStmts = - if optCompress in graph.config.globalOptions: + if optCompress in graph.config.globalOptions or graph.config.cmd == cmdM: newNodeI(nkStmtList, module.info) else: nil @@ -235,14 +241,60 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator raiseAssert "use setPipeLinePass to set a proper PipelinePass" when not defined(nimKochBootstrap): - if optCompress in graph.config.globalOptions and not graph.config.isDefined("nimscript"): + if (optCompress in graph.config.globalOptions or graph.config.cmd == cmdM) and + not graph.config.isDefined("nimscript"): topLevelStmts.add finalNode - writeNifModule(graph.config, module.position.int32, topLevelStmts) + # Collect replay actions from both pragma computations and VM state diff + var replayActions: seq[PNode] = @[] + # Get pragma-recorded replay actions (compile, link, passC, passL, etc.) + if graph.nifReplayActions.hasKey(module.position.int32): + replayActions.add graph.nifReplayActions[module.position.int32] + # Also get VM state diff (macro cache operations) + if graph.vm != nil: + for (m, n) in PCtx(graph.vm).vmstateDiff: + if m == module: + replayActions.add n + # Collect hooks from the module graph for the current module + var hooks = default array[AttachedOp, seq[HookIndexEntry]] + for op in TTypeAttachedOp: + if op == attachedDeepCopy: continue # Not supported in nimony + let nimonyOp = toAttachedOp(op) + for typeId, lazySym in graph.attachedOps[op]: + if typeId.module == module.position.int32: + let sym = lazySym.sym + if sym != nil: + hooks[nimonyOp].add toHookIndexEntry(graph.config, typeId, sym) + # Collect converters from the module's interface + var converters: seq[(nifstreams.SymId, nifstreams.SymId)] = @[] + for lazySym in graph.ifaces[module.position].converters: + let sym = lazySym.sym + if sym != nil: + let entry = toConverterIndexEntry(graph.config, sym) + if entry[0] != nifstreams.SymId(0): + converters.add entry + # Collect methods per type for classes + var classes: seq[ClassIndexEntry] = @[] + for typeId, methodList in graph.methodsPerType: + if typeId.module == module.position.int32: + var methods: seq[MethodIndexEntry] = @[] + for lazySym in methodList: + let sym = lazySym.sym + if sym != nil: + # Generate a method signature (simplified - name and param count) + let sig = sym.name.s & "/" & $sym.typImpl.sonsImpl.len + methods.add toMethodIndexEntry(graph.config, sym, sig) + if methods.len > 0: + classes.add ClassIndexEntry( + cls: toClassSymId(graph.config, typeId), + methods: methods + ) + writeNifModule(graph.config, module.position.int32, topLevelStmts, hooks, converters, classes, replayActions) - if graph.config.backend notin {backendC, backendCpp, backendObjc}: + if graph.config.backend notin {backendC, backendCpp, backendObjc} and graph.config.cmd != cmdM: # We only write rod files here if no C-like backend is active. # The C-like backends have been patched to support the IC mechanism. # They are responsible for closing the rod files. See `cbackend.nim`. + # cmdM uses NIF files only, not ROD files. closeRodFile(graph, module) result = true @@ -261,11 +313,21 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF if result == nil: var cachedModules: seq[FileIndex] = @[] when not defined(nimKochBootstrap): - # Try loading from NIF file first if optCompress is enabled - if optCompress in graph.config.globalOptions and not graph.config.isDefined("nimscript"): + # For cmdM: load imports from NIF files (but compile the main module from source) + # Skip when withinSystem is true (compiling system.nim itself) + if graph.config.cmd == cmdM and + sfMainModule notin flags and + not graph.withinSystem and + not graph.config.isDefined("nimscript"): result = moduleFromNifFile(graph, fileIdx, cachedModules) - if result == nil: - # Fall back to ROD file loading + if result == nil: + let nifPath = toNifFilename(graph.config, fileIdx) + localError(graph.config, unknownLineInfo, + "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) & + " (expected: " & nifPath & ")") + return nil # Don't fall through to compile from source + if result == nil and graph.config.cmd != cmdM: + # Fall back to ROD file loading (not used for cmdM which uses NIF only) result = moduleFromRodFile(graph, fileIdx, cachedModules) let path = toFullPath(graph.config, fileIdx) let filename = AbsoluteFile path @@ -287,8 +349,11 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF partialInitModule(result, graph, fileIdx, filename) for m in cachedModules: registerModuleById(graph, m) - if sfMainModule in flags and graph.config.cmd == cmdM: - discard + if graph.config.cmd == cmdM: + # cmdM uses NIF files - replay from module AST loaded by loadNifModule + let module = graph.getModule(m) + if module != nil and module.ast != nil: + replayStateChanges(module, graph) else: replayStateChanges(graph.packed.pm[m.int].module, graph) replayGenericCacheInformation(graph, m.int) @@ -346,6 +411,20 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx graph.withinSystem = true discard graph.compilePipelineModule(projectFile, {sfMainModule, sfSystemModule}) graph.withinSystem = false + elif graph.config.cmd == cmdM: + # For cmdM: load system.nim from NIF first, then compile the main module + connectPipelineCallbacks(graph) + graph.config.m.systemFileIdx = fileInfoIdx(graph.config, + graph.config.libpath / RelativeFile"system.nim") + var cachedModules: seq[FileIndex] = @[] + when not defined(nimKochBootstrap): + graph.systemModule = moduleFromNifFile(graph, graph.config.m.systemFileIdx, cachedModules) + if graph.systemModule == nil: + let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx) + localError(graph.config, unknownLineInfo, + "nim m requires precompiled NIF for system module (expected: " & nifPath & ")") + return + discard graph.compilePipelineModule(projectFile, {sfMainModule}) else: graph.compilePipelineSystemModule() discard graph.compilePipelineModule(projectFile, {sfMainModule}) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 4dc7e3e26a..c29429370e 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -358,6 +358,9 @@ proc addImportFileDep*(c: PContext; f: FileIndex) = proc addPragmaComputation*(c: PContext; n: PNode) = if c.config.symbolFiles != disabledSf: addPragmaComputation(c.encoder, c.packedRepr, n) + # Also store for NIF-based IC (cmdM mode or optCompress) + if optCompress in c.config.globalOptions or c.config.cmd == cmdM: + addNifReplayAction(c.graph, c.module.position.int32, n) proc inclSym(sq: var seq[PSym], s: PSym): bool = for i in 0.. 0, "Too many iterations in test. Infinite loop?") except TestError: exceptionCaught = true @@ -847,3 +850,50 @@ block: doAssert(w() == 123) doAssert(getCurrentExceptionMsg() == "Outer error") doAssert(getCurrentExceptionMsg() == "") + +block: #25330 (v1) + iterator count1(): int {.closure.} = + yield 1 + raiseTestError() + + iterator count0(): int {.closure.} = + try: + var count = count1 + while true: + yield count() + if finished(count): break + finally: + try: + checkpoint(2) + var count2 = count1 + while true: + yield count2() + if finished(count2): break + discard # removing this outputs "raise" + except: + checkpoint(3) + raise + + testExc(count0, 1, 2, 1, 3) + +block: #25330 (v2) + iterator count1(): int {.closure.} = + yield 1 + raiseTestError() + + iterator count0(): int {.closure.} = + try: + var count = count1 + for x in 0 .. 10: + yield count() + finally: + try: + checkpoint(2) + var count2 = count1 + for x in 0 .. 10: + yield count2() + except: + checkpoint(3) + raise + + testExc(count0, 1, 2, 1, 3)