diff --git a/compiler/ast.nim b/compiler/ast.nim index 6b00935c61..556df74080 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -46,6 +46,12 @@ template loadType(t: PType) = when not defined(nimKochBootstrap): ast2nif.loadType(program, t) +proc loadSymCallback*(s: PSym) {.nimcall.} = + loadSym(s) + +proc loadTypeCallback*(t: PType) {.nimcall.} = + loadType(t) + proc ensureMutable*(s: PSym) {.inline.} = assert s.state != Sealed if s.state == Partial: loadSym(s) @@ -82,9 +88,6 @@ proc setOwner*(s: PType; owner: PSym) {.inline.} = if s.state == Partial: loadType(s) s.ownerFieldImpl = owner -# Accessor procs for TSym fields -# Note: kind is kept as a direct field for case statement compatibility -# but we still provide an accessor that checks state proc kind*(s: PSym): TSymKind {.inline.} = if s.state == Partial: loadSym(s) result = s.kindImpl @@ -227,7 +230,7 @@ proc offset*(s: PSym): int32 {.inline.} = result = s.offsetImpl proc `offset=`*(s: PSym, val: int32) {.inline.} = - assert s.state != Sealed + #assert s.state != Sealed if s.state == Partial: loadSym(s) s.offsetImpl = val @@ -293,7 +296,8 @@ proc incl*(s: PSym; flags: set[TSymFlag]) {.inline.} = s.flagsImpl.incl(flags) proc incl*(s: PSym; flag: TLocFlag) {.inline.} = - assert s.state != Sealed + #assert s.state != Sealed + # locImpl is a backend field so do not protect it against mutations if s.state == Partial: loadSym(s) s.locImpl.flags.incl(flag) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 1c6b9571c0..c419a934d8 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -19,45 +19,16 @@ import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos, nifindexes, nifreader] import "../dist/nimony/src/gear2" / modnames import "../dist/nimony/src/models" / nifindex_tags - +import typekeys import ic / [enum2nif] -# Re-export types needed for hook, converter, and method handling -export nifindexes.AttachedOp, nifindexes.HookIndexEntry, nifindexes.HooksPerType -export nifindexes.ClassIndexEntry, nifindexes.MethodIndexEntry - -proc toAttachedOp*(op: TTypeAttachedOp): AttachedOp = - ## Maps Nim compiler's TTypeAttachedOp to nimony's AttachedOp. - ## Returns attachedDestroy for attachedDeepCopy (caller should skip it). - case op - of attachedDestructor: attachedDestroy - of attachedAsgn: attachedCopy - of attachedWasMoved: nifindexes.attachedWasMoved - of attachedDup: nifindexes.attachedDup - of attachedSink: nifindexes.attachedSink - of attachedTrace: nifindexes.attachedTrace - of attachedDeepCopy: attachedDestroy # Not supported, caller should skip - -proc toTTypeAttachedOp*(op: AttachedOp): TTypeAttachedOp = - ## Maps nimony's AttachedOp back to Nim compiler's TTypeAttachedOp. - case op - of attachedDestroy: attachedDestructor - of attachedCopy: attachedAsgn - of nifindexes.attachedWasMoved: astdef.attachedWasMoved - of nifindexes.attachedDup: astdef.attachedDup - of nifindexes.attachedSink: astdef.attachedSink - of nifindexes.attachedTrace: astdef.attachedTrace - - -proc cachedModuleSuffix*(config: ConfigRef; fileIdx: FileIndex): string = - ## Gets or computes the module suffix for a FileIndex. - ## For NIF modules, the suffix is already stored in the file info. - ## For source files, computes it from the path. - let fullPath = toFullPath(config, fileIdx) - if fileInfoKind(config, fileIdx) == fikNifModule: - result = fullPath # Already a suffix - else: - result = moduleSuffix(fullPath, cast[seq[string]](config.searchPaths)) +proc typeToNifSym(typ: PType; config: ConfigRef): string = + result = "`t" + result.addInt ord(typ.kind) + result.add '.' + result.addInt typ.uniqueId.item + result.add '.' + result.add modname(typ.uniqueId.module, config) proc toHookIndexEntry*(config: ConfigRef; typeId: ItemId; hookSym: PSym): HookIndexEntry = ## Converts a type ItemId and hook symbol to a HookIndexEntry for the NIF index. @@ -150,17 +121,6 @@ proc oldLineInfo(w: var LineInfoWriter; info: PackedLineInfo): TLineInfo = result = TLineInfo(line: x.line.uint16, col: x.col.int16, fileIndex: fileIdx) -# -------------- Module name handling -------------------------------------------- - -proc modname(module: int; conf: ConfigRef): string = - cachedModuleSuffix(conf, module.FileIndex) - -proc modname(module: PSym; conf: ConfigRef): string = - assert module.kindImpl == skModule - modname(module.positionImpl, conf) - - - # ------------- Writer --------------------------------------------------------------- #[ @@ -197,9 +157,10 @@ type decodedFileIndices: HashSet[FileIndex] locals: HashSet[ItemId] # track proc-local symbols inProc: int - writtenTypes: seq[PType] # types written in this module, to be unloaded later - writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later + #writtenTypes: seq[PType] # types written in this module, to be unloaded later + #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later exports: Table[FileIndex, HashSet[string]] # module -> specific symbol names (empty = all) + writtenPackages: HashSet[string] const # Symbol kinds that are always local to a proc and should never have module suffix @@ -217,10 +178,11 @@ proc toNifSymName(w: var Writer; sym: PSym): string = result.addInt sym.disamb if not isLocalSym(sym) and sym.itemId notin w.locals: # Global symbol: ident.disamb.moduleSuffix - let module = sym.itemId.module result.add '.' + let module = if sym.kindImpl == skPackage: w.currentModule else: sym.itemId.module result.add modname(module, w.infos.config) + proc globalName(sym: PSym; config: ConfigRef): string = result = sym.name.s result.add '.' @@ -282,14 +244,6 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) -proc typeToNifSym(typ: PType; config: ConfigRef): string = - result = "`t" - result.addInt ord(typ.kind) - result.add '.' - result.addInt typ.uniqueId.item - result.add '.' - result.add modname(typ.uniqueId.module, config) - proc writeLoc(w: var Writer; dest: var TokenBuf; loc: TLoc) = dest.addIdent toNifTag(loc.k) dest.addIdent toNifTag(loc.storage) @@ -329,8 +283,6 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = elif typ.itemId.module == w.currentModule and typ.state == Complete: typ.state = Sealed writeTypeDef(w, dest, typ) - # Collect for later unloading after entire module is written - w.writtenTypes.add typ else: dest.addSymUse pool.syms.getOrIncl(typeToNifSym(typ, w.infos.config)), NoLineInfo @@ -396,6 +348,8 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = else: dest.addIntLit sym.positionImpl + writeLib(w, dest, sym.annexImpl) + # For routine symbols, pre-collect generic params into w.locals before writing # the type. This ensures they get consistent short names, and their sdefs are # written in the type where lazy loading can find them via extractLocalSymsFromTree. @@ -417,15 +371,11 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeSym(w, dest, sym.instantiatedFromImpl) dest.addParRi - # Collect for later unloading after entire module is written - if sym.kindImpl notin {skPackage}: - # do not unload modules - w.writtenSyms.add sym -proc shouldWriteSymDef(w: Writer; sym: PSym): bool {.inline.} = +proc shouldWriteSymDef(w: var Writer; sym: PSym): bool {.inline.} = # Don't write module/package symbols - they don't have NIF files - if sym.kindImpl in {skPackage}: - return false + if sym.kindImpl == skPackage: + return not w.writtenPackages.containsOrIncl(sym.name.s) # Already written - don't write again if sym.state == Sealed: return false @@ -442,10 +392,6 @@ proc shouldWriteSymDef(w: Writer; sym: PSym): bool {.inline.} = proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = if sym == nil: dest.addDotToken() - elif sym.kindImpl in {skPackage}: - # Write module/package symbols as dots - they're resolved differently - # (by position/FileIndex, not by NIF lookup) - dest.addDotToken() elif shouldWriteSymDef(w, sym): sym.state = Sealed writeSymDef(w, dest, sym) @@ -693,11 +639,55 @@ proc buildExportBuf(w: var Writer): TokenBuf = result.addParRi() let replayTag = registerTag("replay") +let repConverterTag = registerTag("repconverter") +let repDestroyTag = registerTag("repdestroy") +let repWasMovedTag = registerTag("repwasmoved") +let repCopyTag = registerTag("repcopy") +let repSinkTag = registerTag("repsink") +let repDupTag = registerTag("repdup") +let repTraceTag = registerTag("reptrace") +let repDeepCopyTag = registerTag("repdeepcopy") +let repEnumToStrTag = registerTag("repenumtostr") +let repMethodTag = registerTag("repmethod") +#let repClassTag = registerTag("repclass") +let includeTag = registerTag("include") +let importTag = registerTag("import") + +proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = + case op.kind + of HookEntry: + case op.op + of attachedDestructor: + content.addParLe repDestroyTag, NoLineInfo + of attachedAsgn: + content.addParLe repCopyTag, NoLineInfo + of attachedWasMoved: + content.addParLe repWasMovedTag, NoLineInfo + of attachedDup: + content.addParLe repDupTag, NoLineInfo + of attachedSink: + content.addParLe repSinkTag, NoLineInfo + of attachedTrace: + content.addParLe repTraceTag, NoLineInfo + of attachedDeepCopy: + content.addParLe repDeepCopyTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) + content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) + content.addParRi() + of ConverterEntry: + content.addParLe repConverterTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) + content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) + content.addParRi() + of MethodEntry: + discard "to implement" + of EnumToStrEntry: + discard "to implement" + of GenericInstEntry: + discard "will only be written later to ensure it is materialized" proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; - hooks: array[AttachedOp, seq[HookIndexEntry]]; - converters: seq[(nifstreams.SymId, nifstreams.SymId)]; - classes: seq[ClassIndexEntry]; + opsLog: seq[LogEntry]; replayActions: seq[PNode] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) @@ -711,6 +701,10 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; for action in replayActions: writeNode(w, content, action) content.addParRi() + # Only write ops that belong to this module + for op in opsLog: + if op.module == thisModule.int: + writeOp(w, content, op) w.writeToplevelNode content, n @@ -723,25 +717,26 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; var dest = createTokenBuf(600) createStmtList(dest, rootInfo) dest.add w.deps - dest.add content + # do not write the (stmts .. ) wrapper: + for i in 3 ..< content.len-1: + dest.add content[i] + + # ensure the hooks we announced end up in the NIF file regardless of + # whether they have been used: + for op in opsLog: + if op.module == thisModule.int: + let s = op.sym + if s.state != Sealed: + s.state = Sealed + writeSymDef w, dest, s + dest.addParRi() writeFile(dest, d) - # Build index with export, hook, converter, and method information let exportBuf = buildExportBuf(w) createIndex(d, dest[0].info, false, - IndexSections(hooks: hooks, converters: converters, classes: classes, exportBuf: exportBuf)) - - # Don't unload symbols/types yet - they may be needed by other modules that haven't - # had their NIF files written. For recursive module dependencies (like system.nim), - # we need all NIFs to exist before we can safely unload and reload. - # TODO: Implement deferred unloading at end of compilation for memory savings. - #for typ in w.writtenTypes: - # forcePartial(typ) - #for sym in w.writtenSyms: - # forcePartial(sym) - + IndexSections(exportBuf: exportBuf)) # --------------------------- Loader (lazy!) ----------------------------------------------- @@ -1107,6 +1102,8 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: else: loadField s.positionImpl + s.annexImpl = loadAnnex(c, n, thisModule, localSyms) + # Local symbols were already extracted upfront in loadSym, so we can use # the simple loadTypeStub here. s.typImpl = loadTypeStub(c, n, localSyms) @@ -1220,10 +1217,12 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; inc n # skip `sd` tag loadSymFromCursor(c, sym, n, thisModule, localSyms) sym.state = Sealed # mark as fully loaded + result = newSymNode(sym, info) else: sym = c.loadSymStub(name.symId, thisModule, localSyms) skip n # skip the entire sdef for indexed symbols - result = newSymNode(sym, info) + result = newSymNode(sym, info) + result.flags.incl nfLazyType of typeDefTagName: raiseAssert "`td` tag in invalid context" of "none": @@ -1410,24 +1409,6 @@ proc toNifIndexFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.idx.nif").string -proc parseTypeSymIdToItemId*(c: var DecodeContext; symId: nifstreams.SymId): ItemId = - ## Parses a type SymId (format: `"`tN.modulesuffix"`) to extract ItemId. - let s = pool.syms[symId] - if not s.startsWith("`t"): - return ItemId(module: -1, item: 0) - var i = 2 # skip "`t" - var item = 0'i32 - while i < s.len and s[i] in {'0'..'9'}: - item = item * 10 + int32(ord(s[i]) - ord('0')) - inc i - if i < s.len and s[i] == '.': - inc i - let suffix = s.substr(i) - let module = moduleId(c, suffix) - result = ItemId(module: int32(module), item: item) - else: - result = ItemId(module: -1, item: item) - proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym = result = c.syms.getOrDefault(symAsStr)[0] if result != nil: @@ -1456,8 +1437,9 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym = ## Resolves a hook SymId to PSym. + ## Hook symbols are often private (generated =destroy, =wasMoved, etc.) let symAsStr = pool.syms[symId] - result = resolveSym(c, symAsStr, false) + result = resolveSym(c, symAsStr, true) proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym = ## Tries to resolve a compiler proc from a module by checking the NIF index. @@ -1466,10 +1448,115 @@ proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: let symName = name & ".0." & suffix result = resolveSym(c, symName, true) +proc loadLogOp(c: var DecodeContext; logOps: var seq[LogEntry]; s: var Stream; kind: LogEntryKind; op: TTypeAttachedOp; module: int): PackedToken = + result = next(s) + var key = "" + if result.kind == StringLit: + key = pool.strings[result.litId] + result = next(s) + else: + raiseAssert "expected StringLit but got " & $result.kind + if result.kind == Symbol: + let sym = resolveHookSym(c, result.symId) + if sym != nil: + logOps.add LogEntry(kind: kind, op: op, module: module, key: key, sym: sym) + # else: symbol not indexed, skip this hook entry + result = next(s) + if result.kind == ParRi: + result = next(s) + else: + raiseAssert "expected ParRi but got " & $result.kind + +proc skipTree(s: var Stream): PackedToken = + result = next(s) + var nested = 1 + while nested > 0: + if result.kind == ParLe: + inc nested + elif result.kind == ParRi: + dec nested + elif result.kind == EofToken: + break + result = next(s) + +proc nextSubtree(r: var Stream; dest: var TokenBuf; tok: var PackedToken) = + r.parents[0] = tok.info + var nested = 1 + dest.add tok # tag + while true: + tok = r.next() + dest.add tok + if tok.kind == EofToken: + break + elif tok.kind == ParLe: + inc nested + elif tok.kind == ParRi: + dec nested + if nested == 0: break + +proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suffix: string; logOps: var seq[LogEntry]; module: int): PNode = + result = newNode(nkStmtList) + var localSyms = initTable[string, PSym]() + + var t = next(s) # skip dot + var cont = true + while cont and t.kind != EofToken: + if t.kind == ParLe: + if t.tagId == replayTag: + # Always load replay actions (macro cache operations) + t = next(s) # move past (replay + while t.kind != ParRi and t.kind != EofToken: + if t.kind == ParLe: + var buf = createTokenBuf(50) + nextSubtree(s, buf, t) + var cursor = cursorAt(buf, 0) + let replayNode = loadNode(c, cursor, suffix, localSyms) + if replayNode != nil: + result.sons.add replayNode + t = next(s) + if t.kind == ParRi: + t = next(s) + else: + raiseAssert "expected ParRi but got " & $t.kind + elif t.tagId == repConverterTag: + t = loadLogOp(c, logOps, s, ConverterEntry, attachedTrace, module) + elif t.tagId == repDestroyTag: + t = loadLogOp(c, logOps, s, HookEntry, attachedDestructor, module) + elif t.tagId == repWasMovedTag: + t = loadLogOp(c, logOps, s, HookEntry, attachedWasMoved, module) + elif t.tagId == repCopyTag: + t = loadLogOp(c, logOps, s, HookEntry, attachedAsgn, module) + elif t.tagId == repSinkTag: + t = loadLogOp(c, logOps, s, HookEntry, attachedSink, module) + elif t.tagId == repDupTag: + t = loadLogOp(c, logOps, s, HookEntry, attachedDup, module) + elif t.tagId == repTraceTag: + t = loadLogOp(c, logOps, s, HookEntry, attachedTrace, module) + elif t.tagId == repDeepCopyTag: + t = loadLogOp(c, logOps, s, HookEntry, attachedDeepCopy, module) + elif t.tagId == repEnumToStrTag: + t = loadLogOp(c, logOps, s, EnumToStrEntry, attachedTrace, module) + elif t.tagId == repMethodTag: + t = loadLogOp(c, logOps, s, MethodEntry, attachedTrace, module) + #elif t.tagId == repClassTag: + # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) + elif t.tagId == includeTag or t.tagId == importTag: + t = skipTree(s) + elif loadFullAst: + # Parse the full statement + var buf = createTokenBuf(50) + nextSubtree(s, buf, t) + var cursor = cursorAt(buf, 0) + let stmtNode = loadNode(c, cursor, suffix, localSyms) + if stmtNode != nil: + result.sons.add stmtNode + else: + cont = false + else: + cont = false + proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; - hooks: var Table[nifstreams.SymId, HooksPerType]; - converters: var seq[(string, string)]; - classes: var seq[ClassIndexEntry]; + logOps: var seq[LogEntry]; loadFullAst: bool = false): PNode = let suffix = moduleSuffix(c.infos.config, f) @@ -1480,70 +1567,18 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym populateInterfaceTablesFromIndex(c, module, interf, interfHidden, suffix) - # Return hooks from the index - hooks = move c.mods[module].index.hooks - # Return converters from the index - converters = move c.mods[module].index.converters - # Return classes/methods from the index - classes = move c.mods[module].index.classes - # Load the module AST (or just replay actions if loadFullAst is false) - result = newNode(nkStmtList) let s = addr c.mods[module].stream s.r.jumpTo 0 # Start from beginning discard processDirectives(s.r) - var localSyms = initTable[string, PSym]() var t = next(s[]) if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): + t = next(s[]) # skip (stmts t = next(s[]) # skip flags - t = next(s[]) # skip type - # Process all top-level statements - while t.kind != ParRi and t.kind != EofToken: - if t.kind == ParLe: - let tag = pool.tags[t.tagId] - if tag == "replay": - # Always load replay actions (macro cache operations) - t = next(s[]) # move past (replay - while t.kind != ParRi and t.kind != EofToken: - if t.kind == ParLe: - var buf = createTokenBuf(50) - nifcursors.parse(s[], buf, t.info) - var cursor = cursorAt(buf, 0) - let replayNode = loadNode(c, cursor, suffix, localSyms) - if replayNode != nil: - result.sons.add replayNode - t = next(s[]) - elif loadFullAst: - # Parse the full statement - var buf = createTokenBuf(50) - buf.add t # Add the ParLe token we already read - var nested = 1 - while nested > 0: - t = next(s[]) - buf.add t - if t.kind == ParLe: - inc nested - elif t.kind == ParRi: - dec nested - elif t.kind == EofToken: - break - var cursor = cursorAt(buf, 0) - let stmtNode = loadNode(c, cursor, suffix, localSyms) - if stmtNode != nil: - result.sons.add stmtNode - else: - # Skip over the statement by counting parentheses - var nested = 1 - while nested > 0: - t = next(s[]) - if t.kind == ParLe: - inc nested - elif t.kind == ParRi: - dec nested - elif t.kind == EofToken: - break - else: - t = next(s[]) + result = processTopLevel(c, s[], loadFullAst, suffix, logOps, f.int) + else: + result = newNode(nkStmtList) + when isMainModule: import std / syncio diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 2aefc7659f..30b2298fb2 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -990,6 +990,22 @@ proc newStrNode*(strVal: string; info: TLineInfo): PNode = result = newNodeI(nkStrLit, info) result.strVal = strVal +# Hooks, converters, method dispatchers and enum-to-string generated procs need special +# handling for IC, they end up in IC indexes etc. Thus we "log" them in the module graph +# and to pass them around to the NIF writer. This is not very elegant but it works. + +type + LogEntryKind* = enum + HookEntry, ConverterEntry, MethodEntry, EnumToStrEntry, GenericInstEntry + LogEntry* = object + kind*: LogEntryKind + op*: TTypeAttachedOp + isGeneric*: bool + module*: int # Which module this entry belongs to + key*: string + sym*: PSym + + proc forcePartial*(s: PSym) = ## Resets all impl-fields to their default values and sets state to Partial. ## This is useful for creating a stub symbol that can be lazily loaded later. diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 4bc8193ecb..5e37709af9 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3363,7 +3363,7 @@ proc genConstSetup(p: BProc; sym: PSym): bool = useHeader(m, sym) if sym.loc.k == locNone: fillBackendName(p.module, sym) - ensureMutable sym + backendEnsureMutable sym fillLoc(sym.locImpl, locData, sym.astdef, OnStatic) if m.hcrOn: incl(sym, lfIndirect) result = lfNoDecl notin sym.loc.flags @@ -3710,7 +3710,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = inc p.splitDecls genGotoState(p, n) of nkBreakState: genBreakState(p, n, d) - of nkMixinStmt, nkBindStmt: discard + of nkMixinStmt, nkBindStmt, nkReplayAction: discard else: internalError(p.config, n.info, "expr(" & $n.kind & "); unknown node kind") proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo; result: var Builder) = diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index a2b5e32cb8..b09000d005 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -11,7 +11,7 @@ # ------------------------- Name Mangling -------------------------------- -import sighashes, modulegraphs, std/strscans +import sighashes, std/strscans import ../dist/checksums/src/checksums/md5 import std/sequtils @@ -124,7 +124,7 @@ proc fillLocalName(p: BProc; s: PSym) = elif s.kind != skResult: result.add "_" & rope(counter+1) p.sigConflicts.inc(key) - ensureMutable s + backendEnsureMutable s s.locImpl.snippet = result proc scopeMangledParam(p: BProc; param: PSym) = diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 7fd7b0f8bd..c271cbda31 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -16,7 +16,7 @@ import rodutils, renderer, cgendata, aliases, lowerings, lineinfos, pathutils, transf, injectdestructors, astmsgs, modulepaths, pushpoppragmas, - mangleutils, cbuilderbase + mangleutils, cbuilderbase, modulegraphs from expanddefaults import caseObjDefaultBranch @@ -61,18 +61,25 @@ proc hcrOn(p: BProc): bool = p.module.config.hcrOn proc addForwardedProc(m: BModule, prc: PSym) = m.g.forwardedProcs.add(prc) -proc newModule*(g: BModuleList; module: PSym; conf: ConfigRef): BModule +proc newModule*(g: BModuleList; module: PSym; conf: ConfigRef; idgen: IdGenerator): BModule proc findPendingModule(m: BModule, s: PSym): BModule = # TODO fixme if m.config.symbolFiles == v2Sf or optCompress in m.config.globalOptions: let ms = s.itemId.module #getModule(s) result = m.g.modules[ms] + elif m.config.cmd in {cmdNifC, cmdM}: + var ms = getModule(s) + registerModule m.g.graph, ms + if ms.position >= m.g.modules.len: + result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms)) + else: + result = m.g.modules[ms.position] + if result == nil: + result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms)) else: var ms = getModule(s) result = m.g.modules[ms.position] - if result == nil: - result = newModule(m.g, ms, m.config) proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc = result = TLoc(k: k, storage: s, lode: lode, @@ -651,7 +658,7 @@ proc localVarDecl(res: var Builder, p: BProc; n: PNode, let s = n.sym if s.loc.k == locNone: fillLocalName(p, s) - ensureMutable s + backendEnsureMutable s fillLoc(s.locImpl, locLocalVar, n, OnStack) if s.kind == skLet: incl(s, lfNoDeepCopy) @@ -713,7 +720,7 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) = let s = n.sym if s.loc.k == locNone: fillBackendName(p.module, s) - ensureMutable s + backendEnsureMutable s fillLoc(s.locImpl, locGlobalVar, n, OnHeap) if treatGlobalDifferentlyForHCR(p.module, s): incl(s, lfIndirect) @@ -722,7 +729,7 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) = if q != nil and not containsOrIncl(q.declaredThings, s.id): varInDynamicLib(q, s) else: - ensureMutable s + backendEnsureMutable s s.locImpl.snippet = mangleDynLibProc(s) if value != "": internalError(p.config, n.info, ".dynlib variables cannot have a value") @@ -763,13 +770,13 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) = genGlobalVarDecl(p.module.s[cfsVars], p, n, td, initializer = initializer) if p.withinLoop > 0 and value == "": # fixes tests/run/tzeroarray: - ensureMutable s + backendEnsureMutable s resetLoc(p, s.locImpl) proc callGlobalVarCppCtor(p: BProc; v: PSym; vn, value: PNode; didGenTemp: var bool) = let s = vn.sym fillBackendName(p.module, s) - ensureMutable s + backendEnsureMutable s fillLoc(s.locImpl, locGlobalVar, vn, OnHeap) let td = getTypeDesc(p.module, vn.sym.typ, dkVar) var val = genCppParamsForCtor(p, value, didGenTemp) @@ -959,7 +966,7 @@ proc symInDynamicLib(m: BModule, sym: PSym) = var extname = sym.loc.snippet if not isCall: loadDynamicLib(m, lib) var tmp = mangleDynLibProc(sym) - ensureMutable sym + backendEnsureMutable sym sym.locImpl.snippet = tmp # from now on we only need the internal name sym.typ.sym = nil # generate a new name inc(m.labels, 2) @@ -1004,7 +1011,7 @@ proc varInDynamicLib(m: BModule, sym: PSym) = loadDynamicLib(m, lib) incl(sym, lfIndirect) var tmp = mangleDynLibProc(sym) - ensureMutable sym + backendEnsureMutable sym sym.locImpl.snippet = tmp # from now on we only need the internal name inc(m.labels, 2) let t = ptrType(getTypeDesc(m, sym.typ, dkVar)) @@ -1018,7 +1025,7 @@ proc varInDynamicLib(m: BModule, sym: PSym) = m.s[cfsVars].addVar(name = sym.loc.snippet, typ = t) proc symInDynamicLibPartial(m: BModule, sym: PSym) = - ensureMutable sym + backendEnsureMutable sym sym.locImpl.snippet = mangleDynLibProc(sym) sym.typ.sym = nil # generate a new name @@ -1336,9 +1343,9 @@ proc genProcLvl3*(m: BModule, prc: PSym) = returnStmt = extract(returnBuilder) elif sfConstructor in prc.flags: resNode.sym.incl lfIndirect - ensureMutable resNode.sym + backendEnsureMutable resNode.sym fillLoc(resNode.sym.locImpl, locParam, resNode, "this", OnHeap) - ensureMutable prc + backendEnsureMutable prc prc.locImpl.snippet = getTypeDesc(m, resNode.sym.locImpl.t, dkVar) else: fillResult(p.config, resNode, prc.typ) @@ -1352,11 +1359,11 @@ proc genProcLvl3*(m: BModule, prc: PSym) = if sfNoInit in prc.flags: discard elif allPathsAsgnResult(p, procBody) == InitSkippable: discard else: - ensureMutable res + backendEnsureMutable res resetLoc(p, res.locImpl) if skipTypes(res.typ, abstractInst).kind == tyArray: #incl(res.loc.flags, lfIndirect) - ensureMutable res + backendEnsureMutable res res.locImpl.storage = OnUnknown for i in 1..= g.modules.len: setLen(g.modules, module.position + 1) #growCache g.modules, module.position @@ -2387,8 +2395,7 @@ template injectG() {.dirty.} = proc setupCgen*(graph: ModuleGraph; module: PSym; idgen: IdGenerator): PPassContext = injectG() - result = newModule(g, module, graph.config) - result.idgen = idgen + result = newModule(g, module, graph.config, idgen) if optGenIndex in graph.config.globalOptions and g.generatedHeader == nil: let f = if graph.config.headerFile.len > 0: AbsoluteFile graph.config.headerFile else: graph.config.projectFull @@ -2562,7 +2569,7 @@ proc generateLibraryDestroyGlobals(graph: ModuleGraph; m: BModule; body: PNode; result = newSym(skProc, procname, m.idgen, m.module.owner, m.module.info) result.typ = newProcType(m.module.info, m.idgen, m.module.owner) result.typ.callConv = ccCDecl - ensureMutable result + backendEnsureMutable result incl result.flagsImpl, sfExportc result.locImpl.snippet = prefixedName if isDynlib: diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 91147d5e07..1cf5301bc0 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -37,8 +37,7 @@ proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) = if g.backend == nil: g.backend = cgendata.newModuleList(g) assert g.backend != nil - var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config) - bmod.idgen = idgenFromLoadedModule(m) + var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config, idgenFromLoadedModule(m)) proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = var bmod = BModuleList(g.backend).modules[m.module.position] diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index b52ee9f4f0..d338194ea5 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -20,6 +20,8 @@ when not defined(nimKochBootstrap): import ast2nif import "../dist/nimony/src/lib" / [nifstreams, bitabs] +import typekeys + when defined(nimPreviewSlimSystem): import std/assertions @@ -79,6 +81,8 @@ type typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId. procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId. attachedOps*: array[TTypeAttachedOp, Table[ItemId, LazySym]] # Type ID, destructors, etc. + loadedOps: array[TTypeAttachedOp, Table[string, PSym]] # This can later by unified with `attachedOps` once it's stable + opsLog*: seq[LogEntry] methodsPerGenericType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods memberProcsPerType*: Table[ItemId, seq[PSym]] # Type ID, attached member procs (only c++, virtual,member and ctor so far). initializersPerType*: Table[ItemId, PNode] # Type ID, AST call to the default ctor (c++ only) @@ -166,6 +170,9 @@ proc resetForBackend*(g: ModuleGraph) = g.enumToStringProcs.clear() g.dispatchers.setLen(0) g.methodsPerType.clear() + for a in mitems(g.loadedOps): + a.clear() + g.opsLog.setLen(0) const cb64 = [ @@ -361,11 +368,26 @@ proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym = ## if no such operation exists. if g.attachedOps[op].contains(t.itemId): result = resolveAttachedOp(g, g.attachedOps[op][t.itemId]) + elif g.config.cmd in {cmdNifC, cmdM}: + # Fall back to key-based lookup for NIF-loaded hooks + let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) + result = g.loadedOps[op].getOrDefault(key) + #echo "fallback ", key, " ", op, " ", result else: result = nil proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = ## we also need to record this to the packed module. + if not g.attachedOps[op].contains(t.itemId): + let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) + # Use key-based deduplication for opsLog because different type objects + # (e.g. canon vs orig) can have different itemIds but same structural key + if key notin g.loadedOps[op]: + # Hooks should be written to the module where the type is defined, + # not the module that triggered the registration + let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module + g.opsLog.add LogEntry(kind: HookEntry, op: op, module: ownerModule, key: key, sym: value) + g.loadedOps[op][key] = value g.attachedOps[op][t.itemId] = LazySym(sym: value) proc setAttachedOp*(g: ModuleGraph; module: int; typeId: ItemId; op: TTypeAttachedOp; value: PSym) = @@ -414,6 +436,9 @@ proc getToStringProc*(g: ModuleGraph; t: PType): PSym = proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) = g.enumToStringProcs[t.itemId] = LazySym(sym: value) + let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) + let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: value.itemId.module.int + g.opsLog.add LogEntry(kind: EnumToStrEntry, module: ownerModule, key: key, sym: value) iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) = if g.methodsPerGenericType.contains(t.itemId): @@ -422,6 +447,17 @@ iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) = proc addMethodToGeneric*(g: ModuleGraph; module: int; t: PType; col: int; m: PSym) = g.methodsPerGenericType.mgetOrPut(t.itemId, @[]).add (col, LazySym(sym: m)) + let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) + let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module + g.opsLog.add LogEntry(kind: MethodEntry, module: ownerModule, key: key, sym: m) + +proc logGenericInstance*(g: ModuleGraph; inst: PSym) = + ## Log a generic instance so it gets written to the NIF file. + ## This is needed when generic instances are created during compile-time + ## evaluation and may be referenced from other modules compiled in the same run. + if g.config.cmd in {cmdNifC, cmdM}: + let ownerModule = inst.itemId.module.int + g.opsLog.add LogEntry(kind: GenericInstEntry, module: ownerModule, sym: inst) proc hasDisabledAsgn*(g: ModuleGraph; t: PType): bool = let op = getAttachedOp(g, t, attachedAsgn) @@ -783,7 +819,6 @@ when not defined(nimKochBootstrap): ## Returns 'nil' if the module needs to be recompiled. ## Loads module from NIF file when optCompress is enabled. ## When loadFullAst is true, loads the complete module AST for code generation. - if not fileExists(toNifFilename(g.config, fileIdx)): return nil @@ -794,44 +829,29 @@ when not defined(nimKochBootstrap): itemId: ItemId(module: int32(fileIdx), item: 0'i32), name: getIdent(g.cache, splitFile(filename).name), infoImpl: newLineInfo(fileIdx, 1, 1), - positionImpl: int(fileIdx), - ) + positionImpl: int(fileIdx)) setOwner(result, getPackage(g.config, g.cache, fileIdx)) # Register module in graph registerModule(g, result) - var hooks = initTable[nifstreams.SymId, HooksPerType]() - var converters: seq[(string, string)] = @[] - var classes: seq[ClassIndexEntry] = @[] + var opsLog: seq[LogEntry] = @[] result.astImpl = loadNifModule(ast.program, fileIdx, g.ifaces[fileIdx.int].interf, - g.ifaces[fileIdx.int].interfHidden, hooks, converters, classes, loadFullAst) + g.ifaces[fileIdx.int].interfHidden, opsLog, loadFullAst) # Register hooks from NIF index with the module graph - for typSymId, hooksPerType in hooks: - let typeItemId = parseTypeSymIdToItemId(ast.program, typSymId) - if typeItemId.module >= 0: - for op in AttachedOp: - let (hookSymId, isGeneric) = hooksPerType.a[op] - if hookSymId != nifstreams.SymId(0): - let hookSym = resolveHookSym(ast.program, hookSymId) - if hookSym != nil: - setAttachedOp(g, int(fileIdx), typeItemId, toTTypeAttachedOp(op), hookSym) - # Register converters from NIF index with the module's interface - for (destType, convSym) in converters: - let symId = pool.syms.getOrIncl(convSym) - let convPSym = resolveHookSym(ast.program, symId) # reuse hook resolution - if convPSym != nil: - g.ifaces[fileIdx.int].converters.add LazySym(sym: convPSym) + for x in opsLog: + case x.kind + of HookEntry: + g.loadedOps[x.op][x.key] = x.sym + of ConverterEntry: + g.ifaces[fileIdx.int].converters.add LazySym(sym: x.sym) + of MethodEntry: + discard "todo" + of EnumToStrEntry: + discard "todo" + of GenericInstEntry: + raiseAssert "GenericInstEntry should not be in the NIF index" # Register methods per type from NIF index - for classEntry in classes: - let typeItemId = parseTypeSymIdToItemId(ast.program, classEntry.cls) - if typeItemId.module >= 0: - var methodSyms: seq[LazySym] = @[] - for methodEntry in classEntry.methods: - let methodSym = resolveHookSym(ast.program, methodEntry.fn) - if methodSym != nil: - methodSyms.add LazySym(sym: methodSym) - if methodSyms.len > 0: - setMethodsPerType(g, typeItemId, methodSyms) + discard "todo" cachedModules.add fileIdx proc configComplete*(g: ModuleGraph) = diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index 7732844cb1..01c971c9a5 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -52,7 +52,7 @@ proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule = ## Set up a BModule for code generation from a NIF module. if g.backend == nil: g.backend = cgendata.newModuleList(g) - result = cgen.newModule(BModuleList(g.backend), module, g.config) + result = cgen.newModule(BModuleList(g.backend), module, g.config, idGeneratorFromModule(module)) proc generateCodeForModule(g: ModuleGraph; module: PSym) = ## Generate C code for a single module. @@ -82,14 +82,13 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = resetForBackend(g) let mainModule = g.getModule(mainFileIdx) - # Also ensure system module is set up and generated if it exists - if g.systemModule != nil and g.systemModule != mainModule: - let systemBmod = BModuleList(g.backend).modules[g.systemModule.position] - if systemBmod == nil: - discard setupNifBackendModule(g, g.systemModule) - generateCodeForModule(g, g.systemModule) + # Load system module first - it's always needed and contains essential hooks + var cachedModules: seq[FileIndex] = @[] + if g.config.m.systemFileIdx != InvalidFileIdx: + g.systemModule = moduleFromNifFile(g, g.config.m.systemFileIdx, cachedModules) # Load all modules in dependency order using stack traversal + # This must happen BEFORE any code generation so that hooks are loaded into loadedOps let modules = loadModuleDependencies(g, mainFileIdx) if modules.len == 0: rawMessage(g.config, errGenerated, @@ -100,10 +99,17 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = for module in modules: discard setupNifBackendModule(g, module) + # Also ensure system module is set up and generated first if it exists + if g.systemModule != nil and g.systemModule != mainModule: + let systemBmod = BModuleList(g.backend).modules[g.systemModule.position] + if systemBmod == nil: + discard setupNifBackendModule(g, g.systemModule) + generateCodeForModule(g, g.systemModule) + # Generate code for all modules except main (main goes last) # This ensures all modules are added to modulesClosed for module in modules: - if module != mainModule: + if module != mainModule and module != g.systemModule: generateCodeForModule(g, module) # Generate main module last (so all init procs are registered) diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 1c17bae0ba..7834a013c2 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -254,41 +254,8 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator for (m, n) in PCtx(graph.vm).vmstateDiff: if m == module: replayActions.add n - # Collect hooks from the module graph for the current module - var hooks = default array[AttachedOp, seq[HookIndexEntry]] - for op in TTypeAttachedOp: - if op == attachedDeepCopy: continue # Not supported in nimony - let nimonyOp = toAttachedOp(op) - for typeId, lazySym in graph.attachedOps[op]: - if typeId.module == module.position.int32: - let sym = lazySym.sym - if sym != nil: - hooks[nimonyOp].add toHookIndexEntry(graph.config, typeId, sym) - # Collect converters from the module's interface - var converters: seq[(nifstreams.SymId, nifstreams.SymId)] = @[] - for lazySym in graph.ifaces[module.position].converters: - let sym = lazySym.sym - if sym != nil: - let entry = toConverterIndexEntry(graph.config, sym) - if entry[0] != nifstreams.SymId(0): - converters.add entry - # Collect methods per type for classes - var classes: seq[ClassIndexEntry] = @[] - for typeId, methodList in graph.methodsPerType: - if typeId.module == module.position.int32: - var methods: seq[MethodIndexEntry] = @[] - for lazySym in methodList: - let sym = lazySym.sym - if sym != nil: - # Generate a method signature (simplified - name and param count) - let sig = sym.name.s & "/" & $sym.typImpl.sonsImpl.len - methods.add toMethodIndexEntry(graph.config, sym, sig) - if methods.len > 0: - classes.add ClassIndexEntry( - cls: toClassSymId(graph.config, typeId), - methods: methods - ) - writeNifModule(graph.config, module.position.int32, topLevelStmts, hooks, converters, classes, replayActions) + + writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, replayActions) if graph.config.backend notin {backendC, backendCpp, backendObjc} and graph.config.cmd != cmdM: # We only write rod files here if no C-like backend is active. diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 9a2f3ac002..b34c7ef58e 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -450,6 +450,10 @@ proc generateInstance(c: PContext, fn: PSym, pt: LayeredIdTable, entry.compilesId = c.compilesContextId addToGenericProcCache(c, fn, entry) c.generics.add(makeInstPair(fn, entry)) + # Log the generic instance so it gets written to the NIF file. + # This is needed for cyclic module dependencies where generic instances + # may be created in one module but referenced from another. + logGenericInstance(c.graph, result) # bug #12985 bug #22913 # TODO: use the context of the declaration of generic functions instead # TODO: consider fixing options as well diff --git a/compiler/transf.nim b/compiler/transf.nim index 5c56c1997a..124ffa2f78 100644 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -65,16 +65,14 @@ proc newTransNode(a: PNode): PNode {.inline.} = proc newTransNode(kind: TNodeKind, info: TLineInfo, sons: int): PNode {.inline.} = - var x = newNodeI(kind, info) - newSeq(x.sons, sons) - result = x + result = newNodeI(kind, info) + newSeq(result.sons, sons) proc newTransNode(kind: TNodeKind, n: PNode, sons: int): PNode {.inline.} = - var x = newNodeIT(kind, n.info, n.typ) - newSeq(x.sons, sons) -# x.flags = n.flags - result = x + result = newNodeIT(kind, n.info, n.typ) + newSeq(result.sons, sons) + # x.flags = n.flags proc newTransCon(owner: PSym): PTransCon = assert owner != nil @@ -247,6 +245,7 @@ proc hasContinue(n: PNode): bool = case n.kind of nkEmpty..nkNilLit, nkForStmt, nkParForStmt, nkWhileStmt: result = false of nkContinueStmt: result = true + of routineDefs: result = false else: result = false for i in 0.. 0: + c.typeKey(t.skipModifierB, flags, conf) + of tyProc: + withTree c.m, (if tfIterator in t.flagsImpl: "itertype" else: "proctype"): + if CoProc in flags and t.nImpl != nil: + let params = t.nImpl + for i in 1.. 0: + c.typeKey(t.sonsImpl[0], flags, conf) + + c.m.addIdent toNifTag(t.callConvImpl) + if tfVarargs in t.flagsImpl: c.m.addIdent "´varargs" + of tyArray: + withTree c.m, toNifTag(t.kind): + c.typeKey(t.sonsImpl[^1], flags-{CoIgnoreRange}, conf) + c.typeKey(t.sonsImpl[0], flags-{CoIgnoreRange}, conf) + else: + withTree c.m, toNifTag(t.kind): + for i in 1..