diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 623e5eab8f..70fbf1fcb1 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -22,8 +22,9 @@ import "../dist/nimony/src/models" / nifindex_tags import ic / [enum2nif] -# Re-export types needed for hook handling +# Re-export types needed for hook, converter, and method handling export nifindexes.AttachedOp, nifindexes.HookIndexEntry, nifindexes.HooksPerType +export nifindexes.ClassIndexEntry, nifindexes.MethodIndexEntry proc toAttachedOp*(op: TTypeAttachedOp): AttachedOp = ## Maps Nim compiler's TTypeAttachedOp to nimony's AttachedOp. @@ -82,6 +83,23 @@ proc toConverterIndexEntry*(config: ConfigRef; converterSym: PSym): (nifstreams. # Fallback: return empty entry result = (nifstreams.SymId(0), nifstreams.SymId(0)) +proc toMethodIndexEntry*(config: ConfigRef; methodSym: PSym; signature: string): MethodIndexEntry = + ## Converts a method symbol to a MethodIndexEntry. + let methodSymName = methodSym.name.s & "." & $methodSym.disamb & "." & moduleSuffix( + toFullPath(config, methodSym.itemId.module.FileIndex), + cast[seq[string]](config.searchPaths)) + result = MethodIndexEntry( + fn: pool.syms.getOrIncl(methodSymName), + signature: pool.strings.getOrIncl(signature) + ) + +proc toClassSymId*(config: ConfigRef; typeId: ItemId): nifstreams.SymId = + ## Converts a type ItemId to its SymId for the class index. + let typeSymName = "`t" & $typeId.item & "." & moduleSuffix( + toFullPath(config, typeId.module.FileIndex), + cast[seq[string]](config.searchPaths)) + result = pool.syms.getOrIncl(typeSymName) + # ---------------- Line info handling ----------------------------------------- type @@ -654,15 +672,26 @@ proc buildExportBuf(w: var Writer): TokenBuf = result.add identToken(pool.strings.getOrIncl(name), NoLineInfo) result.addParRi() +let replayTag = registerTag("replay") + proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; hooks: array[AttachedOp, seq[HookIndexEntry]]; - converters: seq[(nifstreams.SymId, nifstreams.SymId)]) = + converters: seq[(nifstreams.SymId, nifstreams.SymId)]; + classes: seq[ClassIndexEntry]; + replayActions: seq[PNode] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) let rootInfo = trLineInfo(w, n.info) createStmtList(content, rootInfo) + # Write replay actions first, wrapped in a (replay ...) node + if replayActions.len > 0: + content.addParLe replayTag, rootInfo + for action in replayActions: + writeNode(w, content, action) + content.addParRi() + w.writeToplevelNode content, n content.addParRi() @@ -679,10 +708,10 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; writeFile(dest, d) - # Build index with export, hook, and converter information + # Build index with export, hook, converter, and method information let exportBuf = buildExportBuf(w) createIndex(d, dest[0].info, false, - IndexSections(hooks: hooks, converters: converters, exportBuf: exportBuf)) + IndexSections(hooks: hooks, converters: converters, classes: classes, exportBuf: exportBuf)) # Don't unload symbols/types yet - they may be needed by other modules that haven't # had their NIF files written. For recursive module dependencies (like system.nim), @@ -1403,7 +1432,8 @@ proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym = proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; hooks: var Table[nifstreams.SymId, HooksPerType]; - converters: var seq[(string, string)]): PNode = + converters: var seq[(string, string)]; + classes: var seq[ClassIndexEntry]): PNode = let suffix = moduleSuffix(c.infos.config, f) # Ensure module index is loaded - moduleId returns the FileIndex for this suffix @@ -1417,9 +1447,33 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va hooks = move c.mods[module].index.hooks # Return converters from the index converters = move c.mods[module].index.converters + # Return classes/methods from the index + classes = move c.mods[module].index.classes - # Return empty statement list - actual content is loaded lazily via the index + # Check for replay actions at the start of the NIF file result = newNode(nkStmtList) + let s = addr c.mods[module].stream + s.r.jumpTo 0 # Start from beginning + discard processDirectives(s.r) + var localSyms = initTable[string, PSym]() + # Read root stmts node + var t = next(s[]) + if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): + t = next(s[]) # skip flags + t = next(s[]) # skip type + # Check if first node is a (replay ...) container + if t.kind == ParLe and pool.tags[t.tagId] == "replay": + t = next(s[]) # move past (replay + # Parse all replay actions inside the container + while t.kind != ParRi and t.kind != EofToken: + if t.kind == ParLe: + var buf = createTokenBuf(50) + nifcursors.parse(s[], buf, t.info) + var cursor = cursorAt(buf, 0) + let replayNode = loadNode(c, cursor, suffix, localSyms) + if replayNode != nil: + result.sons.add replayNode + t = next(s[]) when isMainModule: import std / syncio diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index ff9ac4359c..afb98d67c2 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -141,6 +141,7 @@ type cachedFiles*: StringTableRef procGlobals*: seq[PNode] + nifReplayActions*: Table[int32, seq[PNode]] # module position -> replay actions for NIF TPassContext* = object of RootObj # the pass's context idgen*: IdGenerator @@ -398,6 +399,10 @@ proc setMethodsPerType*(g: ModuleGraph; id: ItemId, methods: seq[LazySym]) = # TODO: add it for packed modules g.methodsPerType[id] = methods +proc addNifReplayAction*(g: ModuleGraph; module: int32; n: PNode) = + ## Stores a replay action for NIF-based incremental compilation. + g.nifReplayActions.mgetOrPut(module, @[]).add n + iterator getMethodsPerType*(g: ModuleGraph; t: PType): PSym = if g.methodsPerType.contains(t.itemId): for it in mitems g.methodsPerType[t.itemId]: @@ -772,8 +777,9 @@ when not defined(nimKochBootstrap): registerModule(g, result) var hooks = initTable[nifstreams.SymId, HooksPerType]() var converters: seq[(string, string)] = @[] + var classes: seq[ClassIndexEntry] = @[] result.astImpl = loadNifModule(ast.program, fileIdx, g.ifaces[fileIdx.int].interf, - g.ifaces[fileIdx.int].interfHidden, hooks, converters) + g.ifaces[fileIdx.int].interfHidden, hooks, converters, classes) # Register hooks from NIF index with the module graph for typSymId, hooksPerType in hooks: let typeItemId = parseTypeSymIdToItemId(ast.program, typSymId) @@ -790,6 +796,17 @@ when not defined(nimKochBootstrap): let convPSym = resolveHookSym(ast.program, symId) # reuse hook resolution if convPSym != nil: g.ifaces[fileIdx.int].converters.add LazySym(sym: convPSym) + # Register methods per type from NIF index + for classEntry in classes: + let typeItemId = parseTypeSymIdToItemId(ast.program, classEntry.cls) + if typeItemId.module >= 0: + var methodSyms: seq[LazySym] = @[] + for methodEntry in classEntry.methods: + let methodSym = resolveHookSym(ast.program, methodEntry.fn) + if methodSym != nil: + methodSyms.add LazySym(sym: methodSym) + if methodSyms.len > 0: + setMethodsPerType(g, typeItemId, methodSyms) cachedModules.add fileIdx proc configComplete*(g: ModuleGraph) = diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index de0ab42cea..58f77b6533 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -1,6 +1,6 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs, lineinfos, reorder, options, semdata, cgendata, modules, pathutils, - packages, syntaxes, depends, vm, pragmas, idents, lookups, wordrecg, + packages, syntaxes, depends, vm, vmdef, pragmas, idents, lookups, wordrecg, liftdestructors, nifgen when not defined(nimKochBootstrap): @@ -244,6 +244,16 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator if (optCompress in graph.config.globalOptions or graph.config.cmd == cmdM) and not graph.config.isDefined("nimscript"): topLevelStmts.add finalNode + # Collect replay actions from both pragma computations and VM state diff + var replayActions: seq[PNode] = @[] + # Get pragma-recorded replay actions (compile, link, passC, passL, etc.) + if graph.nifReplayActions.hasKey(module.position.int32): + replayActions.add graph.nifReplayActions[module.position.int32] + # Also get VM state diff (macro cache operations) + if graph.vm != nil: + for (m, n) in PCtx(graph.vm).vmstateDiff: + if m == module: + replayActions.add n # Collect hooks from the module graph for the current module var hooks = default array[AttachedOp, seq[HookIndexEntry]] for op in TTypeAttachedOp: @@ -262,7 +272,23 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator let entry = toConverterIndexEntry(graph.config, sym) if entry[0] != nifstreams.SymId(0): converters.add entry - writeNifModule(graph.config, module.position.int32, topLevelStmts, hooks, converters) + # Collect methods per type for classes + var classes: seq[ClassIndexEntry] = @[] + for typeId, methodList in graph.methodsPerType: + if typeId.module == module.position.int32: + var methods: seq[MethodIndexEntry] = @[] + for lazySym in methodList: + let sym = lazySym.sym + if sym != nil: + # Generate a method signature (simplified - name and param count) + let sig = sym.name.s & "/" & $sym.typImpl.sonsImpl.len + methods.add toMethodIndexEntry(graph.config, sym, sig) + if methods.len > 0: + classes.add ClassIndexEntry( + cls: toClassSymId(graph.config, typeId), + methods: methods + ) + writeNifModule(graph.config, module.position.int32, topLevelStmts, hooks, converters, classes, replayActions) if graph.config.backend notin {backendC, backendCpp, backendObjc} and graph.config.cmd != cmdM: # We only write rod files here if no C-like backend is active. @@ -324,8 +350,10 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF for m in cachedModules: registerModuleById(graph, m) if graph.config.cmd == cmdM: - # cmdM uses NIF files, not ROD files - skip ROD-specific replay - discard + # cmdM uses NIF files - replay from module AST loaded by loadNifModule + let module = graph.getModule(m) + if module != nil and module.ast != nil: + replayStateChanges(module, graph) else: replayStateChanges(graph.packed.pm[m.int].module, graph) replayGenericCacheInformation(graph, m.int) diff --git a/compiler/semdata.nim b/compiler/semdata.nim index 4dc7e3e26a..c29429370e 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -358,6 +358,9 @@ proc addImportFileDep*(c: PContext; f: FileIndex) = proc addPragmaComputation*(c: PContext; n: PNode) = if c.config.symbolFiles != disabledSf: addPragmaComputation(c.encoder, c.packedRepr, n) + # Also store for NIF-based IC (cmdM mode or optCompress) + if optCompress in c.config.globalOptions or c.config.cmd == cmdM: + addNifReplayAction(c.graph, c.module.position.int32, n) proc inclSym(sq: var seq[PSym], s: PSym): bool = for i in 0..