From 5f81a4ac45d8df0ce0c07efcadc3d3a67617e8d8 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 25 Dec 2025 06:58:43 +0100 Subject: [PATCH] API change --- compiler/ast2nif.nim | 65 +++++++++++++++++++-------------------- compiler/modulegraphs.nim | 25 +++++++-------- compiler/nifbackend.nim | 48 ++++++++++++++--------------- compiler/pipelines.nim | 7 +++-- 4 files changed, 72 insertions(+), 73 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 6d8aea4ae8..197b68b349 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -1501,15 +1501,23 @@ proc nextSubtree(r: var Stream; dest: var TokenBuf; tok: var PackedToken) = elif tok.kind == ParRi: dec nested if nested == 0: break +type + ModuleSuffix* = distinct string + PrecompiledModule* = object + topLevel*: PNode # top level statements of the main module + deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of + logOps*: seq[LogEntry] + module*: PSym # set by modulegraphs.nim! -proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[string]; tok: var PackedToken) = +proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = tok = next(s) # skip `(import` if tok.kind == DotToken: tok = next(s) # skip dot if tok.kind == DotToken: tok = next(s) # skip dot if tok.kind == StringLit: - deps.add pool.strings[tok.litId] + deps.add ModuleSuffix(pool.strings[tok.litId]) + tok = next(s) else: raiseAssert "expected StringLit but got " & $tok.kind if tok.kind == ParRi: @@ -1517,13 +1525,8 @@ proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[string]; tok: else: raiseAssert "expected ParRi but got " & $tok.kind -type - NifGraph = object - topLevel: PNode # top level statements of the main module - deps: seq[string] # other modules we need to process the top level statements of - -proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suffix: string; logOps: var seq[LogEntry]; module: int): NifGraph = - result = NifGraph(topLevel: newNode(nkStmtList)) +proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suffix: string; module: int): PrecompiledModule = + result = PrecompiledModule(topLevel: newNode(nkStmtList)) var localSyms = initTable[string, PSym]() var t = next(s) # skip dot @@ -1547,25 +1550,25 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suf else: raiseAssert "expected ParRi but got " & $t.kind elif t.tagId == repConverterTag: - t = loadLogOp(c, logOps, s, ConverterEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, ConverterEntry, attachedTrace, module) elif t.tagId == repDestroyTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedDestructor, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedDestructor, module) elif t.tagId == repWasMovedTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedWasMoved, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedWasMoved, module) elif t.tagId == repCopyTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedAsgn, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedAsgn, module) elif t.tagId == repSinkTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedSink, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedSink, module) elif t.tagId == repDupTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedDup, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedDup, module) elif t.tagId == repTraceTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedTrace, module) elif t.tagId == repDeepCopyTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedDeepCopy, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedDeepCopy, module) elif t.tagId == repEnumToStrTag: - t = loadLogOp(c, logOps, s, EnumToStrEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, EnumToStrEntry, attachedTrace, module) elif t.tagId == repMethodTag: - t = loadLogOp(c, logOps, s, MethodEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module) #elif t.tagId == repClassTag: # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) elif t.tagId == includeTag: @@ -1587,17 +1590,14 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suf else: cont = false -proc loadNifGraph*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; - logOps: var seq[LogEntry]; - loadFullAst: bool = false): NifGraph = - let suffix = moduleSuffix(c.infos.config, f) - - # Ensure module index is loaded - moduleId returns the FileIndex for this suffix - let module = moduleId(c, suffix) +proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable; + loadFullAst: bool = false): PrecompiledModule = + # Ensure module index is loaded - moduleId returns the FileIndex for this suffix + let module = moduleId(c, string(suffix)) # Populate interface tables from the NIF index structure # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym - populateInterfaceTablesFromIndex(c, module, interf, interfHidden, suffix) + populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix)) # Load the module AST (or just replay actions if loadFullAst is false) let s = addr c.mods[module].stream @@ -1607,15 +1607,14 @@ proc loadNifGraph*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): t = next(s[]) # skip (stmts t = next(s[]) # skip flags - result = processTopLevel(c, s[], loadFullAst, suffix, logOps, f.int) + result = processTopLevel(c, s[], loadFullAst, string(suffix), module.int) else: - result = NifGraph(topLevel: newNode(nkStmtList)) + result = PrecompiledModule(topLevel: newNode(nkStmtList)) proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; - logOps: var seq[LogEntry]; - loadFullAst: bool = false): PNode = - let g = loadNifGraph(c, f, interf, interfHidden, logOps, loadFullAst) - result = g.topLevel + loadFullAst: bool = false): PrecompiledModule = + let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f)) + result = loadNifModule(c, suffix, interf, interfHidden, loadFullAst) when isMainModule: import std / syncio diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index d338194ea5..68488dea3a 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -814,31 +814,33 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; when not defined(nimKochBootstrap): proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex; - cachedModules: var seq[FileIndex]; - loadFullAst: bool = false): PSym = + loadFullAst: bool = false): PrecompiledModule = ## Returns 'nil' if the module needs to be recompiled. ## Loads module from NIF file when optCompress is enabled. ## When loadFullAst is true, loads the complete module AST for code generation. if not fileExists(toNifFilename(g.config, fileIdx)): - return nil + return PrecompiledModule(module: nil) # Create module symbol let filename = AbsoluteFile toFullPath(g.config, fileIdx) - result = PSym( + + let m = PSym( kindImpl: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32), name: getIdent(g.cache, splitFile(filename).name), infoImpl: newLineInfo(fileIdx, 1, 1), positionImpl: int(fileIdx)) - setOwner(result, getPackage(g.config, g.cache, fileIdx)) - + setOwner(m, getPackage(g.config, g.cache, fileIdx)) # Register module in graph - registerModule(g, result) - var opsLog: seq[LogEntry] = @[] - result.astImpl = loadNifModule(ast.program, fileIdx, g.ifaces[fileIdx.int].interf, - g.ifaces[fileIdx.int].interfHidden, opsLog, loadFullAst) + registerModule(g, m) + + result = loadNifModule(ast.program, fileIdx, + g.ifaces[fileIdx.int].interf, + g.ifaces[fileIdx.int].interfHidden, loadFullAst) + result.module = m + # Register hooks from NIF index with the module graph - for x in opsLog: + for x in result.logOps: case x.kind of HookEntry: g.loadedOps[x.op][x.key] = x.sym @@ -852,7 +854,6 @@ when not defined(nimKochBootstrap): raiseAssert "GenericInstEntry should not be in the NIF index" # Register methods per type from NIF index discard "todo" - cachedModules.add fileIdx proc configComplete*(g: ModuleGraph) = rememberStartupConfig(g.startupPackedConfig, g.config) diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index 5eb0bab532..960b9d392c 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -28,27 +28,30 @@ import ast, options, lineinfos, modulegraphs, cgendata, cgen, proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[PSym] = ## Traverse the module dependency graph using a stack. ## Returns all modules that need code generation, in dependency order. - var visited = initIntSet() - var stack: seq[FileIndex] = @[mainFileIdx] + let precomp = moduleFromNifFile(g, mainFileIdx, loadFullAst=true) + + var stack: seq[ModuleSuffix] = @[] result = @[] - var cachedModules: seq[FileIndex] = @[] + + if precomp.module != nil: + result.add precomp.module + incl precomp.module.flagsImpl, sfMainModule + for dep in precomp.deps: + stack.add dep + + var visited = initHashSet[string]() while stack.len > 0: - let fileIdx = stack.pop() + let suffix = stack.pop() - if not visited.containsOrIncl(int(fileIdx)): - # Only load full AST for main module; others are loaded lazily by codegen - let isMainModule = fileIdx == mainFileIdx - let module = moduleFromNifFile(g, fileIdx, cachedModules, loadFullAst=isMainModule) - if module != nil: - result.add module - if isMainModule: - incl module.flagsImpl, sfMainModule - # Add dependencies to stack (they come from cachedModules) - for dep in cachedModules: - if not visited.contains(int(dep)): + if not visited.containsOrIncl(suffix.string): + let fileIdx = msgs.fileInfoIdx(g.config, AbsoluteFile suffix.string) + let precomp = moduleFromNifFile(g, fileIdx, loadFullAst=true) + if precomp.module != nil: + result.add precomp.module + for dep in precomp.deps: + if not visited.contains(dep.string): stack.add dep - cachedModules.setLen(0) proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule = ## Set up a BModule for code generation from a NIF module. @@ -85,12 +88,11 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = # Reset backend state resetForBackend(g) - let mainModule = g.getModule(mainFileIdx) # Load system module first - it's always needed and contains essential hooks - var cachedModules: seq[FileIndex] = @[] if g.config.m.systemFileIdx != InvalidFileIdx: - g.systemModule = moduleFromNifFile(g, g.config.m.systemFileIdx, cachedModules) + let precomp = moduleFromNifFile(g, g.config.m.systemFileIdx) + g.systemModule = precomp.module # Load all modules in dependency order using stack traversal # This must happen BEFORE any code generation so that hooks are loaded into loadedOps @@ -105,7 +107,7 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = discard setupNifBackendModule(g, module) # Also ensure system module is set up and generated first if it exists - if g.systemModule != nil and g.systemModule != mainModule: + if g.systemModule != nil: let systemBmod = BModuleList(g.backend).modules[g.systemModule.position] if systemBmod == nil: discard setupNifBackendModule(g, g.systemModule) @@ -115,13 +117,9 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = # This ensures all modules are added to modulesClosed for module in modules: - if module != mainModule and module != g.systemModule: + if module != g.systemModule: generateCodeForModule(g, module) - # Generate main module last (so all init procs are registered) - if mainModule != nil: - generateCodeForModule(g, mainModule) - for m in BModuleList(g.backend).modules: if m != nil: assert m.module != nil diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 7834a013c2..989f9c2d9a 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -286,8 +286,8 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF sfMainModule notin flags and not graph.withinSystem and not graph.config.isDefined("nimscript"): - result = moduleFromNifFile(graph, fileIdx, cachedModules) - if result == nil: + let precomp = moduleFromNifFile(graph, fileIdx) + if precomp.module == nil: let nifPath = toNifFilename(graph.config, fileIdx) localError(graph.config, unknownLineInfo, "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) & @@ -385,7 +385,8 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx graph.config.libpath / RelativeFile"system.nim") var cachedModules: seq[FileIndex] = @[] when not defined(nimKochBootstrap): - graph.systemModule = moduleFromNifFile(graph, graph.config.m.systemFileIdx, cachedModules) + let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx) + graph.systemModule = precomp.module if graph.systemModule == nil: let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx) localError(graph.config, unknownLineInfo,