API change

compiler/ast2nif.nim

@@ -1501,15 +1501,23 @@ proc nextSubtree(r: var Stream; dest: var TokenBuf; tok: var PackedToken) =
     elif tok.kind == ParRi:
       dec nested
       if nested == 0: break
+type
+  ModuleSuffix* = distinct string
+  PrecompiledModule* = object
+    topLevel*: PNode # top level statements of the main module
+    deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of
+    logOps*: seq[LogEntry]
+    module*: PSym # set by modulegraphs.nim!
 
-proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[string]; tok: var PackedToken) =
+proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) =
   tok = next(s) # skip `(import`
   if tok.kind == DotToken:
     tok = next(s) # skip dot
     if tok.kind == DotToken:
       tok = next(s) # skip dot
   if tok.kind == StringLit:
-    deps.add pool.strings[tok.litId]
+    deps.add ModuleSuffix(pool.strings[tok.litId])
+    tok = next(s)
   else:
     raiseAssert "expected StringLit but got " & $tok.kind
   if tok.kind == ParRi:
@@ -1517,13 +1525,8 @@ proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[string]; tok:
   else:
     raiseAssert "expected ParRi but got " & $tok.kind
 
-type
-  NifGraph = object
-    topLevel: PNode # top level statements of the main module
-    deps: seq[string] # other modules we need to process the top level statements of
-
-proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suffix: string; logOps: var seq[LogEntry]; module: int): NifGraph =
-  result = NifGraph(topLevel: newNode(nkStmtList))
+proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suffix: string; module: int): PrecompiledModule =
+  result = PrecompiledModule(topLevel: newNode(nkStmtList))
   var localSyms = initTable[string, PSym]()
 
   var t = next(s) # skip dot
@@ -1547,25 +1550,25 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suf
         else:
           raiseAssert "expected ParRi but got " & $t.kind
       elif t.tagId == repConverterTag:
-        t = loadLogOp(c, logOps, s, ConverterEntry, attachedTrace, module)
+        t = loadLogOp(c, result.logOps, s, ConverterEntry, attachedTrace, module)
       elif t.tagId == repDestroyTag:
-        t = loadLogOp(c, logOps, s, HookEntry, attachedDestructor, module)
+        t = loadLogOp(c, result.logOps, s, HookEntry, attachedDestructor, module)
       elif t.tagId == repWasMovedTag:
-        t = loadLogOp(c, logOps, s, HookEntry, attachedWasMoved, module)
+        t = loadLogOp(c, result.logOps, s, HookEntry, attachedWasMoved, module)
       elif t.tagId == repCopyTag:
-        t = loadLogOp(c, logOps, s, HookEntry, attachedAsgn, module)
+        t = loadLogOp(c, result.logOps, s, HookEntry, attachedAsgn, module)
       elif t.tagId == repSinkTag:
-        t = loadLogOp(c, logOps, s, HookEntry, attachedSink, module)
+        t = loadLogOp(c, result.logOps, s, HookEntry, attachedSink, module)
       elif t.tagId == repDupTag:
-        t = loadLogOp(c, logOps, s, HookEntry, attachedDup, module)
+        t = loadLogOp(c, result.logOps, s, HookEntry, attachedDup, module)
       elif t.tagId == repTraceTag:
-        t = loadLogOp(c, logOps, s, HookEntry, attachedTrace, module)
+        t = loadLogOp(c, result.logOps, s, HookEntry, attachedTrace, module)
       elif t.tagId == repDeepCopyTag:
-        t = loadLogOp(c, logOps, s, HookEntry, attachedDeepCopy, module)
+        t = loadLogOp(c, result.logOps, s, HookEntry, attachedDeepCopy, module)
       elif t.tagId == repEnumToStrTag:
-        t = loadLogOp(c, logOps, s, EnumToStrEntry, attachedTrace, module)
+        t = loadLogOp(c, result.logOps, s, EnumToStrEntry, attachedTrace, module)
       elif t.tagId == repMethodTag:
-        t = loadLogOp(c, logOps, s, MethodEntry, attachedTrace, module)
+        t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module)
         #elif t.tagId == repClassTag:
         #  t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module)
       elif t.tagId == includeTag:
@@ -1587,17 +1590,14 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suf
     else:
       cont = false
 
-proc loadNifGraph*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable;
-                    logOps: var seq[LogEntry];
-                    loadFullAst: bool = false): NifGraph =
-  let suffix = moduleSuffix(c.infos.config, f)
-
-  # Ensure module index is loaded - moduleId returns the FileIndex for this suffix
-  let module = moduleId(c, suffix)
+proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable;
+                    loadFullAst: bool = false): PrecompiledModule =
+    # Ensure module index is loaded - moduleId returns the FileIndex for this suffix
+  let module = moduleId(c, string(suffix))
 
   # Populate interface tables from the NIF index structure
   # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym
-  populateInterfaceTablesFromIndex(c, module, interf, interfHidden, suffix)
+  populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix))
 
   # Load the module AST (or just replay actions if loadFullAst is false)
   let s = addr c.mods[module].stream
@@ -1607,15 +1607,14 @@ proc loadNifGraph*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var
   if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList):
     t = next(s[])  # skip (stmts
     t = next(s[])  # skip flags
-    result = processTopLevel(c, s[], loadFullAst, suffix, logOps, f.int)
+    result = processTopLevel(c, s[], loadFullAst, string(suffix), module.int)
   else:
-    result = NifGraph(topLevel: newNode(nkStmtList))
+    result = PrecompiledModule(topLevel: newNode(nkStmtList))
 
 proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable;
-                    logOps: var seq[LogEntry];
-                    loadFullAst: bool = false): PNode =
-  let g = loadNifGraph(c, f, interf, interfHidden, logOps, loadFullAst)
-  result = g.topLevel
+                    loadFullAst: bool = false): PrecompiledModule =
+  let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f))
+  result = loadNifModule(c, suffix, interf, interfHidden, loadFullAst)
 
 when isMainModule:
   import std / syncio

compiler/modulegraphs.nim

@@ -814,31 +814,33 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex;
 
 when not defined(nimKochBootstrap):
   proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex;
-                          cachedModules: var seq[FileIndex];
-                          loadFullAst: bool = false): PSym =
+                          loadFullAst: bool = false): PrecompiledModule =
     ## Returns 'nil' if the module needs to be recompiled.
     ## Loads module from NIF file when optCompress is enabled.
     ## When loadFullAst is true, loads the complete module AST for code generation.
     if not fileExists(toNifFilename(g.config, fileIdx)):
-      return nil
+      return PrecompiledModule(module: nil)
 
     # Create module symbol
     let filename = AbsoluteFile toFullPath(g.config, fileIdx)
-    result = PSym(
+
+    let m = PSym(
       kindImpl: skModule,
       itemId: ItemId(module: int32(fileIdx), item: 0'i32),
       name: getIdent(g.cache, splitFile(filename).name),
       infoImpl: newLineInfo(fileIdx, 1, 1),
       positionImpl: int(fileIdx))
-    setOwner(result, getPackage(g.config, g.cache, fileIdx))
-
+    setOwner(m, getPackage(g.config, g.cache, fileIdx))
     # Register module in graph
-    registerModule(g, result)
-    var opsLog: seq[LogEntry] = @[]
-    result.astImpl = loadNifModule(ast.program, fileIdx, g.ifaces[fileIdx.int].interf,
-                                   g.ifaces[fileIdx.int].interfHidden, opsLog, loadFullAst)
+    registerModule(g, m)
+
+    result = loadNifModule(ast.program, fileIdx,
+                           g.ifaces[fileIdx.int].interf,
+                           g.ifaces[fileIdx.int].interfHidden, loadFullAst)
+    result.module = m
+
     # Register hooks from NIF index with the module graph
-    for x in opsLog:
+    for x in result.logOps:
       case x.kind
       of HookEntry:
         g.loadedOps[x.op][x.key] = x.sym
@@ -852,7 +854,6 @@ when not defined(nimKochBootstrap):
         raiseAssert "GenericInstEntry should not be in the NIF index"
     # Register methods per type from NIF index
     discard "todo"
-    cachedModules.add fileIdx
 
 proc configComplete*(g: ModuleGraph) =
   rememberStartupConfig(g.startupPackedConfig, g.config)

compiler/nifbackend.nim

@@ -28,27 +28,30 @@ import ast, options, lineinfos, modulegraphs, cgendata, cgen,
 proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[PSym] =
   ## Traverse the module dependency graph using a stack.
   ## Returns all modules that need code generation, in dependency order.
-  var visited = initIntSet()
-  var stack: seq[FileIndex] = @[mainFileIdx]
+  let precomp = moduleFromNifFile(g, mainFileIdx, loadFullAst=true)
+
+  var stack: seq[ModuleSuffix] = @[]
   result = @[]
-  var cachedModules: seq[FileIndex] = @[]
+
+  if precomp.module != nil:
+    result.add precomp.module
+    incl precomp.module.flagsImpl, sfMainModule
+    for dep in precomp.deps:
+      stack.add dep
+
+  var visited = initHashSet[string]()
 
   while stack.len > 0:
-    let fileIdx = stack.pop()
+    let suffix = stack.pop()
 
-    if not visited.containsOrIncl(int(fileIdx)):
-      # Only load full AST for main module; others are loaded lazily by codegen
-      let isMainModule = fileIdx == mainFileIdx
-      let module = moduleFromNifFile(g, fileIdx, cachedModules, loadFullAst=isMainModule)
-      if module != nil:
-        result.add module
-        if isMainModule:
-          incl module.flagsImpl, sfMainModule
-        # Add dependencies to stack (they come from cachedModules)
-        for dep in cachedModules:
-          if not visited.contains(int(dep)):
+    if not visited.containsOrIncl(suffix.string):
+      let fileIdx = msgs.fileInfoIdx(g.config, AbsoluteFile suffix.string)
+      let precomp = moduleFromNifFile(g, fileIdx, loadFullAst=true)
+      if precomp.module != nil:
+        result.add precomp.module
+        for dep in precomp.deps:
+          if not visited.contains(dep.string):
             stack.add dep
-        cachedModules.setLen(0)
 
 proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule =
   ## Set up a BModule for code generation from a NIF module.
@@ -85,12 +88,11 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
 
   # Reset backend state
   resetForBackend(g)
-  let mainModule = g.getModule(mainFileIdx)
 
   # Load system module first - it's always needed and contains essential hooks
-  var cachedModules: seq[FileIndex] = @[]
   if g.config.m.systemFileIdx != InvalidFileIdx:
-    g.systemModule = moduleFromNifFile(g, g.config.m.systemFileIdx, cachedModules)
+    let precomp = moduleFromNifFile(g, g.config.m.systemFileIdx)
+    g.systemModule = precomp.module
 
   # Load all modules in dependency order using stack traversal
   # This must happen BEFORE any code generation so that hooks are loaded into loadedOps
@@ -105,7 +107,7 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
     discard setupNifBackendModule(g, module)
 
   # Also ensure system module is set up and generated first if it exists
-  if g.systemModule != nil and g.systemModule != mainModule:
+  if g.systemModule != nil:
     let systemBmod = BModuleList(g.backend).modules[g.systemModule.position]
     if systemBmod == nil:
       discard setupNifBackendModule(g, g.systemModule)
@@ -115,13 +117,9 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) =
   # This ensures all modules are added to modulesClosed
 
   for module in modules:
-    if module != mainModule and module != g.systemModule:
+    if module != g.systemModule:
       generateCodeForModule(g, module)
 
-  # Generate main module last (so all init procs are registered)
-  if mainModule != nil:
-    generateCodeForModule(g, mainModule)
-
   for m in BModuleList(g.backend).modules:
     if m != nil:
       assert m.module != nil

compiler/pipelines.nim

@@ -286,8 +286,8 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
          sfMainModule notin flags and
          not graph.withinSystem and
          not graph.config.isDefined("nimscript"):
-        result = moduleFromNifFile(graph, fileIdx, cachedModules)
-        if result == nil:
+        let precomp = moduleFromNifFile(graph, fileIdx)
+        if precomp.module == nil:
           let nifPath = toNifFilename(graph.config, fileIdx)
           localError(graph.config, unknownLineInfo,
             "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) &
@@ -385,7 +385,8 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx
         graph.config.libpath / RelativeFile"system.nim")
     var cachedModules: seq[FileIndex] = @[]
     when not defined(nimKochBootstrap):
-      graph.systemModule = moduleFromNifFile(graph, graph.config.m.systemFileIdx, cachedModules)
+      let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx)
+      graph.systemModule = precomp.module
       if graph.systemModule == nil:
         let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx)
         localError(graph.config, unknownLineInfo,