integration logic; WIP

compiler/ast2nif.nim

@@ -805,7 +805,7 @@ template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNod
 
 proc loadNode(c: var DecodeContext; n: var Cursor): PNode =
   result = nil
-  case n.kind:
+  case n.kind
   of DotToken:
     result = nil
     inc n
@@ -904,10 +904,64 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode =
 proc moduleSuffix(conf: ConfigRef; f: FileIndex): string =
   moduleSuffix(toFullPath(conf, f), cast[seq[string]](conf.searchPaths))
 
-proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, inferfHidden: var TStrTable): PNode =
+proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex;
+                           nifName: string; entry: NifIndexEntry): PSym =
+  ## Loads a symbol from the NIF index entry.
+  ## Creates a symbol stub and loads its full definition.
+  let sn = parseSymName(nifName)
+  let val = addr c.mods[module.int32].symCounter
+  inc val[]
+  let id = ItemId(module: module.int32, item: val[])
+
+  # Check if already loaded
+  if c.syms.contains(id):
+    result = c.syms[id][0]
+  else:
+    # Create stub symbol
+    result = PSym(
+      itemId: id,
+      kindImpl: skStub,
+      name: c.cache.getIdent(sn.name),
+      disamb: sn.count.int32,
+      state: Partial
+    )
+    c.syms[id] = (result, entry)
+
+  # Load the full symbol definition if it's still a stub
+  if result.state == Partial:
+    loadSym(c, result)
+
+proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex;
+                                      interf, interfHidden: var TStrTable) =
+  ## Populates interface tables from the NIF index structure.
+  ## Uses the index's public/private tables instead of traversing AST.
+  let idx = addr c.mods[module.int32].index
+
+  # Add all public symbols to interf (exported interface) and interfHidden
+  for nifName, entry in idx.public:
+    let sym = loadSymFromIndexEntry(c, module, nifName, entry)
+    if sym != nil:
+      strTableAdd(interf, sym)
+      strTableAdd(interfHidden, sym)
+
+  when false:
+    # Add private symbols to interfHidden only
+    for nifName, entry in idx.private:
+      let sym = loadSymFromIndexEntry(c, module, nifName, entry)
+      if sym != nil:
+        strTableAdd(interfHidden, sym)
+
+proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable): PNode =
   let suffix = moduleSuffix(c.infos.config, f)
   let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(suffix), ".nif").string
 
+  # Ensure module index is loaded - moduleId returns the FileIndex for this suffix
+  let module = moduleId(c, suffix)
+
+  # Populate interface tables from the NIF index structure
+  # Use the FileIndex returned by moduleId to ensure we access the correct index
+  populateInterfaceTablesFromIndex(c, module, interf, interfHidden)
+
   var buf = createTokenBuf(300)
   var s = nifstreams.open(modFile)
   # XXX We can optimize this here and only load the top level entries!

compiler/modulegraphs.nim

@@ -13,9 +13,12 @@
 
 import std/[intsets, tables, hashes, strtabs, algorithm, os, strutils, parseutils]
 import ../dist/checksums/src/checksums/md5
+import ../dist/checksums/src/checksums/sha1
 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb
 import ic / [packed_ast, ic]
 
+when not defined(nimKochBootstrap):
+  import ast2nif
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -741,8 +744,36 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex;
   else:
     result = nil
 
+when not defined(nimKochBootstrap):
+  var gDecodeContext {.threadvar.}: DecodeContext
+
+  proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex;
+                          cachedModules: var seq[FileIndex]): PSym =
+    ## Returns 'nil' if the module needs to be recompiled.
+    ## Loads module from NIF file when optCompress is enabled.
+
+    # loadNifModule will check if the file exists internally
+
+    # Create module symbol
+    let filename = AbsoluteFile toFullPath(g.config, fileIdx)
+    result = PSym(
+      kindImpl: skModule,
+      itemId: ItemId(module: int32(fileIdx), item: 0'i32),
+      name: getIdent(g.cache, splitFile(filename).name),
+      infoImpl: newLineInfo(fileIdx, 1, 1),
+      positionImpl: int(fileIdx),
+    )
+    setOwner(result, getPackage(g.config, g.cache, fileIdx))
+
+    # Register module in graph
+    registerModule(g, result)
+    result.astImpl = loadNifModule(gDecodeContext, fileIdx, g.ifaces[fileIdx.int].interf, g.ifaces[fileIdx.int].interfHidden)
+    cachedModules.add fileIdx
+
 proc configComplete*(g: ModuleGraph) =
   rememberStartupConfig(g.startupPackedConfig, g.config)
+  when not defined(nimKochBootstrap):
+    gDecodeContext = createDecodeContext(g.config, g.cache)
 
 proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) =
   let conf = graph.config

compiler/pipelines.nim

@@ -260,7 +260,13 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
     discard processPipelineModule(graph, result, idGeneratorFromModule(result), s)
   if result == nil:
     var cachedModules: seq[FileIndex] = @[]
-    result = moduleFromRodFile(graph, fileIdx, cachedModules)
+    when not defined(nimKochBootstrap):
+      # Try loading from NIF file first if optCompress is enabled
+      if optCompress in graph.config.globalOptions and not graph.config.isDefined("nimscript"):
+        result = moduleFromNifFile(graph, fileIdx, cachedModules)
+    if result == nil:
+      # Fall back to ROD file loading
+      result = moduleFromRodFile(graph, fileIdx, cachedModules)
     let path = toFullPath(graph.config, fileIdx)
     let filename = AbsoluteFile path
     # it could be a stdinfile/cmdfile