IC: huge progress

compiler/ic/replayer.nim

@@ -19,8 +19,11 @@ import std/tables
 when defined(nimPreviewSlimSystem):
   import std/assertions
 
-proc replayStateChanges*(module: PSym; g: ModuleGraph) =
-  let list = module.ast
+proc replayStateChanges*(module: PSym; g: ModuleGraph; list: PNode) =
+  ## `list` is an `nkStmtList` of `nkReplayAction` nodes (macro-cache puts/incs/
+  ## adds/incls and a few pragmas) recorded for `module`. Under the NIF backend a
+  ## loaded module's `ast` is never reconstructed, so the caller passes the replay
+  ## actions it parsed out of the module's NIF directly.
   assert list != nil
   assert list.kind == nkStmtList
   for n in list:
@@ -64,8 +67,9 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) =
           g.cacheTables[destKey] = initBTree[string, PNode]()
         if not contains(g.cacheTables[destKey], key):
           g.cacheTables[destKey].add(key, val)
-        else:
-          internalError(g.config, n.info, "key already exists: " & key)
+        # else: the same key was already replayed. Under IC the import closure is
+        # replayed (direct module + transitive deps), so the same registration can
+        # legitimately be reached twice; re-applying it is a no-op, not an error.
       of "incl":
         let destKey = n[1].strVal
         let val = n[2]

compiler/modulegraphs.nim

@@ -146,6 +146,11 @@ type
     cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API; IC: implemented
     cacheCounters*: Table[string, BiggestInt] # IC: implemented
     cacheTables*: Table[string, BTree[string, PNode]] # IC: implemented
+    transitiveReplayActions*: seq[PNode] # macro-cache replay actions collected from
+      # the transitive import closure of a NIF-loaded module (loadTransitiveHooks);
+      # the caller (pipelines) replays them so a dependency's macrocache state — e.g.
+      # nim-serialization's flavor registration — reaches a module that imports it
+      # only indirectly. Drained per moduleFromNifFile call.
     passes*: seq[TPass]
     pipelinePass*: PipelinePass
     onDefinition*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
@@ -944,6 +949,14 @@ when not defined(nimKochBootstrap):
       if not g.hookClosure.containsOrIncl(fileIdx.int):
         let precomp = loadNifModule(ast.program, suffix, interf, interfHidden, {})
         registerLoadedHooks(g, precomp.logOps)
+        # Collect the dependency's macro-cache replay actions (put/inc/add/incl)
+        # so the importer being compiled also sees macrocache state registered
+        # by a transitively-imported module. Pragma replay actions are a backend
+        # concern and are intentionally not collected here.
+        for n in precomp.topLevel:
+          if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
+             n[0].strVal in ["put", "inc", "add", "incl"]:
+            g.transitiveReplayActions.add n
         for d in precomp.deps: stack.add d
 
   proc materializeReexportedModule(g: ModuleGraph; mname, msuffix: string): PSym =

compiler/pipelines.nim

@@ -378,9 +378,30 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF
           if sfSystemModule in flags:
             graph.systemModule = result
           partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx)))
-          # Replay state changes from the loaded NIF module
-          if result.ast != nil:
-            replayStateChanges(result, graph)
+          # Replay the module's recorded state changes: macro-cache operations
+          # (std/macrocache puts/incs/adds/incls) plus a few pragmas. The loader
+          # parsed them into `precomp.topLevel` (mixed with other top-level nodes),
+          # so filter to the replay actions. A loaded module's `ast` is never
+          # rebuilt, so this used to be skipped (`result.ast == nil`) and a
+          # NIF-loaded module's macro cache was lost — e.g. nim-serialization's
+          # flavor registration became invisible to dependents (`DefaultFlavor:
+          # automatic serialization is not enabled`).
+          var replayList = newNodeI(nkStmtList, result.info)
+          for n in precomp.topLevel:
+            # Only macro-cache ops (put/inc/add/incl). The pragma replay actions
+            # (compile/link/passc/hint/...) are a backend/link concern handled by
+            # the nifc closure, and re-emitting a loaded module's hints/warnings on
+            # every import would be wrong — so they are deliberately skipped here.
+            if n.kind == nkReplayAction and n.len >= 1 and n[0].kind == nkStrLit and
+               n[0].strVal in ["put", "inc", "add", "incl"]:
+              replayList.add n
+          # Plus the macro-cache actions of the module's transitive import closure
+          # (collected by the moduleFromNifFile call above via loadTransitiveHooks),
+          # so a flavor/type registered in an indirectly-imported module is visible.
+          for n in graph.transitiveReplayActions: replayList.add n
+          graph.transitiveReplayActions.setLen 0
+          if replayList.len > 0:
+            replayStateChanges(result, graph, replayList)
           # Fill the VM slots of the module's `{.compileTime.}` globals now (sem
           # would have, but a NIF-loaded module is never semchecked).
           initLoadedCompileTimeGlobals(graph, result, precomp.topLevel)

compiler/semexprs.nim

@@ -1905,6 +1905,22 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode =
     n.typ = n.typ.elementType
   result.add(n)
 
+proc markResultVarIsPtr(c: PContext, x: PNode) {.inline.} =
+  ## Set `tfVarIsPtr` on the (result) sym node's type. Under IC that type can be a
+  ## NIF-loaded (Sealed) and interned instance which must not be mutated in place
+  ## (it could corrupt other users of the shared type, and the assert forbids it):
+  ## give this result its own copy carrying the flag, exactly like a from-source
+  ## compile has a fresh result type here.
+  if tfVarIsPtr in x.typ.flags: return
+  if x.typ.state == Sealed:
+    let fresh = copyType(x.typ, c.idgen, x.typ.owner)
+    fresh.incl tfVarIsPtr
+    x.typ = fresh
+    if x.kind == nkSym and x.sym.state != Sealed:
+      x.sym.typ = fresh
+  else:
+    x.typ.incl tfVarIsPtr
+
 proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
   if le.kind == nkHiddenDeref:
     var x = le[0]
@@ -1912,10 +1928,10 @@ proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
       if x.sym.kind == skResult and (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView):
         n[0] = x # 'result[]' --> 'result'
         n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent)
-        x.typ.incl tfVarIsPtr
+        markResultVarIsPtr(c, x)
         #echo x.info, " setting it for this type ", typeToString(x.typ), " ", n.info
       elif sfGlobal in x.sym.flags:
-        x.typ.incl tfVarIsPtr
+        markResultVarIsPtr(c, x)
 
 proc borrowCheck(c: PContext, n, le, ri: PNode) =
   const

tests/ic/mctglobal.nim

@@ -0,0 +1,28 @@
+# Helper module for tcompiletimeglobal.nim (not a test itself; no `discard`).
+#
+# Exercises `{.compileTime.}` module-level globals across the NIF boundary: under
+# `nim ic` this module is compiled to its own NIF and *loaded* (not semchecked)
+# by the importer, so its compile-time globals must be eagerly initialized at
+# load time. A macro that splices such a global into a `quote do:` otherwise
+# reads a nil VM slot.
+
+import macros
+
+let injectedName {.compileTime.} = ident "ctgValue"
+
+proc genAssign*(): NimNode =
+  ## The order-fragile case: the CT global is read inside a proc that the macro
+  ## calls, not in the macro's own `quote`. The lazy VM init attaches to the
+  ## first vmgen'd reference, which need not be the first one executed.
+  result = quote do:
+    `injectedName` = 42
+
+macro defineCtgValue*(): untyped =
+  result = newStmtList()
+  # Read the global via the helper proc FIRST, then via the macro's own quote,
+  # so the proc's reference executes before the macro's: only eager init at load
+  # time makes both see the initialized value.
+  let assign = genAssign()
+  result.add quote do:
+    var `injectedName`: int
+  result.add assign

tests/ic/tcompiletimeglobal.nim

@@ -0,0 +1,16 @@
+discard """
+output: '''42'''
+"""
+
+# Regression test: `{.compileTime.}` module-level globals of a NIF-loaded module
+# must be initialized so macros/compile-time procs that splice them produce valid
+# code. Before the eager-init fix this failed under `nim ic` with
+# "attempt to access a nil address" / "illformed AST: break nil" /
+# "undeclared identifier". `koch ic` only checks the compile succeeds; a nil
+# global makes `defineCtgValue` emit `var <nil>: int` / `<nil> = 42` and the
+# compile fails.
+
+import mctglobal
+
+defineCtgValue()
+echo ctgValue