IC: compilerprocs are handled correctly (#17265)

* IC: compilerprocs are handled correctly * IC: special logic for .exportC'ed procs * IC: 'hello world' compiles for the first round (2nd round fails) * IC: hello world seems to work
2026-02-12 22:33:49 +00:00 · 2021-03-09 00:04:39 +01:00
parent 761ec2ccc8
commit 6cb26d8010
8 changed files with 120 additions and 39 deletions
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -2707,7 +2707,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
      if sfCompileTime in sym.flags:
        localError(p.config, n.info, "request to generate code for .compileTime proc: " &
           sym.name.s)
-      if useAliveDataFromDce in p.module.flags:
+      if useAliveDataFromDce in p.module.flags and sym.typ.callConv != ccInline:
        fillProcLoc(p.module, n)
        genProcPrototype(p.module, sym)
      else:
@@ -2720,6 +2720,8 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
        putIntoDest(p, d, n, genLiteral(p, sym.ast, sym.typ), OnStatic)
      elif useAliveDataFromDce in p.module.flags:
        genConstHeader(p.module, p.module, p, sym)
+        assert((sym.loc.r != nil) and (sym.loc.t != nil))
+        putLocIntoDest(p, d, sym.loc)
      else:
        genComplexConst(p, sym, d)
    of skEnumField:
@@ -2889,7 +2891,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
    if n[genericParamsPos].kind == nkEmpty:
      var prc = n[namePos].sym
      if useAliveDataFromDce in p.module.flags:
-        if p.module.alive.contains(prc.itemId.item):
+        if p.module.alive.contains(prc.itemId.item) and prc.magic in {mNone, mIsolate}:
          genProc(p.module, prc)
      elif prc.skipGenericOwner.kind == skModule and sfCompileTime notin prc.flags:
        if ({sfExportc, sfCompilerProc} * prc.flags == {sfExportc}) or
--- a/compiler/ic/cbackend.nim
+++ b/compiler/ic/cbackend.nim
@@ -83,6 +83,7 @@ proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool
 proc generateCode*(g: ModuleGraph) =
  ## The single entry point, generate C(++) code for the entire
  ## Nim program aka `ModuleGraph`.
+  initStrTable(g.compilerprocs)
  var alive = computeAliveSyms(g.packed, g.config)

  for i in 0..high(g.packed):
--- a/compiler/ic/dce.nim
+++ b/compiler/ic/dce.nim
@@ -47,14 +47,19 @@ proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: i
  ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will
  ## be analysed.
  if not c.alive[module].containsOrIncl(item):
-    let body = g[module].fromDisk.sh.syms[item].ast
+    var body = g[module].fromDisk.sh.syms[item].ast
    if body != emptyNodeId:
      let opt = g[module].fromDisk.sh.syms[item].options
+      if g[module].fromDisk.sh.syms[item].kind in routineKinds:
+        body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos)
      c.stack.add((module, opt, NodePos(body)))

    when false:
-      let name = g[module].fromDisk.sh.strings[g[module].fromDisk.sh.syms[item].name]
-      echo "I was called! ", name, " body exists: ", body != emptyNodeId
+      let nid = g[module].fromDisk.sh.syms[item].name
+      if nid != LitId(0):
+        let name = g[module].fromDisk.sh.strings[nid]
+        if name in ["nimFrame", "callDepthLimitReached"]:
+          echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item

 proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string) =
  let (module, item) = c.compilerProcs[name]
@@ -118,8 +123,8 @@ proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: N
    rangeCheckAnalysis(c, g, tree, n)
  of nkProcDef, nkConverterDef, nkMethodDef, nkLambda, nkDo, nkFuncDef:
    if n.firstSon.kind == nkSym and isNotGeneric(n):
-      if isExportedToC(c, g, n.firstSon.operand):
-        let item = n.operand
+      let item = n.firstSon.operand
+      if isExportedToC(c, g, item):
        # This symbol is alive and everything its body references.
        followLater(c, g, c.thisModule, item)
  else:
@@ -145,6 +150,7 @@ proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms =
      c.thisModule = i
      for p in allNodes(g[i].fromDisk.topLevel):
        aliveCode(c, g, g[i].fromDisk.topLevel, p)
+
  followNow(c, g)
  result = move(c.alive)

--- a/compiler/ic/packed_ast.nim
+++ b/compiler/ic/packed_ast.nim
@@ -464,3 +464,6 @@ iterator allNodes*(tree: PackedTree): NodePos =
    yield NodePos(p)
    let s = span(tree, p)
    inc p, s
+
+proc toPackedItemId*(item: int32): PackedItemId {.inline.} =
+  PackedItemId(module: LitId(0), item: item)
--- a/compiler/ic/replayer.nim
+++ b/compiler/ic/replayer.nim
@@ -128,9 +128,11 @@ proc replayGenericCacheInformation*(g: ModuleGraph; module: int) =
                            PackedItemId(module: LitId(0), item: it))
    methodDef(g, g.idgen, sym)

-  for it in mitems(g.packed[module].fromDisk.compilerProcs):
-    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1]))
-    g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId
+  when false:
+    # not used anymore:
+    for it in mitems(g.packed[module].fromDisk.compilerProcs):
+      let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1]))
+      g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId

  for it in mitems(g.packed[module].fromDisk.converters):
    let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it))
--- a/compiler/ic/to_packed_ast.nim
+++ b/compiler/ic/to_packed_ast.nim
@@ -11,7 +11,7 @@ import std / [hashes, tables, intsets, sha1]
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
  pathutils, condsyms]
-
+#import ".." / [renderer, astalgo]
 from std / os import removeFile, isAbsolute

 type
@@ -57,6 +57,9 @@ type
    symMarker*: IntSet #Table[ItemId, SymId]    # ItemId.item -> SymId
    config*: ConfigRef

+proc isActive*(e: PackedEncoder): bool = e.config != nil
+proc disable*(e: var PackedEncoder) = e.config = nil
+
 template primConfigFields(fn: untyped) {.dirty.} =
  fn backend
  fn selectedGC
@@ -216,7 +219,8 @@ proc addMissing(c: var PackedEncoder; p: PSym) =
  ## consider queuing a symbol for later addition to the packed tree
  if p != nil and p.itemId.module == c.thisModule:
    if p.itemId.item notin c.symMarker:
-      c.pendingSyms.add p
+      if not (sfForward in p.flags and p.kind in routineKinds):
+        c.pendingSyms.add p

 proc addMissing(c: var PackedEncoder; p: PType) =
  ## consider queuing a type for later addition to the packed tree
@@ -332,6 +336,8 @@ proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId
    if s.itemId.item >= m.sh.syms.len:
      setLen m.sh.syms, s.itemId.item+1

+    assert sfForward notin s.flags
+
    var p = PackedSym(kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic,
      position: s.position, offset: s.offset, options: s.options,
      name: s.name.s.toLitId(m))
@@ -418,28 +424,40 @@ proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pa
      toPackedNode(n[i], ir, c, m)
    ir.patch patchPos

-proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
-  var t = newSeq[PackedItemId](i.concreteTypes.len)
-  for j in 0..high(i.concreteTypes):
-    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
-  m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m),
-                                          sym: storeSymLater(i.sym, c, m),
-                                          concreteTypes: t)
-
 proc storeTypeInst*(c: var PackedEncoder; m: var PackedModule; s: PSym; inst: PType) =
  m.typeInstCache.add (storeSymLater(s, c, m), storeTypeLater(inst, c, m))

 proc addPragmaComputation*(c: var PackedEncoder; m: var PackedModule; n: PNode) =
  toPackedNode(n, m.toReplay, c, m)

+proc toPackedProcDef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) =
+  let info = toPackedInfo(n.info, c, m)
+  let patchPos = ir.prepare(n.kind, n.flags,
+                            storeTypeLater(n.typ, c, m), info)
+  for i in 0..<n.len:
+    if i != bodyPos:
+      toPackedNode(n[i], ir, c, m)
+    else:
+      # do not serialize the body of the proc, it's unnecessary since
+      # n[0].sym.ast has the sem'checked variant of it which is what
+      # everybody should use instead.
+      ir.nodes.add PackedNode(kind: nkEmpty, flags: {}, operand: 0,
+                              typeId: nilItemId, info: info)
+  ir.patch patchPos
+
 proc toPackedNodeIgnoreProcDefs(n: PNode, encoder: var PackedEncoder; m: var PackedModule) =
  case n.kind
  of routineDefs:
-    # we serialize n[namePos].sym instead
-    if n[namePos].kind == nkSym:
-      discard storeSym(n[namePos].sym, encoder, m)
-    else:
-      toPackedNode(n, m.topLevel, encoder, m)
+    toPackedProcDef(n, m.topLevel, encoder, m)
+    when false:
+      # we serialize n[namePos].sym instead
+      if n[namePos].kind == nkSym:
+        let s = n[namePos].sym
+        discard storeSym(s, encoder, m)
+        if s.flags * {sfExportc, sfCompilerProc, sfCompileTime} == {sfExportc}:
+          m.exportCProcs.add(s.itemId.item)
+      else:
+        toPackedNode(n, m.topLevel, encoder, m)
  of nkStmtList, nkStmtListExpr:
    for it in n:
      toPackedNodeIgnoreProcDefs(it, encoder, m)
@@ -450,6 +468,24 @@ proc toPackedNodeTopLevel*(n: PNode, encoder: var PackedEncoder; m: var PackedMo
  toPackedNodeIgnoreProcDefs(n, encoder, m)
  flush encoder, m

+proc toPackedGeneratedProcDef*(s: PSym, encoder: var PackedEncoder; m: var PackedModule) =
+  ## Generic procs and generated `=hook`'s need explicit top-level entries so
+  ## that the code generator can work without having to special case these. These
+  ## entries will also be useful for other tools and are the cleanest design
+  ## I can come up with.
+  assert s.kind in routineKinds
+  toPackedProcDef(s.ast, m.topLevel, encoder, m)
+  #flush encoder, m
+
+proc storeInstantiation*(c: var PackedEncoder; m: var PackedModule; s: PSym; i: PInstantiation) =
+  var t = newSeq[PackedItemId](i.concreteTypes.len)
+  for j in 0..high(i.concreteTypes):
+    t[j] = storeTypeLater(i.concreteTypes[j], c, m)
+  m.procInstCache.add PackedInstantiation(key: storeSymLater(s, c, m),
+                                          sym: storeSymLater(i.sym, c, m),
+                                          concreteTypes: t)
+  toPackedGeneratedProcDef(i.sym, c, m)
+
 proc loadError(err: RodFileError; filename: AbsoluteFile) =
  echo "Error: ", $err, " loading file: ", filename.string

@@ -520,6 +556,7 @@ proc storeError(err: RodFileError; filename: AbsoluteFile) =
  removeFile(filename.string)

 proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var PackedModule) =
+  flush encoder, m
  #rememberConfig(encoder, encoder.config)

  var f = rodfiles.create(filename.string)
@@ -572,6 +609,7 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder; m: var Pac
  storeSeqSection enumToStringProcsSection, m.enumToStringProcs

  close(f)
+  encoder.disable()
  if f.err != ok:
    storeError(f.err, filename)

@@ -848,6 +886,8 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
                  info: newLineInfo(fileIdx, 1, 1),
                  position: int(fileIdx))
  m.module.owner = newPackage(conf, cache, fileIdx)
+  if fileIdx == conf.projectMainIdx2:
+    m.module.flags.incl sfMainModule

 proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                       fileIdx: FileIndex; m: var LoadedModule) =
@@ -895,6 +935,7 @@ proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache
      loadError(err, rod)
      g[m].status = outdated
      result = true
+    when false: loadError(err, rod)
  of loading, loaded:
    # For loading: Assume no recompile is required.
    result = false
@@ -1061,6 +1102,13 @@ proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator =
  IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.sh.syms.len,
              typeId: int32 m.fromDisk.sh.types.len)

+proc searchForCompilerproc*(m: LoadedModule; name: string): int32 =
+  # slow, linear search, but the results are cached:
+  for it in items(m.fromDisk.compilerProcs):
+    if m.fromDisk.sh.strings[it[0]] == name:
+      return it[1]
+  return -1
+
 # ------------------------- .rod file viewer ---------------------------------

 proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
@@ -1081,6 +1129,11 @@ proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) =
    for ex in m.reexports:
      echo "  ", m.sh.strings[ex[0]]
    #  reexports*: seq[(LitId, PackedItemId)]
+
+  echo "all symbols"
+  for i in 0..high(m.sh.syms):
+    echo "  ", m.sh.strings[m.sh.syms[i].name], " local ID: ", i
+
  echo "symbols: ", m.sh.syms.len, " types: ", m.sh.types.len,
    " top level nodes: ", m.topLevel.nodes.len, " other nodes: ", m.bodies.nodes.len,
    " strings: ", m.sh.strings.len, " integers: ", m.sh.integers.len,
--- a/compiler/modulegraphs.nim
+++ b/compiler/modulegraphs.nim
@@ -52,6 +52,7 @@ type
  ModuleGraph* = ref object
    ifaces*: seq[Iface]  ## indexed by int32 fileIdx
    packed*: PackedModuleGraph
+    encoders*: seq[PackedEncoder]

    typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId.
    procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId.
@@ -83,7 +84,6 @@ type
    systemModule*: PSym
    sysTypes*: array[TTypeKind, PType]
    compilerprocs*: TStrTable
-    lazyCompilerprocs*: Table[string, FullId]
    exposed*: TStrTable
    packageTypes*: TStrTable
    emptyNode*: PNode
@@ -165,6 +165,13 @@ proc simulateCachedModule*(g: ModuleGraph; moduleSym: PSym; m: PackedModule) =
    echo "simulating ", moduleSym.name.s, " ", moduleSym.position
  simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m)

+proc initEncoder*(g: ModuleGraph; module: PSym) =
+  let id = module.position
+  if id >= g.encoders.len:
+    setLen g.encoders, id+1
+  to_packed_ast.initEncoder(g.encoders[id],
+    g.packed[id].fromDisk, module, g.config, g.startupPackedConfig)
+
 type
  ModuleIter* = object
    fromRod: bool
@@ -260,7 +267,6 @@ proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym =
 proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
  ## we also need to record this to the packed module.
  g.attachedOps[op][t.itemId] = value
-  # XXX Also add to the packed module!

 proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
  ## we also need to record this to the packed module.
@@ -268,7 +274,10 @@ proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttac
  # XXX Also add to the packed module!

 proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) =
-  discard "To implement"
+  if g.config.symbolFiles != disabledSf:
+    assert module < g.encoders.len
+    assert isActive(g.encoders[module])
+    toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk)

 proc getToStringProc*(g: ModuleGraph; t: PType): PSym =
  result = resolveSym(g, g.enumToStringProcs[t.itemId])
@@ -297,12 +306,16 @@ proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) =

 proc loadCompilerProc*(g: ModuleGraph; name: string): PSym =
  if g.config.symbolFiles == disabledSf: return nil
-  let t = g.lazyCompilerprocs.getOrDefault(name)
-  if t.module != 0:
-    assert isCachedModule(g, t.module)
-    result = loadSymFromId(g.config, g.cache, g.packed, t.module, t.packed)
-    if result != nil:
-      strTableAdd(g.compilerprocs, result)
+
+  # slow, linear search, but the results are cached:
+  for module in 0..high(g.packed):
+    #if isCachedModule(g, module):
+    let x = searchForCompilerproc(g.packed[module], name)
+    if x >= 0:
+      result = loadSymFromId(g.config, g.cache, g.packed, module, toPackedItemId(x))
+      if result != nil:
+        strTableAdd(g.compilerprocs, result)
+      return result

 proc `$`*(u: SigHash): string =
  toBase64a(cast[cstring](unsafeAddr u), sizeof(u))
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -147,7 +147,6 @@ type
    selfName*: PIdent
    cache*: IdentCache
    graph*: ModuleGraph
-    encoder*: PackedEncoder
    signatures*: TStrTable
    recursiveDep*: string
    suggestionsMade*: bool
@@ -314,9 +313,10 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
    assert graph.packed[id].status in {undefined, outdated}
    graph.packed[id].status = storing
    graph.packed[id].module = module
-    initEncoder result.encoder, graph.packed[id].fromDisk, module, graph.config, graph.startupPackedConfig
+    initEncoder graph, module

 template packedRepr*(c): untyped = c.graph.packed[c.module.position].fromDisk
+template encoder*(c): untyped = c.graph.encoders[c.module.position]

 proc addIncludeFileDep*(c: PContext; f: FileIndex) =
  if c.config.symbolFiles != disabledSf:
@@ -561,9 +561,10 @@ proc addToGenericCache*(c: PContext; s: PSym; inst: PType) =

 proc saveRodFile*(c: PContext) =
  if c.config.symbolFiles != disabledSf:
-    for (m, n) in PCtx(c.graph.vm).vmstateDiff:
-      if m == c.module:
-        addPragmaComputation(c, n)
+    if c.graph.vm != nil:
+      for (m, n) in PCtx(c.graph.vm).vmstateDiff:
+        if m == c.module:
+          addPragmaComputation(c, n)
    if sfSystemModule in c.module.flags:
      c.graph.systemModuleComplete = true
    c.idgen.sealed = true # no further additions are allowed