IC: next steps (#16550)

* cleanups * ast.nim: cleanups * IC: no more sym.tab field, stored externally in the module graph * nimble compiles again * rodfiles: store bitwidth of integers and the endianness in the cookie because we serialize 'int' directly * rodfiles: added compilerproc and export sections * rodfiles: added all the missing sections * rodfiles: track the missing information * IC: architecture for lazy loading of proc bodies * make tests green again * completed the lazy loading of proc bodies * symbol lookup integration, part 1 * symbol lookup integration, part 2 * symbol lookup integration, part 3 * make tcompilerapi work again * rodfiles: fixed config change handling
2026-01-09 14:32:53 +00:00 · 2021-01-07 20:26:40 +01:00
parent 0da4cb93d1
commit 796498525a
36 changed files with 653 additions and 400 deletions
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -833,18 +833,6 @@ type
      procInstCache*: seq[PInstantiation]
      gcUnsafetyReason*: PSym  # for better error messages wrt gcsafe
      transformedBody*: PNode  # cached body after transf pass
-    of skModule, skPackage:
-      # modules keep track of the generic symbols they use from other modules.
-      # this is because in incremental compilation, when a module is about to
-      # be replaced with a newer version, we must decrement the usage count
-      # of all previously used generics.
-      # For 'import as' we copy the module symbol but shallowCopy the 'tab'
-      # and set the 'usedGenerics' to ... XXX gah! Better set module.name
-      # instead? But this doesn't work either. --> We need an skModuleAlias?
-      # No need, just leave it as skModule but set the owner accordingly and
-      # check for the owner when touching 'usedGenerics'.
-      usedGenerics*: seq[PInstantiation]
-      tab*: TStrTable         # interface table for modules
    of skLet, skVar, skField, skForVar:
      guard*: PSym
      bitsize*: int
@@ -1455,8 +1443,6 @@ proc copySym*(s: PSym; id: ItemId): PSym =
  result.typ = s.typ
  result.flags = s.flags
  result.magic = s.magic
-  if s.kind == skModule:
-    copyStrTable(result.tab, s.tab)
  result.options = s.options
  result.position = s.position
  result.loc = s.loc
@@ -1474,13 +1460,10 @@ proc createModuleAlias*(s: PSym, id: ItemId, newIdent: PIdent, info: TLineInfo;
  result.ast = s.ast
  #result.id = s.id # XXX figure out what to do with the ID.
  result.flags = s.flags
-  system.shallowCopy(result.tab, s.tab)
  result.options = s.options
  result.position = s.position
  result.loc = s.loc
  result.annex = s.annex
-  # XXX once usedGenerics is used, ensure module aliases keep working!
-  assert s.usedGenerics.len == 0

 proc initStrTable*(x: var TStrTable) =
  x.counter = 0
@@ -1915,8 +1898,6 @@ template incompleteType*(t: PType): bool =
 template typeCompleted*(s: PSym) =
  incl s.flags, sfNoForward

-template getBody*(s: PSym): PNode = s.ast[bodyPos]
-
 template detailedInfo*(sym: PSym): string =
  sym.name.s

--- a/compiler/canonicalizer_unused.nim
+++ b/compiler/canonicalizer_unused.nim
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -1292,7 +1292,7 @@ proc rawGenNew(p: BProc, a: var TLoc, sizeExpr: Rope; needsInit: bool) =
    genAssignment(p, a, b, {})
  else:
    let ti = genTypeInfoV1(p.module, typ, a.lode.info)
-    if bt.destructor != nil and not isTrivialProc(bt.destructor):
+    if bt.destructor != nil and not isTrivialProc(p.module.g.graph, bt.destructor):
      # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
      # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
      # convention at least:
@@ -2204,7 +2204,7 @@ proc genDestroy(p: BProc; n: PNode) =
    else: discard "nothing to do"
  else:
    let t = n[1].typ.skipTypes(abstractVar)
-    if t.destructor != nil and t.destructor.ast[bodyPos].len != 0:
+    if t.destructor != nil and getBody(p.module.g.graph, t.destructor).len != 0:
      internalError(p.config, n.info, "destructor turned out to be not trivial")
    discard "ignore calls to the default destructor"

--- a/compiler/ccgtypes.nim
+++ b/compiler/ccgtypes.nim
@@ -36,14 +36,6 @@ proc mangleField(m: BModule; name: PIdent): string =
  if isKeyword(name):
    result.add "_0"

-when false:
-  proc hashOwner(s: PSym): SigHash =
-    var m = s
-    while m.kind != skModule: m = m.owner
-    let p = m.owner
-    assert p.kind == skPackage
-    result = gDebugInfo.register(p.name.s, m.name.s)
-
 proc mangleName(m: BModule; s: PSym): Rope =
  result = s.loc.r
  if result == nil:
@@ -1313,11 +1305,11 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope =
    it = it[0]
  result = makeCString(res)

-proc isTrivialProc(s: PSym): bool {.inline.} = s.ast[bodyPos].len == 0
+proc isTrivialProc(g: ModuleGraph; s: PSym): bool {.inline.} = getBody(g, s).len == 0

 proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
  let theProc = t.attachedOps[op]
-  if theProc != nil and not isTrivialProc(theProc):
+  if theProc != nil and not isTrivialProc(m.g.graph, theProc):
    # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
    # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
    # convention at least:
--- a/compiler/commands.nim
+++ b/compiler/commands.nim
@@ -801,14 +801,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
    expectNoArg(conf, switch, arg, pass, info)
    helpOnError(conf, pass)
  of "symbolfiles": discard "ignore for backwards compat"
-  of "incremental":
-    case arg.normalize
-    of "on": conf.symbolFiles = v2Sf
-    of "off": conf.symbolFiles = disabledSf
-    of "writeonly": conf.symbolFiles = writeOnlySf
-    of "readonly": conf.symbolFiles = readOnlySf
-    of "v2": conf.symbolFiles = v2Sf
-    else: localError(conf, info, "invalid option for --incremental: " & arg)
+  of "incremental", "ic":
+    if pass in {passCmd2, passPP}:
+      case arg.normalize
+      of "on": conf.symbolFiles = v2Sf
+      of "off": conf.symbolFiles = disabledSf
+      of "writeonly": conf.symbolFiles = writeOnlySf
+      of "readonly": conf.symbolFiles = readOnlySf
+      of "v2": conf.symbolFiles = v2Sf
+      else: localError(conf, info, "invalid option for --incremental: " & arg)
  of "skipcfg":
    processOnOffSwitchG(conf, {optSkipSystemConfigFile}, arg, pass, info)
  of "skipprojcfg":
--- a/compiler/evaltempl.nim
+++ b/compiler/evaltempl.nim
@@ -187,7 +187,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
  ctx.instID = instID[]
  ctx.idgen = idgen

-  let body = tmpl.getBody
+  let body = tmpl.ast[bodyPos]
  #echo "instantion of ", renderTree(body, {renderIds})
  if isAtom(body):
    result = newNodeI(nkPar, body.info)
--- a/compiler/ic/packed_ast.nim
+++ b/compiler/ic/packed_ast.nim
@@ -16,28 +16,6 @@ import std / [hashes, tables, strtabs, md5]
 import bitabs
 import ".." / [ast, options]

-const
-  localNamePos* = 0
-  localExportMarkerPos* = 1
-  localPragmaPos* = 2
-  localTypePos* = 3
-  localValuePos* = 4
-
-  typeNamePos* = 0
-  typeExportMarkerPos* = 1
-  typeGenericParamsPos* = 2
-  typePragmaPos* = 3
-  typeBodyPos* = 4
-
-  routineNamePos* = 0
-  routineExportMarkerPos* = 1
-  routinePatternPos* = 2
-  routineGenericParamsPos* = 3
-  routineParamsPos* = 4
-  routineResultPos* = 5
-  routinePragmasPos* = 6
-  routineBodyPos* = 7
-
 const
  nkModuleRef* = nkNone # pair of (ModuleId, SymId)

@@ -55,7 +33,6 @@ type
  TypeId* = PackedItemId

 const
-  nilTypeId* = PackedItemId(module: LitId(0), item: -1.int32)
  nilItemId* = PackedItemId(module: LitId(0), item: -1.int32)

 const
--- a/compiler/ic/rodfiles.nim
+++ b/compiler/ic/rodfiles.nim
@@ -18,6 +18,14 @@ type
    depsSection
    integersSection
    floatsSection
+    exportsSection
+    reexportsSection
+    compilerProcsSection
+    trmacrosSection
+    convertersSection
+    methodsSection
+    pureEnumsSection
+    macroUsagesSection
    topLevelSection
    bodiesSection
    symsSection
@@ -36,26 +44,30 @@ type
 const
  RodVersion = 1
  cookie = [byte(0), byte('R'), byte('O'), byte('D'),
-            byte(0), byte(0), byte(0), byte(RodVersion)]
+            byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)]
+
+proc setError(f: var RodFile; err: RodFileError) {.inline.} =
+  f.err = err
+  #raise newException(IOError, "IO error")

 proc storePrim*(f: var RodFile; s: string) =
  if f.err != ok: return
  if s.len >= high(int32):
-    f.err = tooBig
+    setError f, tooBig
    return
  var lenPrefix = int32(s.len)
  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
  else:
    if s.len != 0:
      if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        f.err = ioFailure
+        setError f, ioFailure

 proc storePrim*[T](f: var RodFile; x: T) =
  if f.err != ok: return
  when supportsCopyMem(T):
    if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      f.err = ioFailure
+      setError f, ioFailure
  elif T is tuple:
    for y in fields(x):
      storePrim(f, y)
@@ -65,11 +77,11 @@ proc storePrim*[T](f: var RodFile; x: T) =
 proc storeSeq*[T](f: var RodFile; s: seq[T]) =
  if f.err != ok: return
  if s.len >= high(int32):
-    f.err = tooBig
+    setError f, tooBig
    return
  var lenPrefix = int32(s.len)
  if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
  else:
    for i in 0..<s.len:
      storePrim(f, s[i])
@@ -78,18 +90,18 @@ proc loadPrim*(f: var RodFile; s: var string) =
  if f.err != ok: return
  var lenPrefix = int32(0)
  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
  else:
    s = newString(lenPrefix)
    if lenPrefix > 0:
      if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len:
-        f.err = ioFailure
+        setError f, ioFailure

 proc loadPrim*[T](f: var RodFile; x: var T) =
  if f.err != ok: return
  when supportsCopyMem(T):
    if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x):
-      f.err = ioFailure
+      setError f, ioFailure
  elif T is tuple:
    for y in fields(x):
      loadPrim(f, y)
@@ -100,7 +112,7 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
  if f.err != ok: return
  var lenPrefix = int32(0)
  if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix):
-    f.err = ioFailure
+    setError f, ioFailure
  else:
    s = newSeq[T](lenPrefix)
    for i in 0..<lenPrefix:
@@ -109,15 +121,15 @@ proc loadSeq*[T](f: var RodFile; s: var seq[T]) =
 proc storeHeader*(f: var RodFile) =
  if f.err != ok: return
  if f.f.writeBytes(cookie, 0, cookie.len) != cookie.len:
-    f.err = ioFailure
+    setError f, ioFailure

 proc loadHeader*(f: var RodFile) =
  if f.err != ok: return
  var thisCookie: array[cookie.len, byte]
  if f.f.readBytes(thisCookie, 0, thisCookie.len) != thisCookie.len:
-    f.err = ioFailure
+    setError f, ioFailure
  elif thisCookie != cookie:
-    f.err = wrongHeader
+    setError f, wrongHeader

 proc storeSection*(f: var RodFile; s: RodSection) =
  if f.err != ok: return
@@ -129,15 +141,15 @@ proc loadSection*(f: var RodFile; expected: RodSection) =
  if f.err != ok: return
  var s: RodSection
  loadPrim(f, s)
-  if expected != s:
-    f.err = wrongSection
+  if expected != s and f.err == ok:
+    setError f, wrongSection

 proc create*(filename: string): RodFile =
  if not open(result.f, filename, fmWrite):
-    result.err = ioFailure
+    setError result, ioFailure

 proc close*(f: var RodFile) = close(f.f)

 proc open*(filename: string): RodFile =
  if not open(result.f, filename, fmRead):
-    result.err = ioFailure
+    setError result, ioFailure
--- a/compiler/ic/to_packed_ast.nim
+++ b/compiler/ic/to_packed_ast.nim
@@ -14,8 +14,6 @@ import ".." / [ast, idents, lineinfos, msgs, ropes, options,

 from std / os import removeFile, isAbsolute

-when not defined(release): import ".." / astalgo # debug()
-
 type
  PackedConfig* = object
    backend: TBackend
@@ -32,6 +30,11 @@ type
    bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position.
    hidden*: PackedTree # instantiated generics and other trees not directly in the source code.
    #producedGenerics*: Table[GenericKey, SymId]
+    exports*: seq[(LitId, int32)]
+    reexports*: seq[(LitId, PackedItemId)]
+    compilerProcs*, trmacros*, converters*, pureEnums*: seq[(LitId, int32)]
+    methods*: seq[(LitId, PackedItemId, int32)]
+    macroUsages*: seq[(PackedItemId, PackedLineInfo)]
    sh*: Shared
    cfg: PackedConfig

@@ -61,19 +64,28 @@ proc definedSymbolsAsString(config: ConfigRef): string =
    result.add ' '
    result.add d

-proc rememberConfig(c: var PackedEncoder; config: ConfigRef) =
+proc rememberConfig(c: var PackedEncoder; config: ConfigRef; pc: PackedConfig) =
  c.m.definedSymbols = definedSymbolsAsString(config)
-
-  template rem(x) =
-    c.m.cfg.x = config.x
-  primConfigFields rem
+  #template rem(x) =
+  #  c.m.cfg.x = config.x
+  #primConfigFields rem
+  c.m.cfg = pc

 proc configIdentical(m: PackedModule; config: ConfigRef): bool =
  result = m.definedSymbols == definedSymbolsAsString(config)
+  #if not result:
+  #  echo "A ", m.definedSymbols, " ", definedSymbolsAsString(config)
  template eq(x) =
    result = result and m.cfg.x == config.x
+    #if not result:
+    #  echo "B ", m.cfg.x, " ", config.x
  primConfigFields eq

+proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) =
+  template rem(x) =
+    dest.x = config.x
+  primConfigFields rem
+
 proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string =
  result = msgs.getHash(conf, fileIdx)
  if result.len == 0:
@@ -104,7 +116,7 @@ proc includesIdentical(m: var PackedModule; config: ConfigRef): bool =
      return false
  result = true

-proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef) =
+proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef; pc: PackedConfig) =
  ## setup a context for serializing to packed ast
  c.m.sh = Shared()
  c.thisModule = m.itemId.module
@@ -122,12 +134,45 @@ proc initEncoder*(c: var PackedEncoder; m: PSym; config: ConfigRef) =
      msgs.setHash(config, thisNimFile, h)
  c.m.includes.add((toLitId(thisNimFile, c), h)) # the module itself

+  rememberConfig(c, config, pc)
+
 proc addIncludeFileDep*(c: var PackedEncoder; f: FileIndex) =
  c.m.includes.add((toLitId(f, c), hashFileCached(c.config, f)))

 proc addImportFileDep*(c: var PackedEncoder; f: FileIndex) =
  c.m.imports.add toLitId(f, c)

+proc addExported*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.exports.add((nameId, s.itemId.item))
+
+proc addConverter*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.converters.add((nameId, s.itemId.item))
+
+proc addTrmacro*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.trmacros.add((nameId, s.itemId.item))
+
+proc addPureEnum*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  assert s.kind == skType
+  c.m.pureEnums.add((nameId, s.itemId.item))
+
+proc addMethod*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  discard "to do"
+  # c.m.methods.add((nameId, s.itemId.item))
+
+proc addReexport*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c),
+                                          item: s.itemId.item)))
+
+proc addCompilerProc*(c: var PackedEncoder; s: PSym) =
+  let nameId = getOrIncl(c.m.sh.strings, s.name.s)
+  c.m.compilerProcs.add((nameId, s.itemId.item))
+
 proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder)
 proc toPackedSym*(s: PSym; c: var PackedEncoder): PackedItemId
 proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId
@@ -197,7 +242,7 @@ template storeNode(dest, src, field) =

 proc toPackedType(t: PType; c: var PackedEncoder): PackedItemId =
  ## serialize a ptype
-  if t.isNil: return nilTypeId
+  if t.isNil: return nilItemId

  if t.uniqueId.module != c.thisModule:
    # XXX Assert here that it already was serialized in the foreign module!
@@ -376,88 +421,49 @@ proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef
  f.loadPrim m.definedSymbols
  f.loadPrim m.cfg

-  if not configIdentical(m, config):
+  if f.err == ok and not configIdentical(m, config):
    f.err = configMismatch

-  f.loadSection stringsSection
-  f.load m.sh.strings
+  template loadSeqSection(section, data) {.dirty.} =
+    f.loadSection section
+    f.loadSeq data

-  f.loadSection checkSumsSection
-  f.loadSeq m.includes
+  template loadTabSection(section, data) {.dirty.} =
+    f.loadSection section
+    f.load data
+
+  loadTabSection stringsSection, m.sh.strings
+
+  loadSeqSection checkSumsSection, m.includes
  if not includesIdentical(m, config):
    f.err = includeFileChanged

-  f.loadSection depsSection
-  f.loadSeq m.imports
+  loadSeqSection depsSection, m.imports

-  f.loadSection integersSection
-  f.load m.sh.integers
-  f.loadSection floatsSection
-  f.load m.sh.floats
+  loadTabSection integersSection, m.sh.integers
+  loadTabSection floatsSection, m.sh.floats

-  f.loadSection topLevelSection
-  f.loadSeq m.topLevel.nodes
+  loadSeqSection exportsSection, m.exports

-  f.loadSection bodiesSection
-  f.loadSeq m.bodies.nodes
+  loadSeqSection reexportsSection, m.reexports

-  f.loadSection symsSection
-  f.loadSeq m.sh.syms
+  loadSeqSection compilerProcsSection, m.compilerProcs

-  f.loadSection typesSection
-  f.loadSeq m.sh.types
+  loadSeqSection trmacrosSection, m.trmacros
+
+  loadSeqSection convertersSection, m.converters
+  loadSeqSection methodsSection, m.methods
+  loadSeqSection pureEnumsSection, m.pureEnums
+  loadSeqSection macroUsagesSection, m.macroUsages
+
+  loadSeqSection topLevelSection, m.topLevel.nodes
+  loadSeqSection bodiesSection, m.bodies.nodes
+  loadSeqSection symsSection, m.sh.syms
+  loadSeqSection typesSection, m.sh.types

  close(f)
  result = f.err

-type
-  ModuleStatus* = enum
-    undefined,
-    loading,
-    loaded,
-    outdated
-
-  LoadedModule* = object
-    status: ModuleStatus
-    symsInit, typesInit: bool
-    fromDisk: PackedModule
-    syms: seq[PSym] # indexed by itemId
-    types: seq[PType]
-
-  PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex
-
-proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef;
-                    fileIdx: FileIndex): bool =
-  let m = int(fileIdx)
-  if m >= g.len:
-    g.setLen(m+1)
-
-  case g[m].status
-  of undefined:
-    g[m].status = loading
-    let fullpath = msgs.toFullPath(conf, fileIdx)
-    let rod = toRodFile(conf, AbsoluteFile fullpath)
-    let err = loadRodFile(rod, g[m].fromDisk, conf)
-    if err == ok:
-      result = false
-      # check its dependencies:
-      for dep in g[m].fromDisk.imports:
-        let fid = toFileIndex(dep, g[m].fromDisk, conf)
-        # Warning: we need to traverse the full graph, so
-        # do **not use break here**!
-        if needsRecompile(g, conf, fid):
-          result = true
-
-      g[m].status = if result: outdated else: loaded
-    else:
-      loadError(err, rod)
-      g[m].status = outdated
-      result = true
-  of loading, loaded:
-    result = false
-  of outdated:
-    result = true
-
 # -------------------------------------------------------------------------

 proc storeError(err: RodFileError; filename: AbsoluteFile) =
@@ -465,7 +471,7 @@ proc storeError(err: RodFileError; filename: AbsoluteFile) =
  removeFile(filename.string)

 proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) =
-  rememberConfig(encoder, encoder.config)
+  #rememberConfig(encoder, encoder.config)

  var f = rodfiles.create(filename.string)
  f.storeHeader()
@@ -473,40 +479,50 @@ proc saveRodFile*(filename: AbsoluteFile; encoder: var PackedEncoder) =
  f.storePrim encoder.m.definedSymbols
  f.storePrim encoder.m.cfg

-  f.storeSection stringsSection
-  f.store encoder.m.sh.strings
+  template storeSeqSection(section, data) {.dirty.} =
+    f.storeSection section
+    f.storeSeq data

-  f.storeSection checkSumsSection
-  f.storeSeq encoder.m.includes
+  template storeTabSection(section, data) {.dirty.} =
+    f.storeSection section
+    f.store data

-  f.storeSection depsSection
-  f.storeSeq encoder.m.imports
+  storeTabSection stringsSection, encoder.m.sh.strings

-  f.storeSection integersSection
-  f.store encoder.m.sh.integers
+  storeSeqSection checkSumsSection, encoder.m.includes

-  f.storeSection floatsSection
-  f.store encoder.m.sh.floats
+  storeSeqSection depsSection, encoder.m.imports

-  f.storeSection topLevelSection
-  f.storeSeq encoder.m.topLevel.nodes
+  storeTabSection integersSection, encoder.m.sh.integers
+  storeTabSection floatsSection, encoder.m.sh.floats

-  f.storeSection bodiesSection
-  f.storeSeq encoder.m.bodies.nodes
+  storeSeqSection exportsSection, encoder.m.exports

-  f.storeSection symsSection
-  f.storeSeq encoder.m.sh.syms
+  storeSeqSection reexportsSection, encoder.m.reexports

-  f.storeSection typesSection
-  f.storeSeq encoder.m.sh.types
+  storeSeqSection compilerProcsSection, encoder.m.compilerProcs
+
+  storeSeqSection trmacrosSection, encoder.m.trmacros
+  storeSeqSection convertersSection, encoder.m.converters
+  storeSeqSection methodsSection, encoder.m.methods
+  storeSeqSection pureEnumsSection, encoder.m.pureEnums
+  storeSeqSection macroUsagesSection, encoder.m.macroUsages
+
+  storeSeqSection topLevelSection, encoder.m.topLevel.nodes
+
+  storeSeqSection bodiesSection, encoder.m.bodies.nodes
+  storeSeqSection symsSection, encoder.m.sh.syms
+
+  storeSeqSection typesSection, encoder.m.sh.types
  close(f)
  if f.err != ok:
-    loadError(f.err, filename)
+    storeError(f.err, filename)

-  when true:
+  when false:
    # basic loader testing:
    var m2: PackedModule
    discard loadRodFile(filename, m2, encoder.config)
+    echo "loaded ", filename.string

 # ----------------------------------------------------------------------------

@@ -516,7 +532,25 @@ type
    lastLit*: LitId
    lastFile*: FileIndex # remember the last lookup entry.
    config*: ConfigRef
-    ident: IdentCache
+    cache: IdentCache
+
+type
+  ModuleStatus* = enum
+    undefined,
+    loading,
+    loaded,
+    outdated
+
+  LoadedModule* = object
+    status*: ModuleStatus
+    symsInit, typesInit: bool
+    fromDisk: PackedModule
+    syms: seq[PSym] # indexed by itemId
+    types: seq[PType]
+    module*: PSym # the one true module symbol.
+    iface: Table[PIdent, seq[PackedItemId]] # PackedItemId so that it works with reexported symbols too
+
+  PackedModuleGraph* = seq[LoadedModule] # indexed by FileIndex

 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType
 proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym
@@ -546,7 +580,7 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph;
  of nkEmpty, nkNilLit, nkType:
    discard
  of nkIdent:
-    result.ident = getIdent(c.ident, g[c.thisModule].fromDisk.sh.strings[n.litId])
+    result.ident = getIdent(c.cache, g[c.thisModule].fromDisk.sh.strings[n.litId])
  of nkSym:
    result.sym = loadSym(c, g, PackedItemId(module: LitId(0), item: tree.nodes[n.int].operand))
  of directIntLit:
@@ -567,6 +601,31 @@ proc loadNodes(c: var PackedDecoder; g: var PackedModuleGraph;
    for n0 in sonsReadonly(tree, n):
      result.add loadNodes(c, g, tree, n0)

+proc loadProcHeader(c: var PackedDecoder; g: var PackedModuleGraph;
+                    tree: PackedTree; n: NodePos): PNode =
+  # do not load the body of the proc. This will be done later in
+  # getProcBody, if required.
+  let k = n.kind
+  result = newNodeIT(k, translateLineInfo(c, g, n.info),
+    loadType(c, g, n.typ))
+  result.flags = n.flags
+  assert k in {nkProcDef, nkMethodDef, nkIteratorDef, nkFuncDef, nkConverterDef}
+  var i = 0
+  for n0 in sonsReadonly(tree, n):
+    if i != bodyPos:
+      result.add loadNodes(c, g, tree, n0)
+    else:
+      result.add nil
+    inc i
+
+proc loadProcBody(c: var PackedDecoder; g: var PackedModuleGraph;
+                  tree: PackedTree; n: NodePos): PNode =
+  var i = 0
+  for n0 in sonsReadonly(tree, n):
+    if i == bodyPos:
+      result = loadNodes(c, g, tree, n0)
+    inc i
+
 proc moduleIndex*(c: var PackedDecoder; g: var PackedModuleGraph;
                  s: PackedItemId): int32 {.inline.} =
  result = if s.module == LitId(0): c.thisModule
@@ -579,13 +638,17 @@ proc symHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
    info: translateLineInfo(c, g, s.info),
    options: s.options,
    position: s.position,
-    name: getIdent(c.ident, g[si].fromDisk.sh.strings[s.name])
+    name: getIdent(c.cache, g[si].fromDisk.sh.strings[s.name])
  )

 template loadAstBody(p, field) =
  if p.field != emptyNodeId:
    result.field = loadNodes(c, g, g[si].fromDisk.bodies, NodePos p.field)

+template loadAstBodyLazy(p, field) =
+  if p.field != emptyNodeId:
+    result.field = loadProcHeader(c, g, g[si].fromDisk.bodies, NodePos p.field)
+
 proc loadLib(c: var PackedDecoder; g: var PackedModuleGraph;
             si, item: int32; l: PackedLib): PLib =
  # XXX: hack; assume a zero LitId means the PackedLib is all zero (empty)
@@ -600,7 +663,10 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
                       s: PackedSym; si, item: int32; result: PSym) =
  result.typ = loadType(c, g, s.typ)
  loadAstBody(s, constraint)
-  loadAstBody(s, ast)
+  if result.kind in {skProc, skFunc, skIterator, skConverter, skMethod}:
+    loadAstBodyLazy(s, ast)
+  else:
+    loadAstBody(s, ast)
  result.annex = loadLib(c, g, si, item, s.annex)
  when hasFFI:
    result.cname = g[si].fromDisk.sh.strings[s.cname]
@@ -615,7 +681,7 @@ proc symBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
    result.loc.r = rope externalName

 proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): PSym =
-  if s == nilTypeId:
+  if s == nilItemId:
    result = nil
  else:
    let si = moduleIndex(c, g, s)
@@ -626,10 +692,16 @@ proc loadSym(c: var PackedDecoder; g: var PackedModuleGraph; s: PackedItemId): P

    if g[si].syms[s.item] == nil:
      let packed = addr(g[si].fromDisk.sh.syms[s.item])
-      result = symHeaderFromPacked(c, g, packed[], si, s.item)
-      # store it here early on, so that recursions work properly:
-      g[si].syms[s.item] = result
-      symBodyFromPacked(c, g, packed[], si, s.item, result)
+
+      if packed.kind != skModule:
+        result = symHeaderFromPacked(c, g, packed[], si, s.item)
+        # store it here early on, so that recursions work properly:
+        g[si].syms[s.item] = result
+        symBodyFromPacked(c, g, packed[], si, s.item, result)
+      else:
+        result = g[si].module
+        assert result != nil
+
    else:
      result = g[si].syms[s.item]

@@ -654,7 +726,7 @@ proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph;
    result.methods.add((gen, loadSym(c, g, id)))

 proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId): PType =
-  if t == nilTypeId:
+  if t == nilItemId:
    result = nil
  else:
    let si = moduleIndex(c, g, t)
@@ -672,15 +744,142 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; t: PackedItemId):
    else:
      result = g[si].types[t.item]

+proc setupLookupTables(m: var LoadedModule; conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex) =
+  m.iface = initTable[PIdent, seq[PackedItemId]]()
+  for e in m.fromDisk.exports:
+    let nameLit = e[0]
+    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(PackedItemId(module: LitId(0), item: e[1]))
+  for re in m.fromDisk.reexports:
+    let nameLit = re[0]
+    m.iface.mgetOrPut(cache.getIdent(m.fromDisk.sh.strings[nameLit]), @[]).add(re[1])

-when false:
-  proc initGenericKey*(s: PSym; types: seq[PType]): GenericKey =
-    result.module = s.owner.itemId.module
-    result.name = s.name.s
-    result.types = mapIt types: hashType(it, {CoType, CoDistinct}).MD5Digest
+  let filename = AbsoluteFile toFullPath(conf, fileIdx)
+  # We cannot call ``newSym`` here, because we have to circumvent the ID
+  # mechanism, which we do in order to assign each module a persistent ID.
+  m.module = PSym(kind: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32),
+                  name: getIdent(cache, splitFile(filename).name),
+                  info: newLineInfo(fileIdx, 1, 1))
+
+proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
+                    fileIdx: FileIndex): bool =
+  let m = int(fileIdx)
+  if m >= g.len:
+    g.setLen(m+1)
+
+  case g[m].status
+  of undefined:
+    g[m].status = loading
+    let fullpath = msgs.toFullPath(conf, fileIdx)
+    let rod = toRodFile(conf, AbsoluteFile fullpath)
+    let err = loadRodFile(rod, g[m].fromDisk, conf)
+    if err == ok:
+      result = false
+      # check its dependencies:
+      for dep in g[m].fromDisk.imports:
+        let fid = toFileIndex(dep, g[m].fromDisk, conf)
+        # Warning: we need to traverse the full graph, so
+        # do **not use break here**!
+        if needsRecompile(g, conf, cache, fid):
+          result = true
+
+      if not result:
+        setupLookupTables(g[m], conf, cache, fileIdx)
+      g[m].status = if result: outdated else: loaded
+    else:
+      loadError(err, rod)
+      g[m].status = outdated
+      result = true
+  of loading, loaded:
+    result = false
+  of outdated:
+    result = true
+
+proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
+                        fileIdx: FileIndex): PSym =
+  ## Returns 'nil' if the module needs to be recompiled.
+  if needsRecompile(g, conf, cache, fileIdx):
+    result = nil
+  else:
+    result = g[int fileIdx].module
+    assert result != nil
+
+template setupDecoder() {.dirty.} =
+  var decoder = PackedDecoder(
+    thisModule: int32(module),
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+
+proc loadProcBody*(config: ConfigRef, cache: IdentCache;
+                   g: var PackedModuleGraph; s: PSym): PNode =
+  let mId = s.itemId.module
+  var decoder = PackedDecoder(
+    thisModule: mId,
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+  let pos = g[mId].fromDisk.sh.syms[s.itemId.item].ast
+  assert pos != emptyNodeId
+  result = loadProcBody(decoder, g, g[mId].fromDisk.bodies, NodePos pos)
+
+type
+  RodIter* = object
+    decoder: PackedDecoder
+    values: seq[PackedItemId]
+    i: int
+
+proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache;
+                  g: var PackedModuleGraph; module: FileIndex;
+                  name: PIdent): PSym =
+  it.decoder = PackedDecoder(
+    thisModule: int32(module),
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+  it.values = g[int module].iface.getOrDefault(name)
+  it.i = 0
+  if it.i < it.values.len:
+    result = loadSym(it.decoder, g, it.values[it.i])
+    inc it.i
+
+proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache;
+                         g: var PackedModuleGraph; module: FileIndex): PSym =
+  it.decoder = PackedDecoder(
+    thisModule: int32(module),
+    lastLit: LitId(0),
+    lastFile: FileIndex(-1),
+    config: config,
+    cache: cache)
+  it.values = @[]
+  for v in g[int module].iface.values:
+    it.values.add v
+  it.i = 0
+  if it.i < it.values.len:
+    result = loadSym(it.decoder, g, it.values[it.i])
+    inc it.i
+
+proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym =
+  if it.i < it.values.len:
+    result = loadSym(it.decoder, g, it.values[it.i])
+    inc it.i
+
+iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache;
+                           g: var PackedModuleGraph; module: FileIndex;
+                           name: PIdent): PSym =
+  setupDecoder()
+  let values = g[int module].iface.getOrDefault(name)
+  for pid in values:
+    let s = loadSym(decoder, g, pid)
+    assert s != nil
+    yield s
+
+proc interfaceSymbol*(config: ConfigRef, cache: IdentCache;
+                      g: var PackedModuleGraph; module: FileIndex;
+                      name: PIdent): PSym =
+  setupDecoder()
+  let values = g[int module].iface.getOrDefault(name)
+  result = loadSym(decoder, g, values[0])

-  proc addGeneric*(m: var Module; c: var PackedEncoder; key: GenericKey; s: PSym) =
-    ## add a generic to the module
-    if key notin m.generics:
-      m.generics[key] = toPackedSym(s, m.ast, c)
-      toPackedNode(s.ast, m.ast, c)
--- a/compiler/idents.nim
+++ b/compiler/idents.nim
@@ -113,3 +113,8 @@ proc newIdentCache*(): IdentCache =
 proc whichKeyword*(id: PIdent): TSpecialWord =
  if id.id < 0: result = wInvalid
  else: result = TSpecialWord(id.id)
+
+proc hash*(x: PIdent): Hash {.inline.} = x.h
+proc `==`*(a, b: PIdent): bool {.inline.} =
+  if a.isNil or b.isNil: result = system.`==`(a, b)
+  else: result = a.id == b.id
--- a/compiler/importer.nim
+++ b/compiler/importer.nim
@@ -11,7 +11,8 @@

 import
  intsets, ast, astalgo, msgs, options, idents, lookups,
-  semdata, modulepaths, sigmatch, lineinfos, sets
+  semdata, modulepaths, sigmatch, lineinfos, sets,
+  modulegraphs

 proc readExceptSet*(c: PContext, n: PNode): IntSet =
  assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
@@ -108,7 +109,7 @@ proc rawImportSymbol(c: PContext, s, origin: PSym; importSet: var IntSet) =

 proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
  let ident = lookups.considerQuotedIdent(c, n)
-  let s = strTableGet(fromMod.tab, ident)
+  let s = someSym(c.graph, fromMod, ident)
  if s == nil:
    errorUndeclaredIdentifier(c, n.info, ident.s)
  else:
@@ -118,16 +119,16 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym; importSet: var IntSet) =
    # for an enumeration we have to add all identifiers
    if multiImport:
      # for a overloadable syms add all overloaded routines
-      var it: TIdentIter
-      var e = initIdentIter(it, fromMod.tab, s.name)
+      var it: ModuleIter
+      var e = initModuleIter(it, c.graph, fromMod, s.name)
      while e != nil:
        if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
        if s.kind in ExportableSymKinds:
          rawImportSymbol(c, e, fromMod, importSet)
-        e = nextIdentIter(it, fromMod.tab)
+        e = nextModuleIter(it, c.graph)
    else:
      rawImportSymbol(c, s, fromMod, importSet)
-    suggestSym(c.config, n.info, s, c.graph.usageSym, false)
+    suggestSym(c.graph, n.info, s, c.graph.usageSym, false)

 proc addImport(c: PContext; im: sink ImportedModule) =
  for i in 0..high(c.imports):
@@ -176,18 +177,6 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
  c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
  addUnnamedIt(c, fromMod, it.id notin exceptSet)

-  when false:
-    var i: TTabIter
-    var s = initTabIter(i, fromMod.tab)
-    while s != nil:
-      if s.kind != skModule:
-        if s.kind != skEnumField:
-          if s.kind notin ExportableSymKinds:
-            internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
-          if exceptSet.isNil or s.name.id notin exceptSet:
-            rawImportSymbol(c, s, fromMod)
-      s = nextIter(i, fromMod.tab)
-
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
  c.addImport ImportedModule(m: fromMod, mode: importAll)
  addUnnamedIt(c, fromMod, true)
@@ -256,7 +245,7 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
        message(c.config, n.info, warnDeprecated, result.constraint.strVal & "; " & result.name.s & " is deprecated")
      else:
        message(c.config, n.info, warnDeprecated, result.name.s & " is deprecated")
-    suggestSym(c.config, n.info, result, c.graph.usageSym, false)
+    suggestSym(c.graph, n.info, result, c.graph.usageSym, false)
    importStmtResult.add newSymNode(result, n.info)
    #newStrNode(toFullPath(c.config, f), n.info)

--- a/compiler/jsgen.nim
+++ b/compiler/jsgen.nim
@@ -887,7 +887,7 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
          var v = copyNode(e[0])
          inc(totalRange, int(e[1].intVal - v.intVal))
          if totalRange > 65535:
-            localError(p.config, n.info, 
+            localError(p.config, n.info,
                       "Your case statement contains too many branches, consider using if/else instead!")
          while v.intVal <= e[1].intVal:
            gen(p, v, cond)
@@ -1076,7 +1076,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
      useMagic(p, "nimCopy")
      # supports proc getF(): var T
      if x.kind in {nkHiddenDeref, nkDerefExpr} and x[0].kind in nkCallKinds:
-          lineF(p, "nimCopy($1, $2, $3);$n", 
+          lineF(p, "nimCopy($1, $2, $3);$n",
                [a.res, b.res, genTypeInfo(p, y.typ)])
      else:
        lineF(p, "$1 = nimCopy($1, $2, $3);$n",
@@ -1426,7 +1426,7 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
    if lfNoDecl in s.loc.flags or s.magic != mNone or
       {sfImportc, sfInfixCall} * s.flags != {}:
      discard
-    elif s.kind == skMethod and s.getBody.kind == nkEmpty:
+    elif s.kind == skMethod and getBody(p.module.graph, s).kind == nkEmpty:
      # we cannot produce code for the dispatcher yet:
      discard
    elif sfForward in s.flags:
--- a/compiler/lookups.nim
+++ b/compiler/lookups.nim
@@ -11,7 +11,8 @@

 import
  intsets, ast, astalgo, idents, semdata, types, msgs, options,
-  renderer, nimfix/prettybase, lineinfos, strutils
+  renderer, nimfix/prettybase, lineinfos, strutils,
+  modulegraphs

 proc ensureNoMissingOrUnusedSymbols(c: PContext; scope: PScope)

@@ -107,8 +108,9 @@ proc localSearchInScope*(c: PContext, s: PIdent): PSym =
    scope = scope.parent
    result = strTableGet(scope.symbols, s)

-proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; name: PIdent): PSym =
-  result = initIdentIter(ti, im.m.tab, name)
+proc initIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule; name: PIdent;
+                   g: ModuleGraph): PSym =
+  result = initModuleIter(ti, g, im.m, name)
  while result != nil:
    let b =
      case im.mode
@@ -117,11 +119,12 @@ proc initIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule; n
      of importExcept: name.id notin im.exceptSet
    if b and not containsOrIncl(marked, result.id):
      return result
-    result = nextIdentIter(ti, im.m.tab)
+    result = nextModuleIter(ti, g)

-proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule): PSym =
+proc nextIdentIter(ti: var ModuleIter; marked: var IntSet; im: ImportedModule;
+                   g: ModuleGraph): PSym =
  while true:
-    result = nextIdentIter(ti, im.m.tab)
+    result = nextModuleIter(ti, g)
    if result == nil: return nil
    case im.mode
    of importAll:
@@ -134,17 +137,17 @@ proc nextIdentIter(ti: var TIdentIter; marked: var IntSet; im: ImportedModule):
      if result.name.id notin im.exceptSet and not containsOrIncl(marked, result.id):
        return result

-iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent): PSym =
-  var ti: TIdentIter
-  var candidate = initIdentIter(ti, marked, im, name)
+iterator symbols(im: ImportedModule; marked: var IntSet; name: PIdent; g: ModuleGraph): PSym =
+  var ti: ModuleIter
+  var candidate = initIdentIter(ti, marked, im, name, g)
  while candidate != nil:
    yield candidate
-    candidate = nextIdentIter(ti, marked, im)
+    candidate = nextIdentIter(ti, marked, im, g)

 iterator importedItems*(c: PContext; name: PIdent): PSym =
  var marked = initIntSet()
  for im in c.imports.mitems:
-    for s in symbols(im, marked, name):
+    for s in symbols(im, marked, name, c.graph):
      yield s

 proc allPureEnumFields(c: PContext; name: PIdent): seq[PSym] =
@@ -169,15 +172,15 @@ iterator allSyms*(c: PContext): (PSym, int, bool) =
  dec scopeN
  isLocal = false
  for im in c.imports.mitems:
-    for s in im.m.tab.data:
-      if s != nil:
-        yield (s, scopeN, isLocal)
+    for s in modulegraphs.allSyms(c.graph, im.m):
+      assert s != nil
+      yield (s, scopeN, isLocal)

 proc someSymFromImportTable*(c: PContext; name: PIdent; ambiguous: var bool): PSym =
  var marked = initIntSet()
  result = nil
  for im in c.imports.mitems:
-    for s in symbols(im, marked, name):
+    for s in symbols(im, marked, name, c.graph):
      if result == nil:
        result = s
      else:
@@ -214,7 +217,7 @@ proc searchInScopesFilterBy*(c: PContext, s: PIdent, filter: TSymKinds): seq[PSy
  if result.len == 0:
    var marked = initIntSet()
    for im in c.imports.mitems:
-      for s in symbols(im, marked, s):
+      for s in symbols(im, marked, s, c.graph):
        if s.kind in filter:
          result.add s

@@ -240,6 +243,7 @@ type
    oimSymChoiceLocalLookup
  TOverloadIter* = object
    it*: TIdentIter
+    mit*: ModuleIter
    m*: PSym
    mode*: TOverloadIterMode
    symChoiceIndex*: int
@@ -306,7 +310,7 @@ proc addDeclAt*(c: PContext; scope: PScope, sym: PSym) =
 proc addInterfaceDeclAux(c: PContext, sym: PSym) =
  if sfExported in sym.flags:
    # add to interface:
-    if c.module != nil: strTableAdd(c.module.tab, sym)
+    if c.module != nil: exportSym(c, sym)
    else: internalError(c.config, sym.info, "addInterfaceDeclAux")

 proc addInterfaceDeclAt*(c: PContext, scope: PScope, sym: PSym) =
@@ -475,7 +479,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
        if m == c.module:
          result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
        else:
-          result = strTableGet(m.tab, ident).skipAlias(n, c.config)
+          result = someSym(c.graph, m, ident).skipAlias(n, c.config)
        if result == nil and checkUndeclared in flags:
          fixSpelling(n[1], ident, searchInScopes)
          errorUndeclaredIdentifier(c, n[1].info, ident.s)
@@ -509,7 +513,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
        scope = scope.parent
        if scope == nil:
          for i in 0..c.imports.high:
-            result = initIdentIter(o.it, o.marked, c.imports[i], ident).skipAlias(n, c.config)
+            result = initIdentIter(o.mit, o.marked, c.imports[i], ident, c.graph).skipAlias(n, c.config)
            if result != nil:
              o.currentScope = nil
              o.importIdx = i
@@ -535,7 +539,7 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
                                 ident).skipAlias(n, c.config)
          o.mode = oimSelfModule
        else:
-          result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
+          result = initModuleIter(o.mit, c.graph, o.m, ident).skipAlias(n, c.config)
      else:
        noidentError(c.config, n[1], n)
        result = errorSym(c, n[1])
@@ -568,7 +572,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym
  var idx = o.importIdx+1
  o.importIdx = c.imports.len # assume the other imported modules lack this symbol too
  while idx < c.imports.len:
-    result = initIdentIter(o.it, o.marked, c.imports[idx], o.it.name).skipAlias(n, c.config)
+    result = initIdentIter(o.mit, o.marked, c.imports[idx], o.it.name, c.graph).skipAlias(n, c.config)
    if result != nil:
      # oh, we were wrong, some other module had the symbol, so remember that:
      o.importIdx = idx
@@ -578,7 +582,7 @@ proc nextOverloadIterImports(o: var TOverloadIter, c: PContext, n: PNode): PSym
 proc symChoiceExtension(o: var TOverloadIter; c: PContext; n: PNode): PSym =
  assert o.currentScope == nil
  while o.importIdx < c.imports.len:
-    result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config)
+    result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
    #while result != nil and result.id in o.marked:
    #  result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx])
    if result != nil:
@@ -602,12 +606,12 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
        else:
          o.importIdx = 0
          if c.imports.len > 0:
-            result = initIdentIter(o.it, o.marked, c.imports[o.importIdx], o.it.name).skipAlias(n, c.config)
+            result = initIdentIter(o.mit, o.marked, c.imports[o.importIdx], o.it.name, c.graph).skipAlias(n, c.config)
            if result == nil:
              result = nextOverloadIterImports(o, c, n)
          break
    elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config)
+      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
      if result == nil:
        result = nextOverloadIterImports(o, c, n)
    else:
@@ -615,7 +619,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
  of oimSelfModule:
    result = nextIdentIter(o.it, c.topLevelScope.symbols).skipAlias(n, c.config)
  of oimOtherModule:
-    result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
+    result = nextModuleIter(o.mit, c.graph).skipAlias(n, c.config)
  of oimSymChoice:
    if o.symChoiceIndex < n.len:
      result = n[o.symChoiceIndex].sym
@@ -654,7 +658,7 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
        incl o.marked, result.id

    elif o.importIdx < c.imports.len:
-      result = nextIdentIter(o.it, o.marked, c.imports[o.importIdx]).skipAlias(n, c.config)
+      result = nextIdentIter(o.mit, o.marked, c.imports[o.importIdx], c.graph).skipAlias(n, c.config)
      #assert result.id notin o.marked
      #while result != nil and result.id in o.marked:
      #  result = nextIdentIter(o.it, c.imports[o.importIdx]).skipAlias(n, c.config)
--- a/compiler/magicsys.nim
+++ b/compiler/magicsys.nim
@@ -26,7 +26,7 @@ proc newSysType(g: ModuleGraph; kind: TTypeKind, size: int): PType =
  result.align = size.int16

 proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
-  result = strTableGet(g.systemModule.tab, getIdent(g.cache, name))
+  result = systemModuleSym(g, getIdent(g.cache, name))
  if result == nil:
    localError(g.config, info, "system module needs: " & name)
    result = newSym(skError, getIdent(g.cache, name), nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
@@ -34,15 +34,12 @@ proc getSysSym*(g: ModuleGraph; info: TLineInfo; name: string): PSym =
  if result.kind == skAlias: result = result.owner

 proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSym =
-  var ti: TIdentIter
  let id = getIdent(g.cache, name)
-  var r = initIdentIter(ti, g.systemModule.tab, id)
-  while r != nil:
+  for r in systemModuleSyms(g, id):
    if r.magic == m:
      # prefer the tyInt variant:
      if r.typ[0] != nil and r.typ[0].kind == tyInt: return r
      result = r
-    r = nextIdentIter(ti, g.systemModule.tab)
  if result != nil: return result
  localError(g.config, info, "system module needs: " & name)
  result = newSym(skError, id, nextSymId(g.idgen), g.systemModule, g.systemModule.info, {})
--- a/compiler/modulegraphs.nim
+++ b/compiler/modulegraphs.nim
@@ -10,25 +10,11 @@
 ## This module implements the module graph data structure. The module graph
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
-##
-## The caching of modules is critical for 'nimsuggest' and is tricky to get
-## right. If module E is being edited, we need autocompletion (and type
-## checking) for E but we don't want to recompile depending
-## modules right away for faster turnaround times. Instead we mark the module's
-## dependencies as 'dirty'. Let D be a dependency of E. If D is dirty, we
-## need to recompile it and all of its dependencies that are marked as 'dirty'.
-## 'nimsuggest sug' actually is invoked for the file being edited so we know
-## its content changed and there is no need to compute any checksums.
-## Instead of a recursive algorithm, we use an iterative algorithm:
-##
-## - If a module gets recompiled, its dependencies need to be updated.
-## - Its dependent module stays the same.
-##

-import ast, intsets, tables, options, lineinfos, hashes, idents,
+import ast, astalgo, intsets, tables, options, lineinfos, hashes, idents,
  btrees, md5

-# import ic / packed_ast
+import ic / to_packed_ast

 type
  SigHash* = distinct MD5Digest
@@ -39,9 +25,12 @@ type
    converters*: seq[PSym]
    patterns*: seq[PSym]
    pureEnums*: seq[PSym]
+    interf: TStrTable

  ModuleGraph* = ref object
    ifaces*: seq[Iface]  ## indexed by int32 fileIdx
+    packed: PackedModuleGraph
+    startupPackedConfig*: PackedConfig
    packageSyms*: TStrTable
    deps*: IntSet # the dependency graph or potentially its transitive closure.
    importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
@@ -64,6 +53,7 @@ type
    sysTypes*: array[TTypeKind, PType]
    compilerprocs*: TStrTable
    exposed*: TStrTable
+    packageTypes*: TStrTable
    intTypeCache*: array[-5..64, PType]
    opContains*, opNot*: PSym
    emptyNode*: PNode
@@ -132,6 +122,62 @@ proc toBase64a(s: cstring, len: int): string =
    result.add cb64[a shr 2]
    result.add cb64[(a and 3) shl 4]

+template semtab*(m: PSym; g: ModuleGraph): TStrTable =
+  g.ifaces[m.position].interf
+
+proc cachedModule(g: ModuleGraph; m: PSym): bool {.inline.} =
+  m.position < g.packed.len and g.packed[m.position].status == loaded
+
+type
+  ModuleIter* = object
+    fromRod: bool
+    modIndex: int
+    ti: TIdentIter
+    rodIt: RodIter
+
+proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym =
+  assert m.kind == skModule
+  mi.modIndex = m.position
+  mi.fromRod = mi.modIndex < g.packed.len and g.packed[mi.modIndex].status == loaded
+  if mi.fromRod:
+    result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name)
+  else:
+    result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interf, name)
+
+proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym =
+  if mi.fromRod:
+    result = nextRodIter(mi.rodIt, g.packed)
+  else:
+    result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interf)
+
+iterator allSyms*(g: ModuleGraph; m: PSym): PSym =
+  if cachedModule(g, m):
+    var rodIt: RodIter
+    var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position)
+    while r != nil:
+      yield r
+      r = nextRodIter(rodIt, g.packed)
+  else:
+    for s in g.ifaces[m.position].interf.data:
+      if s != nil:
+        yield s
+
+proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym =
+  if cachedModule(g, m):
+    result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name)
+  else:
+    result = strTableGet(g.ifaces[m.position].interf, name)
+
+proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym =
+  result = someSym(g, g.systemModule, name)
+
+iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym =
+  var mi: ModuleIter
+  var r = initModuleIter(mi, g, g.systemModule, name)
+  while r != nil:
+    yield r
+    r = nextModuleIter(mi, g)
+
 proc `$`*(u: SigHash): string =
  toBase64a(cast[cstring](unsafeAddr u), sizeof(u))

@@ -186,6 +232,7 @@ proc registerModule*(g: ModuleGraph; m: PSym) =
  if m.position >= g.ifaces.len:
    setLen(g.ifaces, m.position + 1)
  g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[])
+  initStrTable(g.ifaces[m.position].interf)

 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
  result = ModuleGraph()
@@ -202,6 +249,7 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
  result.methods = @[]
  initStrTable(result.compilerprocs)
  initStrTable(result.exposed)
+  initStrTable(result.packageTypes)
  result.opNot = createMagic(result, "not", mNot)
  result.opContains = createMagic(result, "contains", mInSet)
  result.emptyNode = newNode(nkEmpty)
@@ -226,8 +274,11 @@ proc resetAllModules*(g: ModuleGraph) =
  initStrTable(g.exposed)

 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
-  if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len:
-    result = g.ifaces[fileIdx.int32].module
+  if fileIdx.int32 >= 0:
+    if fileIdx.int32 < g.packed.len and g.packed[fileIdx.int32].status == loaded:
+      result = g.packed[fileIdx.int32].module
+    elif fileIdx.int32 < g.ifaces.len:
+      result = g.ifaces[fileIdx.int32].module

 proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b

@@ -280,3 +331,18 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =

 proc isDirty*(g: ModuleGraph; m: PSym): bool =
  result = g.suggestMode and sfDirty in m.flags
+
+proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
+  result = s.ast[bodyPos]
+  if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf}:
+    result = loadProcBody(g.config, g.cache, g.packed, s)
+    s.ast[bodyPos] = result
+  assert result != nil
+
+proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## Returns 'nil' if the module needs to be recompiled.
+  if g.config.symbolFiles in {readOnlySf, v2Sf}:
+    result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx)
+
+proc configComplete*(g: ModuleGraph) =
+  rememberStartupConfig(g.startupPackedConfig, g.config)
--- a/compiler/modules.nim
+++ b/compiler/modules.nim
@@ -31,43 +31,49 @@ proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
    pck = getPackageName(graph.config, filename.string)
    pck2 = if pck.len > 0: pck else: "unknown"
    pack = getIdent(graph.cache, pck2)
-  var packSym = graph.packageSyms.strTableGet(pack)
-  if packSym == nil:
-    packSym = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info)
-    initStrTable(packSym.tab)
-    graph.packageSyms.strTableAdd(packSym)
+  result = graph.packageSyms.strTableGet(pack)
+  if result == nil:
+    result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info)
+    #initStrTable(packSym.tab)
+    graph.packageSyms.strTableAdd(result)
  else:
-    let existing = strTableGet(packSym.tab, name)
-    if existing != nil and existing.info.fileIndex != info.fileIndex:
-      when false:
-        # we used to produce an error:
-        localError(graph.config, info,
-          "module names need to be unique per Nimble package; module clashes with " &
-            toFullPath(graph.config, existing.info.fileIndex))
-      else:
-        # but starting with version 0.20 we now produce a fake Nimble package instead
-        # to resolve the conflicts:
-        let pck3 = fakePackageName(graph.config, filename)
-        # this makes the new `packSym`'s owner be the original `packSym`
-        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), packSym, info)
-        initStrTable(packSym.tab)
-        graph.packageSyms.strTableAdd(packSym)
-  result = packSym
+    # we now produce a fake Nimble package instead
+    # to resolve the conflicts:
+    let pck3 = fakePackageName(graph.config, filename)
+    # this makes the new `packSym`'s owner be the original `packSym`
+    result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info)
+    #initStrTable(packSym.tab)
+    graph.packageSyms.strTableAdd(result)
+
+    when false:
+      let existing = strTableGet(packSym.tab, name)
+      if existing != nil and existing.info.fileIndex != info.fileIndex:
+        when false:
+          # we used to produce an error:
+          localError(graph.config, info,
+            "module names need to be unique per Nimble package; module clashes with " &
+              toFullPath(graph.config, existing.info.fileIndex))
+        else:
+          # but starting with version 0.20 we now produce a fake Nimble package instead
+          # to resolve the conflicts:
+          let pck3 = fakePackageName(graph.config, filename)
+          # this makes the new `packSym`'s owner be the original `packSym`
+          packSym = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), packSym, info)
+          #initStrTable(packSym.tab)
+          graph.packageSyms.strTableAdd(packSym)

 proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
  let packSym = getPackage(graph, fileIdx)
  result.owner = packSym
  result.position = int fileIdx

-  graph.registerModule(result)
-
-  initStrTable(result.tab)
+  #initStrTable(result.tab(graph))
  when false:
    strTableAdd(result.tab, result) # a module knows itself
    # This is now implemented via
    #   c.moduleScope.addSym(module) # a module knows itself
    # in sem.nim, around line 527
-  strTableAdd(packSym.tab, result)
+  #strTableAdd(packSym.tab, result)

 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
@@ -79,6 +85,7 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
  if not isNimIdentifier(result.name.s):
    rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
  partialInitModule(result, graph, fileIdx, filename)
+  graph.registerModule(result)

 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
  var flags = flags
@@ -92,21 +99,20 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
      elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput)
    discard processModule(graph, result, idGeneratorFromModule(result), s)
  if result == nil:
+    result = moduleFromRodFile(graph, fileIdx)
    let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-    when false:
-      # XXX entry point for module loading from the rod file
-      result = loadModuleSym(graph, fileIdx, filename)
-    when true:
+    if result == nil:
      result = newModule(graph, fileIdx)
      result.flags.incl flags
      registerModule(graph, result)
+      processModuleAux()
    else:
      partialInitModule(result, graph, fileIdx, filename)
-    processModuleAux()
+      # XXX replay the pragmas here!
  elif graph.isDirty(result):
    result.flags.excl sfDirty
    # reset module fields:
-    initStrTable(result.tab)
+    initStrTable(result.semtab(graph))
    result.ast = nil
    processModuleAux()
    graph.markClientsDirty(fileIdx)
@@ -152,6 +158,8 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
  connectCallbacks(graph)
  let conf = graph.config
  wantMainModule(conf)
+  configComplete(graph)
+
  let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
  let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
  conf.projectMainIdx2 = projectFile
--- a/compiler/nimeval.nim
+++ b/compiler/nimeval.nim
@@ -24,11 +24,8 @@ type
 iterator exportedSymbols*(i: Interpreter): PSym =
  assert i != nil
  assert i.mainModule != nil, "no main module selected"
-  var it: TTabIter
-  var s = initTabIter(it, i.mainModule.tab)
-  while s != nil:
+  for s in modulegraphs.allSyms(i.graph, i.mainModule):
    yield s
-    s = nextIter(it, i.mainModule.tab)

 proc selectUniqueSymbol*(i: Interpreter; name: string;
                         symKinds: set[TSymKind] = {skLet, skVar}): PSym =
@@ -37,14 +34,14 @@ proc selectUniqueSymbol*(i: Interpreter; name: string;
  assert i != nil
  assert i.mainModule != nil, "no main module selected"
  let n = getIdent(i.graph.cache, name)
-  var it: TIdentIter
-  var s = initIdentIter(it, i.mainModule.tab, n)
+  var it: ModuleIter
+  var s = initModuleIter(it, i.graph, i.mainModule, n)
  result = nil
  while s != nil:
    if s.kind in symKinds:
      if result == nil: result = s
      else: return nil # ambiguous
-    s = nextIdentIter(it, i.mainModule.tab)
+    s = nextModuleIter(it, i.graph)

 proc selectRoutine*(i: Interpreter; name: string): PSym =
  ## Selects a declared routine (proc/func/etc) from the main module.
@@ -70,7 +67,7 @@ proc evalScript*(i: Interpreter; scriptStream: PLLStream = nil) =
  ## This can also be used to *reload* the script.
  assert i != nil
  assert i.mainModule != nil, "no main module selected"
-  initStrTable(i.mainModule.tab)
+  initStrTable(i.mainModule.semtab(i.graph))
  i.mainModule.ast = nil

  let s = if scriptStream != nil: scriptStream
--- a/compiler/packagehandling.nim
+++ b/compiler/packagehandling.nim
@@ -44,7 +44,8 @@ proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string =
  # in different directory get different name and they can be
  # placed in a directory.
  # foo-#head/../bar becomes @foo-@hhead@s..@sbar
-  result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace({$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})
+  result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace(
+    {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})

 proc demanglePackageName*(path: string): string =
  result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"})
--- a/compiler/passes.nim
+++ b/compiler/passes.nim
@@ -110,6 +110,12 @@ proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
 proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
  result = module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")

+proc partOfStdlib(x: PSym): bool =
+  var it = x.owner
+  while it != nil and it.kind == skPackage and it.owner != nil:
+    it = it.owner
+  result = it != nil and it.name.s == "stdlib"
+
 proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
                    stream: PLLStream): bool {.discardable.} =
  if graph.stopCompile(): return true
@@ -131,7 +137,7 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
  while true:
    openParser(p, fileIdx, s, graph.cache, graph.config)

-    if module.owner == nil or module.owner.name.s != "stdlib" or module.name.s == "distros":
+    if not partOfStdlib(module) or module.name.s == "distros":
      # XXX what about caching? no processing then? what if I change the
      # modules to include between compilation runs? we'd need to track that
      # in ROD files. I think we should enable this feature only
--- a/compiler/plugins/itersgen.nim
+++ b/compiler/plugins/itersgen.nim
@@ -9,7 +9,7 @@

 ## Plugin to transform an inline iterator into a data structure.

-import ".." / [ast, lookups, semdata, lambdalifting, msgs]
+import ".." / [ast, modulegraphs, lookups, semdata, lambdalifting, msgs]

 proc iterToProcImpl*(c: PContext, n: PNode): PNode =
  result = newNodeI(nkStmtList, n.info)
@@ -29,7 +29,7 @@ proc iterToProcImpl*(c: PContext, n: PNode): PNode =
    localError(c.config, n[2].info,
        "type must be a non-generic ref|ptr to object with state field")
    return
-  let body = liftIterToProc(c.graph, iter.sym, iter.sym.getBody, t, c.idgen)
+  let body = liftIterToProc(c.graph, iter.sym, getBody(c.graph, iter.sym), t, c.idgen)

  let prc = newSym(skProc, n[3].ident, nextSymId c.idgen, iter.sym.owner, iter.sym.info)
  prc.typ = copyType(iter.sym.typ, nextTypeId c.idgen, prc)
--- a/compiler/pragmas.nim
+++ b/compiler/pragmas.nim
@@ -14,6 +14,8 @@ import
  wordrecg, ropes, options, strutils, extccomp, math, magicsys, trees,
  types, lookups, lineinfos, pathutils, linter

+from ic / to_packed_ast import addCompilerProc
+
 const
  FirstCallConv* = wNimcall
  LastCallConv* = wNoconv
@@ -702,6 +704,8 @@ proc markCompilerProc(c: PContext; s: PSym) =
  incl(s.flags, sfCompilerProc)
  incl(s.flags, sfUsed)
  registerCompilerProc(c.graph, s)
+  if c.config.symbolFiles != disabledSf:
+    addCompilerProc(c.encoder, s)

 proc deprecatedStmt(c: PContext; outerPragma: PNode) =
  let pragma = outerPragma[1]
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -19,7 +19,8 @@ import
  lowerings, plugins/active, lineinfos, strtabs, int128,
  isolation_check, typeallowed

-from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward
+from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward,
+  systemModuleSym, semtab, getBody, someSym, allSyms

 when defined(nimfix):
  import nimfix/prettybase
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -268,7 +268,7 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext =
  result.typesWithOps = @[]
  result.features = graph.config.features
  if graph.config.symbolFiles != disabledSf:
-    initEncoder result.encoder, module, graph.config
+    initEncoder result.encoder, module, graph.config, graph.startupPackedConfig

 proc addIncludeFileDep*(c: PContext; f: FileIndex) =
  if c.config.symbolFiles != disabledSf:
@@ -286,15 +286,29 @@ proc inclSym(sq: var seq[PSym], s: PSym) =
 proc addConverter*(c: PContext, conv: PSym) =
  inclSym(c.converters, conv)
  inclSym(c.graph.ifaces[c.module.position].converters, conv)
-  #addConverter(c.graph, c.module, conv) # upcoming
+  if c.config.symbolFiles != disabledSf:
+    addConverter(c.encoder, conv)

 proc addPureEnum*(c: PContext, e: PSym) =
  inclSym(c.graph.ifaces[c.module.position].pureEnums, e)
+  if c.config.symbolFiles != disabledSf:
+    addPureEnum(c.encoder, e)

 proc addPattern*(c: PContext, p: PSym) =
  inclSym(c.patterns, p)
  inclSym(c.graph.ifaces[c.module.position].patterns, p)
-  #addPattern(c.graph, c.module, p) # upcoming
+  if c.config.symbolFiles != disabledSf:
+    addTrmacro(c.encoder, p)
+
+proc exportSym*(c: PContext; s: PSym) =
+  strTableAdd(c.module.semtab(c.graph), s)
+  if c.config.symbolFiles != disabledSf:
+    addExported(c.encoder, s)
+
+proc reexportSym*(c: PContext; s: PSym) =
+  strTableAdd(c.module.semtab(c.graph), s)
+  if c.config.symbolFiles != disabledSf:
+    addReexport(c.encoder, s)

 proc newLib*(kind: TLibKind): PLib =
  new(result)
@@ -489,4 +503,4 @@ proc storeRodNode*(c: PContext, n: PNode) =

 proc saveRodFile*(c: PContext) =
  if c.config.symbolFiles != disabledSf:
-    saveRodFile(toRodFile(c.config, c.filename.AbsoluteFile), c.encoder)
+    saveRodFile(toRodFile(c.config, AbsoluteFile toFullPath(c.config, FileIndex c.module.position)), c.encoder)
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -1014,7 +1014,7 @@ proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
 proc buildEchoStmt(c: PContext, n: PNode): PNode =
  # we MUST not check 'n' for semantics again here! But for now we give up:
  result = newNodeI(nkCall, n.info)
-  var e = strTableGet(c.graph.systemModule.tab, getIdent(c.cache, "echo"))
+  let e = systemModuleSym(c.graph, getIdent(c.cache, "echo"))
  if e != nil:
    result.add(newSymNode(e))
  else:
@@ -1897,7 +1897,7 @@ proc lookUpForDeclared(c: PContext, n: PNode, onlyCurrentScope: bool): PSym =
      if m == c.module:
        result = strTableGet(c.topLevelScope.symbols, ident)
      else:
-        result = strTableGet(m.tab, ident)
+        result = someSym(c.graph, m, ident)
  of nkSym:
    result = n.sym
  of nkOpenSymChoice, nkClosedSymChoice:
@@ -2504,7 +2504,7 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode =
    elif labl.owner == nil:
      labl.owner = c.p.owner
    n[0] = newSymNode(labl, n[0].info)
-    suggestSym(c.config, n[0].info, labl, c.graph.usageSym)
+    suggestSym(c.graph, n[0].info, labl, c.graph.usageSym)
    styleCheckDef(c.config, labl)
    onDef(n[0].info, labl)
  n[1] = semExpr(c, n[1], flags)
@@ -2522,15 +2522,12 @@ proc semExportExcept(c: PContext, n: PNode): PNode =
  let exceptSet = readExceptSet(c, n)
  let exported = moduleName.sym
  result = newNodeI(nkExportStmt, n.info)
-  strTableAdd(c.module.tab, exported)
-  var i: TTabIter
-  var s = initTabIter(i, exported.tab)
-  while s != nil:
+  reexportSym(c, exported)
+  for s in allSyms(c.graph, exported):
    if s.kind in ExportableSymKinds+{skModule} and
       s.name.id notin exceptSet and sfError notin s.flags:
-      strTableAdd(c.module.tab, s)
+      reexportSym(c, s)
      result.add newSymNode(s, n.info)
-    s = nextIter(i, exported.tab)
  markUsed(c, n.info, exported)

 proc semExport(c: PContext, n: PNode): PNode =
@@ -2548,15 +2545,12 @@ proc semExport(c: PContext, n: PNode): PNode =
      localError(c.config, a.info, errGenerated, "cannot export: " & renderTree(a))
    elif s.kind == skModule:
      # forward everything from that module:
-      strTableAdd(c.module.tab, s)
-      var ti: TTabIter
-      var it = initTabIter(ti, s.tab)
-      while it != nil:
+      reexportSym(c, s)
+      for it in allSyms(c.graph, s):
        if it.kind in ExportableSymKinds+{skModule}:
-          strTableAdd(c.module.tab, it)
+          reexportSym(c, it)
          result.add newSymNode(it, a.info)
          specialSyms(c, it)
-        it = nextIter(ti, s.tab)
      markUsed(c, n.info, s)
    else:
      while s != nil:
@@ -2565,7 +2559,7 @@ proc semExport(c: PContext, n: PNode): PNode =
            "; enum field cannot be exported individually")
        if s.kind in ExportableSymKinds+{skModule} and sfError notin s.flags:
          result.add(newSymNode(s, a.info))
-          strTableAdd(c.module.tab, s)
+          reexportSym(c, s)
          markUsed(c, n.info, s)
          specialSyms(c, s)
          if s.kind == skType and sfPure notin s.flags:
@@ -2575,7 +2569,7 @@ proc semExport(c: PContext, n: PNode): PNode =
                var e = etyp.n[j].sym
                if e.kind != skEnumField:
                  internalError(c.config, s.info, "rawImportSymbol")
-                strTableAdd(c.module.tab, e)
+                reexportSym(c, e)

        s = nextOverloadIter(o, c, a)

--- a/compiler/semfields.nim
+++ b/compiler/semfields.nim
@@ -106,7 +106,7 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode =
  # so that 'break' etc. work as expected, we produce
  # a 'while true: stmt; break' loop ...
  result = newNodeI(nkWhileStmt, n.info, 2)
-  var trueSymbol = strTableGet(c.graph.systemModule.tab, getIdent(c.cache, "true"))
+  var trueSymbol = systemModuleSym(c.graph, getIdent(c.cache, "true"))
  if trueSymbol == nil:
    localError(c.config, n.info, "system needs: 'true'")
    trueSymbol = newSym(skUnknown, getIdent(c.cache, "true"), nextSymId c.idgen, getCurrOwner(c), n.info)
--- a/compiler/semgnrc.nim
+++ b/compiler/semgnrc.nim
@@ -481,7 +481,7 @@ proc semGenericStmt(c: PContext, n: PNode,
      if sfGenSym in s.flags and s.ast == nil:
        body = n[bodyPos]
      else:
-        body = s.getBody
+        body = getBody(c.graph, s)
    else: body = n[bodyPos]
    n[bodyPos] = semGenericStmtScope(c, body, flags, ctx)
    closeScope(c)
--- a/compiler/seminst.nim
+++ b/compiler/seminst.nim
@@ -160,7 +160,7 @@ proc fixupInstantiatedSymbols(c: PContext, s: PSym) =
      pushInfoContext(c.config, oldPrc.info)
      openScope(c)
      var n = oldPrc.ast
-      n[bodyPos] = copyTree(s.getBody)
+      n[bodyPos] = copyTree(getBody(c.graph, s))
      instantiateBody(c, n, oldPrc.typ.n, oldPrc, s)
      closeScope(c)
      popInfoContext(c.config)
@@ -383,7 +383,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
    if n[pragmasPos].kind != nkEmpty:
      pragma(c, result, n[pragmasPos], allRoutinePragmas)
    if isNil(n[bodyPos]):
-      n[bodyPos] = copyTree(fn.getBody)
+      n[bodyPos] = copyTree(getBody(c.graph, fn))
    if c.inGenericContext == 0:
      instantiateBody(c, n, fn.typ.n, result, fn)
    sideEffectsCheck(c, result)
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -66,7 +66,7 @@ proc semBreakOrContinue(c: PContext, n: PNode): PNode =
        x.info = n.info
        incl(s.flags, sfUsed)
        n[0] = x
-        suggestSym(c.config, x.info, s, c.graph.usageSym)
+        suggestSym(c.graph, x.info, s, c.graph.usageSym)
        onUse(x.info, s)
      else:
        localError(c.config, n.info, errInvalidControlFlowX % s.name.s)
@@ -332,7 +332,7 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
      discard
    result = n.info
  let info = getLineInfo(n)
-  suggestSym(c.config, info, result, c.graph.usageSym)
+  suggestSym(c.graph, info, result, c.graph.usageSym)

 proc checkNilable(c: PContext; v: PSym) =
  if {sfGlobal, sfImportc} * v.flags == {sfGlobal} and v.typ.requiresInit:
@@ -1055,14 +1055,14 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) =
    if pkg.isNil or pkg.kind != skPackage:
      localError(c.config, name.info, "unknown package name: " & pkgName.s)
    else:
-      let typsym = pkg.tab.strTableGet(typName)
+      let typsym = c.graph.packageTypes.strTableGet(typName)
      if typsym.isNil:
        s = semIdentDef(c, name[1], skType)
        onDef(name[1].info, s)
        s.typ = newTypeS(tyObject, c)
        s.typ.sym = s
        s.flags.incl sfForward
-        pkg.tab.strTableAdd s
+        c.graph.packageTypes.strTableAdd s
        addInterfaceDecl(c, s)
      elif typsym.kind == skType and sfForward in typsym.flags:
        s = typsym
@@ -1088,7 +1088,7 @@ proc typeDefLeftSidePass(c: PContext, typeSection: PNode, i: int) =
      if not isTopLevel(c) or pkg.isNil:
        localError(c.config, name.info, "only top level types in a package can be 'package'")
      else:
-        let typsym = pkg.tab.strTableGet(s.name)
+        let typsym = c.graph.packageTypes.strTableGet(s.name)
        if typsym != nil:
          if sfForward notin typsym.flags or sfNoForward notin typsym.flags:
            typeCompleted(typsym)
@@ -1956,7 +1956,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
    if not comesFromShadowScope:
      excl(proto.flags, sfForward)
      incl(proto.flags, sfWasForwarded)
-    suggestSym(c.config, s.info, proto, c.graph.usageSym)
+    suggestSym(c.graph, s.info, proto, c.graph.usageSym)
    closeScope(c)         # close scope with wrong parameter symbols
    openScope(c)          # open scope for old (correct) parameter symbols
    if proto.ast[genericParamsPos].kind != nkEmpty:
--- a/compiler/semtempl.nim
+++ b/compiler/semtempl.nim
@@ -250,7 +250,7 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
    else: result = newSymNode(s, n.info)
    # Issue #12832
    when defined(nimsuggest):
-      suggestSym(c.config, n.info, s, c.graph.usageSym, false)
+      suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
    if {optStyleHint, optStyleError} * c.config.globalOptions != {}:
      styleCheckUse(c.config, n.info, s)

--- a/compiler/semtypes.nim
+++ b/compiler/semtypes.nim
@@ -140,7 +140,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
    if result.sym != nil and sfExported in result.sym.flags:
      incl(e.flags, sfUsed)
      incl(e.flags, sfExported)
-      if not isPure: strTableAdd(c.module.tab, e)
+      if not isPure: exportSym(c, e)
    result.n.add symNode
    styleCheckDef(c.config, e)
    onDef(e.info, e)
@@ -779,7 +779,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int,
                   n[i][1].info
                 else:
                   n[i].info
-      suggestSym(c.config, info, f, c.graph.usageSym)
+      suggestSym(c.graph, info, f, c.graph.usageSym)
      f.typ = typ
      f.position = pos
      f.options = c.config.options
--- a/compiler/sighashes.nim
+++ b/compiler/sighashes.nim
@@ -371,7 +371,7 @@ proc symBodyDigest*(graph: ModuleGraph, sym: PSym): SigHash =
  if sym.ast != nil:
    md5Init(c)
    c.md5Update(cast[cstring](result.addr), sizeof(result))
-    hashBodyTree(graph, c, sym.ast[bodyPos])
+    hashBodyTree(graph, c, getBody(graph, sym))
    c.md5Final(result.MD5Digest)
    graph.symBodyHashes[sym.id] = result

--- a/compiler/suggest.nim
+++ b/compiler/suggest.nim
@@ -56,10 +56,10 @@ proc findDocComment(n: PNode): PNode =
  elif n.kind in {nkAsgn, nkFastAsgn} and n.len == 2:
    result = findDocComment(n[1])

-proc extractDocComment(s: PSym): string =
+proc extractDocComment(g: ModuleGraph; s: PSym): string =
  var n = findDocComment(s.ast)
  if n.isNil and s.kind in routineKinds and s.ast != nil:
-    n = findDocComment(s.ast[bodyPos])
+    n = findDocComment(getBody(g, s))
  if not n.isNil:
    result = n.comment.replace("\n##", "\n").strip
  else:
@@ -117,7 +117,7 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int
    elif sourceIdent != ident:
      result = 0

-proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
+proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
                  quality: range[0..100]; prefix: PrefixMatch;
                  inTypeContext: bool; scope: int;
                  useSuppliedInfo = false): Suggest =
@@ -136,7 +136,7 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
      if u.fileIndex == info.fileIndex: inc c
    result.localUsages = c
  result.symkind = byte s.kind
-  if optIdeTerse notin conf.globalOptions:
+  if optIdeTerse notin g.config.globalOptions:
    result.qualifiedPath = @[]
    if not isLocal and s.kind != skModule:
      let ow = s.owner
@@ -156,20 +156,20 @@ proc symToSuggest(conf: ConfigRef; s: PSym, isLocal: bool, section: IdeCmd, info
    else:
      result.forth = ""
    when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
-      result.doc = s.extractDocComment
+      result.doc = extractDocComment(g, s)
  let infox =
    if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}:
      info
    else:
      s.info
-  result.filePath = toFullPath(conf, infox)
+  result.filePath = toFullPath(g.config, infox)
  result.line = toLinenumber(infox)
  result.column = toColumn(infox)
-  result.version = conf.suggestVersion
+  result.version = g.config.suggestVersion
  result.tokenLen = if section != ideHighlight:
                      s.name.s.len
                    else:
-                      getTokenLenFromSource(conf, s.name.s, infox)
+                      getTokenLenFromSource(g.config, s.name.s, infox)

 proc `$`*(suggest: Suggest): string =
  result = $suggest.section
@@ -261,7 +261,7 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} =
 proc suggestField(c: PContext, s: PSym; f: PNode; info: TLineInfo; outputs: var Suggestions) =
  var pm: PrefixMatch
  if filterSym(s, f, pm) and fieldVisible(c, s):
-    outputs.add(symToSuggest(c.config, s, isLocal=true, ideSug, info, 100, pm, c.inTypeContext > 0, 0))
+    outputs.add(symToSuggest(c.graph, s, isLocal=true, ideSug, info, 100, pm, c.inTypeContext > 0, 0))

 proc getQuality(s: PSym): range[0..100] =
  if s.typ != nil and s.typ.len > 1:
@@ -275,7 +275,7 @@ template wholeSymTab(cond, section: untyped) {.dirty.} =
    let it = item
    var pm: PrefixMatch
    if cond:
-      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, section, info, getQuality(it),
+      outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, section, info, getQuality(it),
                                pm, c.inTypeContext > 0, scopeN))

 proc suggestSymList(c: PContext, list, f: PNode; info: TLineInfo, outputs: var Suggestions) =
@@ -346,7 +346,7 @@ proc suggestEverything(c: PContext, n, f: PNode, outputs: var Suggestions) =
  for (it, scopeN, isLocal) in allSyms(c):
    var pm: PrefixMatch
    if filterSym(it, f, pm):
-      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug, n.info, 0, pm,
+      outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug, n.info, 0, pm,
                                c.inTypeContext > 0, scopeN))

 proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions) =
@@ -365,10 +365,10 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
        let m = c.graph.importModuleCallback(c.graph, c.module, fileInfoIdx(c.config, fullPath))
        if m == nil: typ = nil
        else:
-          for it in items(n.sym.tab):
+          for it in allSyms(c.graph, n.sym):
            if filterSym(it, field, pm):
-              outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -100))
-          outputs.add(symToSuggest(c.config, m, isLocal=false, ideMod, n.info, 100, PrefixMatch.None,
+              outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -100))
+          outputs.add(symToSuggest(c.graph, m, isLocal=false, ideMod, n.info, 100, PrefixMatch.None,
            c.inTypeContext > 0, -99))

  if typ == nil:
@@ -378,11 +378,11 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
        # all symbols accessible, because we are in the current module:
        for it in items(c.topLevelScope.symbols):
          if filterSym(it, field, pm):
-            outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
+            outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
      else:
-        for it in items(n.sym.tab):
+        for it in allSyms(c.graph, n.sym):
          if filterSym(it, field, pm):
-            outputs.add(symToSuggest(c.config, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
+            outputs.add(symToSuggest(c.graph, it, isLocal=false, ideSug, n.info, 100, pm, c.inTypeContext > 0, -99))
    else:
      # fallback:
      suggestEverything(c, n, field, outputs)
@@ -440,27 +440,27 @@ when defined(nimsuggest):
      if infoB.infoToInt == infoAsInt: return
    s.allUsages.add(info)

-proc findUsages(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) =
-  if conf.suggestVersion == 1:
-    if usageSym == nil and isTracked(info, conf.m.trackPos, s.name.s.len):
+proc findUsages(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym) =
+  if g.config.suggestVersion == 1:
+    if usageSym == nil and isTracked(info, g.config.m.trackPos, s.name.s.len):
      usageSym = s
-      suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
+      suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
    elif s == usageSym:
-      if conf.lastLineInfo != info:
-        suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
-      conf.lastLineInfo = info
+      if g.config.lastLineInfo != info:
+        suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideUse, info, 100, PrefixMatch.None, false, 0))
+      g.config.lastLineInfo = info

 when defined(nimsuggest):
-  proc listUsages*(conf: ConfigRef; s: PSym) =
+  proc listUsages*(g: ModuleGraph; s: PSym) =
    #echo "usages ", s.allUsages.len
    for info in s.allUsages:
      let x = if info == s.info and info.col == s.info.col: ideDef else: ideUse
-      suggestResult(conf, symToSuggest(conf, s, isLocal=false, x, info, 100, PrefixMatch.None, false, 0))
+      suggestResult(g.config, symToSuggest(g, s, isLocal=false, x, info, 100, PrefixMatch.None, false, 0))

-proc findDefinition(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym) =
+proc findDefinition(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym) =
  if s.isNil: return
-  if isTracked(info, conf.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags):
-    suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym))
+  if isTracked(info, g.config.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags):
+    suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym))
    if sfForward notin s.flags:
      suggestQuit()
    else:
@@ -472,8 +472,9 @@ proc ensureIdx[T](x: var T, y: int) =
 proc ensureSeq[T](x: var seq[T]) =
  if x == nil: newSeq(x, 0)

-proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.inline.} =
+proc suggestSym*(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym; isDecl=true) {.inline.} =
  ## misnamed: should be 'symDeclared'
+  let conf = g.config
  when defined(nimsuggest):
    if conf.suggestVersion == 0:
      if s.allUsages.len == 0:
@@ -482,15 +483,15 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym;
        s.addNoDup(info)

    if conf.ideCmd == ideUse:
-      findUsages(conf, info, s, usageSym)
+      findUsages(g, info, s, usageSym)
    elif conf.ideCmd == ideDef:
-      findDefinition(conf, info, s, usageSym)
+      findDefinition(g, info, s, usageSym)
    elif conf.ideCmd == ideDus and s != nil:
      if isTracked(info, conf.m.trackPos, s.name.s.len):
-        suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0))
-      findUsages(conf, info, s, usageSym)
+        suggestResult(conf, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0))
+      findUsages(g, info, s, usageSym)
    elif conf.ideCmd == ideHighlight and info.fileIndex == conf.m.trackPos.fileIndex:
-      suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideHighlight, info, 100, PrefixMatch.None, false, 0))
+      suggestResult(conf, symToSuggest(g, s, isLocal=false, ideHighlight, info, 100, PrefixMatch.None, false, 0))
    elif conf.ideCmd == ideOutline and isDecl:
      # if a module is included then the info we have is inside the include and
      # we need to walk up the owners until we find the outer most module,
@@ -503,7 +504,7 @@ proc suggestSym*(conf: ConfigRef; info: TLineInfo; s: PSym; usageSym: var PSym;
        parentModule = parentModule.owner

      if parentFileIndex == conf.m.trackPos.fileIndex:
-        suggestResult(conf, symToSuggest(conf, s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0))
+        suggestResult(conf, symToSuggest(g, s, isLocal=false, ideOutline, info, 100, PrefixMatch.None, false, 0))

 proc extractPragma(s: PSym): PNode =
  if s.kind in routineKinds:
@@ -572,7 +573,7 @@ proc markUsed(c: PContext; info: TLineInfo; s: PSym) =

    if sfError in s.flags: userError(conf, info, s)
  when defined(nimsuggest):
-    suggestSym(conf, info, s, c.graph.usageSym, false)
+    suggestSym(c.graph, info, s, c.graph.usageSym, false)
  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
    styleCheckUse(conf, info, s)
  markOwnerModuleAsUsed(c, s)
@@ -658,7 +659,7 @@ proc suggestSentinel*(c: PContext) =
  for (it, scopeN, isLocal) in allSyms(c):
    var pm: PrefixMatch
    if filterSymNoOpr(it, nil, pm):
-      outputs.add(symToSuggest(c.config, it, isLocal = isLocal, ideSug,
+      outputs.add(symToSuggest(c.graph, it, isLocal = isLocal, ideSug,
          newLineInfo(c.config.m.trackPos.fileIndex, 0, -1), 0,
          PrefixMatch.None, false, scopeN))

--- a/compiler/transf.nim
+++ b/compiler/transf.nim
@@ -126,7 +126,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
  var tc = c.transCon
  if sfBorrow in s.flags and s.kind in routineKinds:
    # simply exchange the symbol:
-    b = s.getBody
+    b = getBody(c.graph, s)
    if b.kind != nkSym: internalError(c.graph.config, n.info, "wrong AST for borrowed symbol")
    b = newSymNode(b.sym, n.info)
  elif c.inlining > 0:
@@ -594,7 +594,7 @@ proc findWrongOwners(c: PTransf, n: PNode) =
  else:
    for i in 0..<n.safeLen: findWrongOwners(c, n[i])

-proc isSimpleIteratorVar(iter: PSym): bool =
+proc isSimpleIteratorVar(c: PTransf; iter: PSym): bool =
  proc rec(n: PNode; owner: PSym; dangerousYields: var int) =
    case n.kind
    of nkEmpty..nkNilLit: discard
@@ -607,7 +607,7 @@ proc isSimpleIteratorVar(iter: PSym): bool =
      for c in n: rec(c, owner, dangerousYields)

  var dangerousYields = 0
-  rec(iter.ast[bodyPos], iter, dangerousYields)
+  rec(getBody(c.graph, iter), iter, dangerousYields)
  result = dangerousYields == 0

 proc transformFor(c: PTransf, n: PNode): PNode =
@@ -650,7 +650,7 @@ proc transformFor(c: PTransf, n: PNode): PNode =
      for j in 0..<n[i].len-1:
        addVar(v, copyTree(n[i][j])) # declare new vars
    else:
-      if n[i].kind == nkSym and isSimpleIteratorVar(iter):
+      if n[i].kind == nkSym and isSimpleIteratorVar(c, iter):
        incl n[i].sym.flags, sfCursor
      addVar(v, copyTree(n[i])) # declare new vars
  stmtList.add(v)
@@ -1087,12 +1087,12 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; cache: bool):

  if prc.transformedBody != nil:
    result = prc.transformedBody
-  elif nfTransf in prc.ast[bodyPos].flags or prc.kind in {skTemplate}:
-    result = prc.ast[bodyPos]
+  elif nfTransf in getBody(g, prc).flags or prc.kind in {skTemplate}:
+    result = getBody(g, prc)
  else:
    prc.transformedBody = newNode(nkEmpty) # protects from recursion
    var c = openTransf(g, prc.getModule, "", idgen)
-    result = liftLambdas(g, prc, prc.ast[bodyPos], c.tooEarly, c.idgen)
+    result = liftLambdas(g, prc, getBody(g, prc), c.tooEarly, c.idgen)
    result = processTransf(c, result, prc)
    liftDefer(c, result)
    result = liftLocalsIfRequested(prc, result, g.cache, g.config, c.idgen)
@@ -1108,6 +1108,7 @@ proc transformBody*(g: ModuleGraph; idgen: IdGenerator; prc: PSym; cache: bool):
      prc.transformedBody = result
    else:
      prc.transformedBody = nil
+    # XXX Rodfile support for transformedBody!

  #if prc.name.s == "main":
  #  echo "transformed into ", renderTree(result, {renderIds})
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -1204,7 +1204,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
          VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]),
                 currentException: c.currentExceptionA,
                 currentLineInfo: c.debug[pc]))
-      elif importcCond(prc):
+      elif importcCond(c, prc):
        if compiletimeFFI notin c.config.features:
          globalError(c.config, c.debug[pc], "VM not allowed to do FFI, see `compiletimeFFI`")
        # we pass 'tos.slots' instead of 'regs' so that the compiler can keep
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -31,6 +31,8 @@ import
  strutils, ast, types, msgs, renderer, vmdef,
  intsets, magicsys, options, lowerings, lineinfos, transf

+from modulegraphs import getBody
+
 const
  debugEchoCode* = defined(nimVMDebug)

@@ -1570,11 +1572,11 @@ proc genTypeLit(c: PCtx; t: PType; dest: var TDest) =
  n.typ = t
  genLit(c, n, dest)

-proc importcCond*(s: PSym): bool {.inline.} =
+proc importcCond*(c: PCtx; s: PSym): bool {.inline.} =
  ## return true to importc `s`, false to execute its body instead (refs #8405)
  if sfImportc in s.flags:
    if s.kind in routineKinds:
-      return s.ast[bodyPos].kind == nkEmpty
+      return getBody(c.graph, s).kind == nkEmpty

 proc importcSym(c: PCtx; info: TLineInfo; s: PSym) =
  when hasFFI:
@@ -1615,7 +1617,7 @@ proc genRdVar(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
    elif s.position == 0:
      cannotEval(c, n)
    if s.position == 0:
-      if importcCond(s) or isImportcVar: c.importcSym(n.info, s)
+      if importcCond(c, s) or isImportcVar: c.importcSym(n.info, s)
      else: genGlobalInit(c, n, s)
    if dest < 0: dest = c.getTemp(n.typ)
    assert s.typ != nil
@@ -1841,7 +1843,7 @@ proc genVarSection(c: PCtx; n: PNode) =
      checkCanEval(c, a[0])
      if s.isGlobal:
        if s.position == 0:
-          if importcCond(s): c.importcSym(a.info, s)
+          if importcCond(c, s): c.importcSym(a.info, s)
          else:
            let sa = getNullValue(s.typ, a.info, c.config)
            #if s.ast.isNil: getNullValue(s.typ, a.info)
@@ -1964,6 +1966,7 @@ proc matches(s: PSym; x: string): bool =
    if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"):
      return false
    s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner
+    while s != nil and s.kind == skPackage and s.owner != nil: s = s.owner
  result = true

 proc procIsCallback(c: PCtx; s: PSym): bool =
@@ -1989,7 +1992,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
      if s.kind == skIterator and s.typ.callConv == TCallingConvention.ccClosure:
        globalError(c.config, n.info, "Closure iterators are not supported by VM!")
      if procIsCallback(c, s): discard
-      elif importcCond(s): c.importcSym(n.info, s)
+      elif importcCond(c, s): c.importcSym(n.info, s)
      genLit(c, n, dest)
    of skConst:
      let constVal = if s.ast != nil: s.ast else: s.typ.n
--- a/nimsuggest/nimsuggest.nim
+++ b/nimsuggest/nimsuggest.nim
@@ -195,7 +195,7 @@ proc executeNoHooks(cmd: IdeCmd, file, dirtyfile: AbsoluteFile, line, col: int;
  if conf.ideCmd in {ideUse, ideDus}:
    let u = if conf.suggestVersion != 1: graph.symFromInfo(conf.m.trackPos) else: graph.usageSym
    if u != nil:
-      listUsages(conf, u)
+      listUsages(graph, u)
    else:
      localError(conf, conf.m.trackPos, "found no symbol at this position " & (conf $ conf.m.trackPos))