progress

compiler/ast2nif.nim

@@ -13,7 +13,7 @@ import std / [assertions, tables, sets]
 from std / strutils import startsWith
 import astdef, idents, msgs, options
 import lineinfos as astli
-import pathutils
+import pathutils #, modulegraphs
 import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos,
   nifindexes, nifreader]
 import "../dist/nimony/src/gear2" / modnames
@@ -258,6 +258,10 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) =
 proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) =
   dest.addParLe sdefTag, trLineInfo(w, sym.infoImpl)
   dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo
+  if sfExported in sym.flagsImpl:
+    dest.addIdent "x"
+  else:
+    dest.addDotToken
   if sym.magicImpl == mNone:
     dest.addDotToken
   else:
@@ -537,48 +541,37 @@ type
 
   DecodeContext* = object
     infos: LineInfoWriter
-    moduleIds: Table[string, int32]
+    #moduleIds: Table[string, int32]
     types: Table[ItemId, (PType, NifIndexEntry)]
     syms: Table[ItemId, (PSym, NifIndexEntry)]
     mods: seq[NifModule]
     cache: IdentCache
-    moduleToNifSuffix: Table[FileIndex, string]
+    #moduleToNifSuffix: Table[FileIndex, string]
 
 proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext =
   ## Supposed to be a global variable
   result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache)
 
-proc idToIdx(x: int32): int {.inline.} =
-  assert x <= -2'i32
-  result = -(x+2)
-
-proc cursorFromIndexEntry(c: var DecodeContext; module: int32; entry: NifIndexEntry;
+proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry;
                           buf: var TokenBuf): Cursor =
-  let m = idToIdx(module)
-  let s = addr c.mods[m].stream
+  let s = addr c.mods[module.int32].stream
   s.r.jumpTo entry.offset
   var buf = createTokenBuf(30)
   nifcursors.parse(s[], buf, entry.info)
   result = cursorAt(buf, 0)
 
-proc moduleId(c: var DecodeContext; suffix: string): int32 =
-  # We don't know the "real" FileIndex due to our mapping to a short "Module suffix"
-  # This is not a problem, we use negative `ItemId.module` values here and then
-  # there is no interference with in-memory-modules. Modulegraphs.nim already uses -1
-  # so we start at -2 here.
-  result = c.moduleIds.getOrDefault(suffix)
-  if result == 0:
-    result = -int32(c.moduleIds.len + 2) # negative index!
+proc moduleId(c: var DecodeContext; suffix: string): FileIndex =
+  var isKnownFile = false
+  result = c.infos.config.registerNifSuffix(suffix, isKnownFile)
+  if not isKnownFile:
     let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string
     let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".idx.nif")).string
-    c.moduleIds[suffix] = result
-    c.mods.add NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile))
-    assert c.mods.len-1 == idToIdx(result)
+    if result.int >= c.mods.len:
+      c.mods.setLen(result.int + 1)
+    c.mods[result.int] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile))
 
-proc getOffset(c: var DecodeContext; module: int32; nifName: string): NifIndexEntry =
-  assert module < 0'i32
-  let index = idToIdx(module)
-  let ii = addr c.mods[index].index
+proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry =
+  let ii = addr c.mods[module.int32].index
   result = ii.public.getOrDefault(nifName)
   if result.offset == 0:
     result = ii.private.getOrDefault(nifName)
@@ -602,10 +595,10 @@ proc loadTypeStub(c: var DecodeContext; t: SymId): PType =
     inc i
   if i < name.len and name[i] == '.': inc i
   let suffix = name.substr(i)
-  let id = ItemId(module: moduleId(c, suffix), item: itemId)
+  let id = ItemId(module: moduleId(c, suffix).int32, item: itemId)
   result = c.types.getOrDefault(id)[0]
   if result == nil:
-    let offs = c.getOffset(id.module, name)
+    let offs = c.getOffset(id.module.FileIndex, name)
     result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial)
     c.types[id] = (result, offs)
 
@@ -628,10 +621,10 @@ proc loadSymStub(c: var DecodeContext; t: SymId): PSym =
   let symAsStr = pool.syms[t]
   let sn = parseSymName(symAsStr)
   let module = moduleId(c, sn.module)
-  let val = addr c.mods[idToIdx(module)].symCounter
+  let val = addr c.mods[module.int32].symCounter
   inc val[]
 
-  let id = ItemId(module: module, item: val[])
+  let id = ItemId(module: module.int32, item: val[])
   result = c.syms.getOrDefault(id)[0]
   if result == nil:
     let offs = c.getOffset(module, symAsStr)
@@ -697,7 +690,7 @@ proc loadType*(c: var DecodeContext; t: PType) =
   if t.state != Partial: return
   t.state = Sealed
   var buf = createTokenBuf(30)
-  var n = cursorFromIndexEntry(c, t.itemId.module, c.types[t.itemId][1], buf)
+  var n = cursorFromIndexEntry(c, t.itemId.module.FileIndex, c.types[t.itemId][1], buf)
 
   expect n, ParLe
   if n.tagId != tdefTag:
@@ -746,7 +739,7 @@ proc loadSym*(c: var DecodeContext; s: PSym) =
   if s.state != Partial: return
   s.state = Sealed
   var buf = createTokenBuf(30)
-  var n = cursorFromIndexEntry(c, s.itemId.module, c.syms[s.itemId][1], buf)
+  var n = cursorFromIndexEntry(c, s.itemId.module.FileIndex, c.syms[s.itemId][1], buf)
 
   expect n, ParLe
   if n.tagId != sdefTag:
@@ -755,6 +748,17 @@ proc loadSym*(c: var DecodeContext; s: PSym) =
   expect n, SymbolDef
   # ignore the symbol's name, we have already used it to create this PSym instance!
   inc n
+  if n.kind == Ident:
+    if pool.strings[n.litId] == "x":
+      s.flagsImpl.incl sfExported
+      inc n
+    else:
+      raiseAssert "expected `x` as the export marker"
+  elif n.kind == DotToken:
+    inc n
+  else:
+    raiseAssert "expected `x` or '.' but got " & $n.kind
+
   loadField s.magicImpl
   loadField s.flagsImpl
   loadField s.optionsImpl
@@ -895,20 +899,20 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode =
   else:
     raiseAssert "Not yet implemented " & $n.kind
 
+when false:
+  proc loadNifModule*(c: var DecodeContext; f: FileIndex): PNode =
+    let moduleSuffix = moduleSuffix(c.infos.config, f)
+    let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(moduleSuffix), ".nif").string
 
-proc loadNifModule*(c: var DecodeContext; f: FileIndex): PNode =
-  let moduleSuffix = modname(c.moduleToNifSuffix, f.int, c.infos.config)
-  let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(moduleSuffix), ".nif").string
-
-  var buf = createTokenBuf(300)
-  var s = nifstreams.open(modFile)
-  # XXX We can optimize this here and only load the top level entries!
-  try:
-    nifcursors.parse(s, buf, NoLineInfo)
-  finally:
-    nifstreams.close(s)
-  var n = cursorAt(buf, 0)
-  result = loadNode(c, n)
+    var buf = createTokenBuf(300)
+    var s = nifstreams.open(modFile)
+    # XXX We can optimize this here and only load the top level entries!
+    try:
+      nifcursors.parse(s, buf, NoLineInfo)
+    finally:
+      nifstreams.close(s)
+    var n = cursorAt(buf, 0)
+    result = loadNode(c, n)
 
 when isMainModule:
   import std / syncio

compiler/ccgexprs.nim

@@ -3435,7 +3435,7 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) =
 
 proc genConstStmt(p: BProc, n: PNode) =
   # This code is only used in the new DCE implementation.
-  assert useAliveDataFromDce in p.module.flags
+  assert delayedCodegen(p.module)
   let m = p.module
   for it in n:
     if it[0].kind == nkSym:
@@ -3453,7 +3453,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     var sym = n.sym
     case sym.kind
     of skMethod:
-      if useAliveDataFromDce in p.module.flags or {sfDispatcher, sfForward} * sym.flags != {}:
+      if delayedCodegen(p.module) or {sfDispatcher, sfForward} * sym.flags != {}:
         # we cannot produce code for the dispatcher yet:
         fillProcLoc(p.module, n)
         genProcPrototype(p.module, sym)
@@ -3466,7 +3466,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
       if sfCompileTime in sym.flags:
         localError(p.config, n.info, "request to generate code for .compileTime proc: " &
            sym.name.s)
-      if useAliveDataFromDce in p.module.flags and sym.typ.callConv != ccInline:
+      if delayedCodegen(p.module) and sym.typ.callConv != ccInline:
         fillProcLoc(p.module, n)
         genProcPrototype(p.module, sym)
       else:
@@ -3479,7 +3479,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
         var lit = newBuilder("")
         genLiteral(p, sym.astdef, sym.typ, lit)
         putIntoDest(p, d, n, extract(lit), OnStatic)
-      elif useAliveDataFromDce in p.module.flags:
+      elif delayedCodegen(p.module):
         genConstHeader(p.module, p.module, p, sym)
         assert((sym.loc.snippet != "") and (sym.loc.t != nil))
         putLocIntoDest(p, d, sym.loc)
@@ -3611,7 +3611,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkWhileStmt: genWhileStmt(p, n)
   of nkVarSection, nkLetSection: genVarStmt(p, n)
   of nkConstSection:
-    if useAliveDataFromDce in p.module.flags:
+    if delayedCodegen(p.module):
       genConstStmt(p, n)
     else: # enforce addressable consts for exportc
       let m = p.module
@@ -3677,7 +3677,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     if n[genericParamsPos].kind == nkEmpty:
       var prc = n[namePos].sym
-      if useAliveDataFromDce in p.module.flags:
+      if delayedCodegen(p.module):
         if p.module.alive.contains(prc.itemId.item) and
             prc.magic in generatedMagics:
           genProc(p.module, prc)

compiler/cgen.nim

@@ -1445,6 +1445,7 @@ proc genProcPrototype(m: BModule, sym: PSym) =
               getModuleDllPath(m, sym),
               '"' & name & '"')
   elif not containsOrIncl(m.declaredProtos, sym.id):
+    m.queue.add(sym)
     let asPtr = isReloadable(m, sym)
     var header = newBuilder("")
     var visibility: DeclVisibility = None
@@ -2506,6 +2507,11 @@ proc writeModule(m: BModule, pending: bool) =
   let cfile = getCFile(m)
   if moduleHasChanged(m.g.graph, m.module):
     genInitCode(m)
+
+    while m.queue.len > 0:
+      let sym = m.queue.pop()
+      genProcAux(m, sym)
+
     finishTypeDescriptions(m)
     if sfMainModule in m.module.flags:
       # generate main file:

compiler/cgendata.nim

@@ -155,6 +155,7 @@ type
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
+    queue*: seq[PSym]         # queue of procs to generate
     alive*: IntSet            # symbol IDs of alive data as computed by `dce.nim`
     headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
@@ -178,6 +179,9 @@ template config*(m: BModule): ConfigRef = m.g.config
 template config*(p: BProc): ConfigRef = p.module.g.config
 template vccAndC*(p: BProc): bool = p.module.config.cCompiler == ccVcc and p.module.config.backend == backendC
 
+proc delayedCodegen*(m: BModule): bool {.inline.} =
+  useAliveDataFromDce in m.flags or m.config.globalOptions.contains(optCompress)
+
 proc includeHeader*(this: BModule; header: string) =
   if not this.headerFiles.contains header:
     this.headerFiles.add header

compiler/msgs.nim

@@ -133,6 +133,16 @@ proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile): FileIndex =
   var dummy: bool = false
   fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), dummy)
 
+proc registerNifSuffix*(conf: ConfigRef; suffix: string; isKnownFile: var bool): FileIndex =
+  result = conf.m.filenameToIndexTbl.getOrDefault(suffix, InvalidFileIdx)
+  if result == InvalidFileIdx:
+    isKnownFile = false
+    result = conf.m.fileInfos.len.FileIndex
+    conf.m.fileInfos.add(newFileInfo(AbsoluteFile suffix, RelativeFile suffix))
+    conf.m.filenameToIndexTbl[suffix] = result
+  else:
+    isKnownFile = true
+
 proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
   result = TLineInfo(fileIndex: fileInfoIdx)
   if line < int high(uint16):