diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 6d28f2987a..0286cfc771 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -126,6 +126,8 @@ type moduleToNifSuffix: Table[FileIndex, string] locals: HashSet[ItemId] # track proc-local symbols inProc: int + writtenTypes: seq[PType] # types written in this module, to be unloaded later + writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, @@ -238,6 +240,8 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = elif typ.itemId.module == w.currentModule and typ.state == Complete: typ.state = Sealed writeTypeDef(w, dest, typ) + # Collect for later unloading after entire module is written + w.writtenTypes.add typ else: dest.addSymUse pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo @@ -297,6 +301,8 @@ proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = elif sym.itemId.module == w.currentModule and sym.state == Complete: sym.state = Sealed writeSymDef(w, dest, sym) + # Collect for later unloading after entire module is written + w.writtenSyms.add sym else: # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! @@ -311,8 +317,12 @@ proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): writeType(w, dest, n.typField) writeSymDef(w, dest, sym) + # Collect for later unloading after entire module is written + w.writtenSyms.add sym else: writeSymDef(w, dest, sym) + # Collect for later unloading after entire module is written + w.writtenSyms.add sym else: # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! @@ -481,6 +491,13 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = writeFile(dest, d) createIndex(d, false, dest[0].info) + # Unload all written types and symbols from memory after the entire module is written + # This handles cyclic references correctly since everything is written before unloading + for typ in w.writtenTypes: + forcePartial(typ) + for sym in w.writtenSyms: + forcePartial(sym) + # --------------------------- Loader (lazy!) ----------------------------------------------- @@ -567,7 +584,7 @@ proc moduleId(c: var DecodeContext; suffix: string): FileIndex = result = c.infos.config.registerNifSuffix(suffix, isKnownFile) if not isKnownFile: let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string - let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".idx.nif")).string + let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.idx.nif")).string if result.int >= c.mods.len: c.mods.setLen(result.int + 1) c.mods[result.int] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile)) @@ -913,23 +930,16 @@ proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; inc val[] let id = ItemId(module: module.int32, item: val[]) - # Check if already loaded - if c.syms.contains(id): - result = c.syms[id][0] - else: - # Create stub symbol - result = PSym( - itemId: id, - kindImpl: skStub, - name: c.cache.getIdent(sn.name), - disamb: sn.count.int32, - state: Partial - ) - c.syms[id] = (result, entry) - - # Load the full symbol definition if it's still a stub - if result.state == Partial: - loadSym(c, result) + # Create stub symbol + result = PSym( + itemId: id, + kindImpl: skStub, + name: c.cache.getIdent(sn.name), + disamb: sn.count.int32, + state: Partial + ) + c.syms[id] = (result, entry) + loadSym(c, result) proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; interf, interfHidden: var TStrTable) = @@ -939,10 +949,13 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; # Add all public symbols to interf (exported interface) and interfHidden for nifName, entry in idx.public: - let sym = loadSymFromIndexEntry(c, module, nifName, entry) - if sym != nil: - strTableAdd(interf, sym) - strTableAdd(interfHidden, sym) + if not nifName.startsWith("`t"): + # do not load types, they are not part of an interface but an implementation detail! + echo "LOADING SYM ", nifName, " ", entry.offset + let sym = loadSymFromIndexEntry(c, module, nifName, entry) + if sym != nil: + strTableAdd(interf, sym) + strTableAdd(interfHidden, sym) when false: # Add private symbols to interfHidden only @@ -955,6 +968,10 @@ proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string +proc toNifIndexFilename*(conf: ConfigRef; f: FileIndex): string = + let suffix = moduleSuffix(conf, f) + result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.idx.nif").string + proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable): PNode = let suffix = moduleSuffix(c.infos.config, f) let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(suffix), ".nif").string @@ -968,6 +985,7 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va var buf = createTokenBuf(300) var s = nifstreams.open(modFile) + discard processDirectives(s.r) # XXX We can optimize this here and only load the top level entries! try: nifcursors.parse(s, buf, NoLineInfo) diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index bd92e4d86f..f52d336e2c 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -13,7 +13,6 @@ import std/[intsets, tables, hashes, strtabs, algorithm, os, strutils, parseutils] import ../dist/checksums/src/checksums/md5 -import ../dist/checksums/src/checksums/sha1 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb import ic / [packed_ast, ic] @@ -752,7 +751,6 @@ when not defined(nimKochBootstrap): ## Returns 'nil' if the module needs to be recompiled. ## Loads module from NIF file when optCompress is enabled. - # loadNifModule will check if the file exists internally if not fileExists(toNifFilename(g.config, fileIdx)): return nil diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 8853a66754..9071fdcd8a 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -235,7 +235,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator raiseAssert "use setPipeLinePass to set a proper PipelinePass" when not defined(nimKochBootstrap): - if optCompress in graph.config.globalOptions and not graph.config.isDefined("nimscript"): + if optCompress in graph.config.globalOptions and not graph.withinSystem and not graph.config.isDefined("nimscript"): topLevelStmts.add finalNode writeNifModule(graph.config, module.position.int32, topLevelStmts) @@ -262,7 +262,7 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF var cachedModules: seq[FileIndex] = @[] when not defined(nimKochBootstrap): # Try loading from NIF file first if optCompress is enabled - if optCompress in graph.config.globalOptions and not graph.config.isDefined("nimscript"): + if optCompress in graph.config.globalOptions and not graph.withinSystem and not graph.config.isDefined("nimscript"): result = moduleFromNifFile(graph, fileIdx, cachedModules) if result == nil: # Fall back to ROD file loading @@ -321,10 +321,12 @@ proc connectPipelineCallbacks*(graph: ModuleGraph) = proc compilePipelineSystemModule*(graph: ModuleGraph) = if graph.systemModule == nil: + graph.withinSystem = true connectPipelineCallbacks(graph) graph.config.m.systemFileIdx = fileInfoIdx(graph.config, graph.config.libpath / RelativeFile"system.nim") discard graph.compilePipelineModule(graph.config.m.systemFileIdx, {sfSystemModule}) + graph.withinSystem = false proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = connectPipelineCallbacks(graph) @@ -341,7 +343,9 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx graph.importStack.add projectFile if projectFile == systemFileIdx: + graph.withinSystem = true discard graph.compilePipelineModule(projectFile, {sfMainModule, sfSystemModule}) + graph.withinSystem = false else: graph.compilePipelineSystemModule() discard graph.compilePipelineModule(projectFile, {sfMainModule})