diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index dc7a422228..c766aa9765 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -159,7 +159,6 @@ type inProc: int #writtenTypes: seq[PType] # types written in this module, to be unloaded later #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later - exports: Table[FileIndex, HashSet[string]] # module -> specific symbol names (empty = all) writtenPackages: HashSet[string] const @@ -474,6 +473,40 @@ proc trImport(w: var Writer; n: PNode) = w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi +proc trExport(w: var Writer; n: PNode) = + # Collect export information for the index + # nkExportStmt children are nkSym nodes + # When exporting a module (export dollars), the module symbol is a child + # followed by all symbols from that module - we use empty set to mean "export all" + # When exporting specific symbols (export foo, bar), we collect their names + w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) + w.deps.addDotToken # flags + w.deps.addDotToken # type + for child in n: + if child.kind == nkSym: + let s = child.sym + if s.kindImpl == skModule: + discard "do not write module syms here" + else: + w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(s)), NoLineInfo + w.deps.addParRi + +let replayTag = registerTag("replay") +let repConverterTag = registerTag("repconverter") +let repDestroyTag = registerTag("repdestroy") +let repWasMovedTag = registerTag("repwasmoved") +let repCopyTag = registerTag("repcopy") +let repSinkTag = registerTag("repsink") +let repDupTag = registerTag("repdup") +let repTraceTag = registerTag("reptrace") +let repDeepCopyTag = registerTag("repdeepcopy") +let repEnumToStrTag = registerTag("repenumtostr") +let repMethodTag = registerTag("repmethod") +#let repClassTag = registerTag("repclass") +let includeTag = registerTag("include") +let importTag = registerTag("import") +let implTag = registerTag("implementation") + proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: dest.addDotToken @@ -581,37 +614,9 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = of nkIncludeStmt: trInclude w, n of nkExportStmt, nkExportExceptStmt: - # Collect export information for the index - # nkExportStmt children are nkSym nodes - # When exporting a module (export dollars), the module symbol is a child - # followed by all symbols from that module - we use empty set to mean "export all" - # When exporting specific symbols (export foo, bar), we collect their names # Note: nkExportExceptStmt is transformed to nkExportStmt by semExportExcept, # but we handle both just in case - var exportAllModules = initHashSet[FileIndex]() - for child in n: - if child.kind == nkSym: - let s = child.sym - if s.kindImpl == skModule: - # Export all from this module - use empty set - let modIdx = s.positionImpl.FileIndex - exportAllModules.incl modIdx - if modIdx notin w.exports: - w.exports[modIdx] = initHashSet[string]() # empty means "export all" - else: - # Export specific symbol, but only if we're not already exporting all from this module - let modIdx = s.itemId.module.FileIndex - if modIdx notin exportAllModules: - if modIdx notin w.exports: - w.exports[modIdx] = initHashSet[string]() - w.exports[modIdx].incl s.name.s - # Write the export statement as a regular node - w.withNode dest, n: - for i in 0 ..< n.len: - if n[i].kind == nkSym and n[i].sym.kindImpl == skModule: - discard "do not write module syms here" - else: - writeNode(w, dest, n[i], forAst) + trExport w, n else: w.withNode dest, n: for i in 0 ..< n.len: @@ -663,40 +668,6 @@ proc createStmtList(buf: var TokenBuf; info: PackedLineInfo) {.inline.} = buf.addDotToken # flags buf.addDotToken # type -proc buildExportBuf(w: var Writer): TokenBuf = - ## Build the export section for the NIF index from collected exports - result = createTokenBuf(32) - for modIdx, names in w.exports: - let path = toFullPath(w.infos.config, modIdx) - if names.len == 0: - # Export all from this module - result.addParLe(TagId(ExportIdx), NoLineInfo) - result.add strToken(pool.strings.getOrIncl(path), NoLineInfo) - result.addParRi() - else: - # Export specific symbols - result.addParLe(TagId(FromexportIdx), NoLineInfo) - result.add strToken(pool.strings.getOrIncl(path), NoLineInfo) - for name in names: - result.add identToken(pool.strings.getOrIncl(name), NoLineInfo) - result.addParRi() - -let replayTag = registerTag("replay") -let repConverterTag = registerTag("repconverter") -let repDestroyTag = registerTag("repdestroy") -let repWasMovedTag = registerTag("repwasmoved") -let repCopyTag = registerTag("repcopy") -let repSinkTag = registerTag("repsink") -let repDupTag = registerTag("repdup") -let repTraceTag = registerTag("reptrace") -let repDeepCopyTag = registerTag("repdeepcopy") -let repEnumToStrTag = registerTag("repenumtostr") -let repMethodTag = registerTag("repmethod") -#let repClassTag = registerTag("repclass") -let includeTag = registerTag("include") -let importTag = registerTag("import") -let implTag = registerTag("implementation") - proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = case op.kind of HookEntry: @@ -784,10 +755,6 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; writeFile(dest, d) - let exportBuf = buildExportBuf(w) - createIndex(d, dest[0].info, false, - IndexSections(exportBuf: exportBuf)) - # --------------------------- Loader (lazy!) ----------------------------------------------- proc nodeKind(n: Cursor): TNodeKind {.inline.} = @@ -845,7 +812,7 @@ type NifModule = ref object stream: nifstreams.Stream symCounter: int32 - index: NifIndex + index: Table[string, NifIndexEntry] # Simple embedded index for offsets suffix: string DecodeContext* = object @@ -871,25 +838,61 @@ type LoadFlag* = enum LoadFullAst, AlwaysLoadInterface +proc readEmbeddedIndex(s: var Stream): Table[string, NifIndexEntry] = + ## Reads the simple embedded index (index (kv sym offset)...) from indexStartsAt position. + result = initTable[string, NifIndexEntry]() + let indexPos = indexStartsAt(s.r) + if indexPos <= 0: + return + let contentPos = offset(s.r) # Save position + s.r.jumpTo(indexPos) + + var previousOffset = 0 + var t = next(s) + let exportedTagId = pool.tags.getOrIncl("x") + if t.kind == ParLe and pool.tags[t.tagId] == ".index": + t = next(s) + while t.kind != EofToken and t.kind != ParRi: + if t.kind == ParLe: + let vis = if t.tagId == exportedTagId: Exported else: Hidden + let info = t.info + t = next(s) # skip (kv + var key = "" + if t.kind == Symbol: + key = pool.syms[t.symId] + elif t.kind == Ident: + key = pool.strings[t.litId] + t = next(s) # skip symbol + if t.kind == IntLit: + let offset = int(pool.integers[t.intId]) + previousOffset + result[key] = NifIndexEntry(offset: offset, info: info, vis: vis) + previousOffset = offset + t = next(s) # skip offset + if t.kind == ParRi: + t = next(s) # skip ) + else: + t = next(s) + + s.r.jumpTo(contentPos) # Restore position + proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): FileIndex = var isKnownFile = false result = c.infos.config.registerNifSuffix(suffix, isKnownFile) if not isKnownFile or AlwaysLoadInterface in flags: let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string - let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.idx.nif")).string if not fileExists(modFile): raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile & ". This can happen when loading a module from NIF that references another module " & "whose NIF file hasn't been written yet." - c.mods[result] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile), suffix: suffix) + var stream = nifstreams.open(modFile) + let index = readEmbeddedIndex(stream) + c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix) proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = let ii = addr c.mods[module].index - result = ii.public.getOrDefault(nifName) + result = ii[].getOrDefault(nifName) if result.offset == 0: - result = ii.private.getOrDefault(nifName) - if result.offset == 0: - raiseAssert "symbol has no offset: " & nifName + raiseAssert "symbol has no offset: " & nifName proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PNode @@ -1395,83 +1398,30 @@ proc extractBasename(nifName: string): string = proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; interf, interfHidden: var TStrTable; thisModule: string) = ## Populates interface tables from the NIF index structure. - ## Uses the index's public/private tables instead of traversing AST. + ## Uses the simple embedded index for offsets, exports passed from processTopLevel. - # Move the public table and exports list out to avoid iterator invalidation + # Move the index table out to avoid iterator invalidation # (moduleId can add to c.mods which would invalidate Table iterators) - # We move them back after iteration. - var publicTab = move c.mods[module].index.public - var exportsList = move c.mods[module].index.exports + var indexTab = move c.mods[module].index - # Add all public symbols to interf (exported interface) and interfHidden - for nifName, entry in publicTab: + # Add all symbols to interf (exported interface) and interfHidden + for nifName, entry in indexTab: if not nifName.startsWith("`t"): # do not load types, they are not part of an interface but an implementation detail! #echo "LOADING SYM ", nifName, " ", entry.offset let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) if sym != nil: - strTableAdd(interf, sym) - strTableAdd(interfHidden, sym) - - # Move public table back - c.mods[module].index.public = move publicTab - - # Process exports (re-exports from other modules) - for exp in exportsList: - let (path, kind, names) = exp - # Convert path to module suffix - let expSuffix = moduleSuffix(path, cast[seq[string]](c.infos.config.searchPaths)) - # Load the exported module's index - let expModule = moduleId(c, expSuffix) - - # Move the exported module's public table out to avoid iterator invalidation - var expPublicTab = move c.mods[expModule].index.public - - # Build a set of names for filtering - var nameSet = initHashSet[string]() - for nameId in names: - nameSet.incl pool.strings[nameId] - - # Add symbols based on export kind - for nifName, entry in expPublicTab: - if nifName.startsWith("`t"): - continue # skip types - - let basename = extractBasename(nifName) - let shouldInclude = - case kind - of ExportIdx: true # export all - of FromexportIdx: basename in nameSet # only specific names - of ExportexceptIdx: basename notin nameSet # all except specific names - else: false - - if shouldInclude: - let sym = loadSymFromIndexEntry(c, expModule, nifName, entry, expSuffix) - if sym != nil: + if entry.vis == Exported: strTableAdd(interf, sym) - strTableAdd(interfHidden, sym) - - # Move exported module's public table back - c.mods[expModule].index.public = move expPublicTab - - # Move exports list back - c.mods[module].index.exports = move exportsList - - when false: - # Add private symbols to interfHidden only - for nifName, entry in idx.private: - let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) - if sym != nil: strTableAdd(interfHidden, sym) + # Move index table back + c.mods[module].index = move indexTab + proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string -proc toNifIndexFilename*(conf: ConfigRef; f: FileIndex): string = - let suffix = moduleSuffix(conf, f) - result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.idx.nif").string - proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym = result = c.syms.getOrDefault(symAsStr)[0] if result != nil: @@ -1482,14 +1432,11 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo return nil # Local symbols shouldn't be hooks let module = moduleId(c, sn.module) # Look up the symbol in the module's index - var offs = c.mods[module].index.public.getOrDefault(symAsStr) + var offs = c.mods[module].index.getOrDefault(symAsStr) if offs.offset == 0: - if alsoConsiderPrivate: - offs = c.mods[module].index.private.getOrDefault(symAsStr) - if offs.offset == 0: - return nil - else: - return nil + return nil + if not alsoConsiderPrivate and offs.vis == Hidden: + return nil # Create a stub symbol let val = addr c.mods[module].symCounter inc val[] @@ -1581,12 +1528,14 @@ proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix] else: raiseAssert "expected ParRi but got " & $tok.kind -proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] = {}; suffix: string; module: int): PrecompiledModule = +proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; + interf: var TStrTable; suffix: string; module: int): PrecompiledModule = result = PrecompiledModule(topLevel: newNode(nkStmtList)) var localSyms = initTable[string, PSym]() var t = next(s) # skip dot var cont = true + let exportTag = pool.tags.getOrIncl"export" while cont and t.kind != EofToken: if t.kind == ParLe: if t.tagId == replayTag: @@ -1627,6 +1576,24 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] = t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module) #elif t.tagId == repClassTag: # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) + elif t.tagId == exportTag: + t = next(s) # skip (export + if t.kind == DotToken: + t = next(s) # skip dot + if t.kind == DotToken: + t = next(s) # skip dot + while true: + if t.kind == Symbol: + let symAsStr = pool.syms[t.symId] + let sym = resolveSym(c, symAsStr, false) + if sym != nil: + strTableAdd(interf, sym) + t = next(s) + elif t.kind == ParRi: + break + else: + raiseAssert "expected Symbol or ParRi but got " & $t.kind + t = next(s) elif t.tagId == includeTag: t = skipTree(s) elif t.tagId == importTag: @@ -1649,25 +1616,25 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] = proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable; flags: set[LoadFlag] = {}): PrecompiledModule = - # Ensure module index is loaded - moduleId returns the FileIndex for this suffix + # Ensure module index is loaded - moduleId returns the FileIndex for this suffix let module = moduleId(c, string(suffix), flags) - # Populate interface tables from the NIF index structure - # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym - populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix)) - # Load the module AST (or just replay actions if loadFullAst is false) + # processTopLevel also collects export instructions let s = addr c.mods[module].stream - s.r.jumpTo 0 # Start from beginning - discard processDirectives(s.r) var t = next(s[]) if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): t = next(s[]) # skip (stmts t = next(s[]) # skip flags - result = processTopLevel(c, s[], flags, string(suffix), module.int) + result = processTopLevel(c, s[], flags, interf, string(suffix), module.int) else: result = PrecompiledModule(topLevel: newNode(nkStmtList)) + # Populate interface tables from the NIF index structure + # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym + # Use exports collected by processTopLevel + populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix)) + proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; flags: set[LoadFlag] = {}): PrecompiledModule = let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f)) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 6a74f4a298..2e619c8065 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1996,7 +1996,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = owner = m.module.position.int32 m.g.typeInfoMarker[sig] = (str: result, owner: owner) - rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result) + #rememberEmittedTypeInfo(m.g.graph, FileIndex(owner), $result) case t.kind of tyEmpty, tyVoid: result = cIntValue(0) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 591979aea1..c56964d09e 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -30,7 +30,6 @@ when not defined(leanCompiler): import std/strutils except `%`, addf # collides with ropes.`%` -from ic / ic import ModuleBackendFlag import std/[dynlib, math, tables, sets, os, intsets, hashes] const @@ -1926,36 +1925,6 @@ proc genMainProc(m: BModule) = if m.config.cppCustomNamespace.len > 0: openNamespaceNim(m.config.cppCustomNamespace, m.s[cfsProcs]) -proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) = - ## Called from the IC backend. - if HasDatInitProc in flags: - let datInit = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "DatInit000" - g.mainModProcs.addDeclWithVisibility(Private): - g.mainModProcs.addProcHeader(ccNimCall, datInit, CVoid, cProcParams()) - g.mainModProcs.finishProcHeaderAsProto() - g.mainDatInit.addCallStmt(datInit) - if HasModuleInitProc in flags: - let init = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "Init000" - g.mainModProcs.addDeclWithVisibility(Private): - g.mainModProcs.addProcHeader(ccNimCall, init, CVoid, cProcParams()) - g.mainModProcs.finishProcHeaderAsProto() - if sfMainModule in m.flags: - g.mainModInit.addCallStmt(init) - elif sfSystemModule in m.flags: - g.mainDatInit.addCallStmt(init) # systemInit must called right after systemDatInit if any - else: - g.otherModsInit.addCallStmt(init) - -proc whichInitProcs*(m: BModule): set[ModuleBackendFlag] = - # called from IC. - result = {} - if m.hcrOn or m.preInitProc.s(cpsInit).buf.len > 0 or m.preInitProc.s(cpsStmts).buf.len > 0: - result.incl HasModuleInitProc - for i in cfsTypeInit1..cfsDynLibInit: - if m.s[i].buf.len != 0: - result.incl HasDatInitProc - break - proc registerModuleToMain(g: BModuleList; m: BModule) = let init = m.getInitName diff --git a/compiler/commands.nim b/compiler/commands.nim index 7de69f8840..ecb00a35b2 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -494,7 +494,6 @@ proc parseCommand*(command: string): Command = of "gendepend": cmdGendepend of "dump": cmdDump of "parse": cmdParse - of "rod": cmdRod of "secret": cmdInteractive of "nop", "help": cmdNop of "jsonscript": cmdJsonscript diff --git a/compiler/ic/bitabs.nim b/compiler/ic/bitabs.nim deleted file mode 100644 index 0c9994c83f..0000000000 --- a/compiler/ic/bitabs.nim +++ /dev/null @@ -1,178 +0,0 @@ -## A BiTable is a table that can be seen as an optimized pair -## of `(Table[LitId, Val], Table[Val, LitId])`. - -import std/hashes -import rodfiles - -when defined(nimPreviewSlimSystem): - import std/assertions - -type - LitId* = distinct uint32 - - BiTable*[T] = object - vals: seq[T] # indexed by LitId - keys: seq[LitId] # indexed by hash(val) - -proc initBiTable*[T](): BiTable[T] = BiTable[T](vals: @[], keys: @[]) - -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = (h + 1) and maxHash - -template maxHash(t): untyped = high(t.keys) -template isFilled(x: LitId): bool = x.uint32 > 0'u32 - -proc `$`*(x: LitId): string {.borrow.} -proc `<`*(x, y: LitId): bool {.borrow.} -proc `<=`*(x, y: LitId): bool {.borrow.} -proc `==`*(x, y: LitId): bool {.borrow.} -proc hash*(x: LitId): Hash {.borrow.} - - -proc len*[T](t: BiTable[T]): int = t.vals.len - -proc mustRehash(length, counter: int): bool {.inline.} = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - -const - idStart = 1 - -template idToIdx(x: LitId): int = x.int - idStart - -proc hasLitId*[T](t: BiTable[T]; x: LitId): bool = - let idx = idToIdx(x) - result = idx >= 0 and idx < t.vals.len - -proc enlarge[T](t: var BiTable[T]) = - var n: seq[LitId] - newSeq(n, len(t.keys) * 2) - swap(t.keys, n) - for i in 0..high(n): - let eh = n[i] - if isFilled(eh): - var j = hash(t.vals[idToIdx eh]) and maxHash(t) - while isFilled(t.keys[j]): - j = nextTry(j, maxHash(t)) - t.keys[j] = move n[i] - -proc getKeyId*[T](t: BiTable[T]; v: T): LitId = - let origH = hash(v) - var h = origH and maxHash(t) - if t.keys.len != 0: - while true: - let litId = t.keys[h] - if not isFilled(litId): break - if t.vals[idToIdx t.keys[h]] == v: return litId - h = nextTry(h, maxHash(t)) - return LitId(0) - -proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId = - let origH = hash(v) - var h = origH and maxHash(t) - if t.keys.len != 0: - while true: - let litId = t.keys[h] - if not isFilled(litId): break - if t.vals[idToIdx t.keys[h]] == v: return litId - h = nextTry(h, maxHash(t)) - # not found, we need to insert it: - if mustRehash(t.keys.len, t.vals.len): - enlarge(t) - # recompute where to insert: - h = origH and maxHash(t) - while true: - let litId = t.keys[h] - if not isFilled(litId): break - h = nextTry(h, maxHash(t)) - else: - setLen(t.keys, 16) - h = origH and maxHash(t) - - result = LitId(t.vals.len + idStart) - t.keys[h] = result - t.vals.add v - - -proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} = - let idx = idToIdx litId - assert idx < t.vals.len - result = t.vals[idx] - -proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} = - let idx = idToIdx litId - assert idx < t.vals.len - result = t.vals[idx] - -proc hash*[T](t: BiTable[T]): Hash = - ## as the keys are hashes of the values, we simply use them instead - var h: Hash = 0 - for i, n in pairs t.keys: - h = h !& hash((i, n)) - result = !$h - -proc store*[T](f: var RodFile; t: BiTable[T]) = - storeSeq(f, t.vals) - storeSeq(f, t.keys) - -proc load*[T](f: var RodFile; t: var BiTable[T]) = - loadSeq(f, t.vals) - loadSeq(f, t.keys) - -proc sizeOnDisc*(t: BiTable[string]): int = - result = 4 - for x in t.vals: - result += x.len + 4 - result += t.keys.len * sizeof(LitId) - -when isMainModule: - - var t: BiTable[string] - - echo getOrIncl(t, "hello") - - echo getOrIncl(t, "hello") - echo getOrIncl(t, "hello3") - echo getOrIncl(t, "hello4") - echo getOrIncl(t, "helloasfasdfdsa") - echo getOrIncl(t, "hello") - echo getKeyId(t, "hello") - echo getKeyId(t, "none") - - for i in 0 ..< 100_000: - discard t.getOrIncl($i & "___" & $i) - - for i in 0 ..< 100_000: - assert t.getOrIncl($i & "___" & $i).idToIdx == i + 4 - echo "begin" - echo t.vals.len - - echo t.vals[0] - echo t.vals[1004] - - echo "middle" - - var tf: BiTable[float] - - discard tf.getOrIncl(0.4) - discard tf.getOrIncl(16.4) - discard tf.getOrIncl(32.4) - echo getKeyId(tf, 32.4) - - var f2 = open("testblah.bin", fmWrite) - echo store(f2, tf) - f2.close - - var f1 = open("testblah.bin", fmRead) - - var t2: BiTable[float] - - echo f1.load(t2) - echo t2.vals.len - - echo getKeyId(t2, 32.4) - - echo "end" - - - f1.close diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim deleted file mode 100644 index 0ea7d66e59..0000000000 --- a/compiler/ic/cbackend.nim +++ /dev/null @@ -1,179 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## New entry point into our C/C++ code generator. Ideally -## somebody would rewrite the old backend (which is 8000 lines of crufty Nim code) -## to work on packed trees directly and produce the C code as an AST which can -## then be rendered to text in a very simple manner. Unfortunately nobody wrote -## this code. So instead we wrap the existing cgen.nim and its friends so that -## we call directly into the existing code generation logic but avoiding the -## naive, outdated `passes` design. Thus you will see some -## `useAliveDataFromDce in flags` checks in the old code -- the old code is -## also doing cross-module dependency tracking and DCE that we don't need -## anymore. DCE is now done as prepass over the entire packed module graph. - -import std/[packedsets, algorithm, tables] - -when defined(nimPreviewSlimSystem): - import std/assertions - -import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen, - pathutils, extccomp, msgs, modulepaths] - -import packed_ast, ic, dce, rodfiles - -proc unpackTree(g: ModuleGraph; thisModule: int; - tree: PackedTree; n: NodePos): PNode = - var decoder = initPackedDecoder(g.config, g.cache) - result = loadNodes(decoder, g.packed, thisModule, tree, n) - -proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) = - if g.backend == nil: - g.backend = cgendata.newModuleList(g) - assert g.backend != nil - var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config, idgenFromLoadedModule(m)) - -proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = - var bmod = BModuleList(g.backend).mods[m.module.position] - assert bmod != nil - bmod.flags.incl useAliveDataFromDce - bmod.alive = move alive[m.module.position] - - for p in allNodes(m.fromDisk.topLevel): - let n = unpackTree(g, m.module.position, m.fromDisk.topLevel, p) - cgen.genTopLevelStmt(bmod, n) - - finalCodegenActions(g, bmod, newNodeI(nkStmtList, m.module.info)) - for disp in getDispatchers(g): - genProcLvl3(bmod, disp) - m.fromDisk.backendFlags = cgen.whichInitProcs(bmod) - -proc replayTypeInfo(g: ModuleGraph; m: var LoadedModule; origin: FileIndex) = - for x in mitems(m.fromDisk.emittedTypeInfo): - #echo "found type ", x, " for file ", int(origin) - g.emittedTypeInfo[x] = origin - -proc addFileToLink(config: ConfigRef; m: PSym) = - let filename = AbsoluteFile toFullPath(config, m.position.FileIndex) - let ext = - if config.backend == backendCpp: ".nim.cpp" - elif config.backend == backendObjc: ".nim.m" - else: ".nim.c" - let cfile = changeFileExt(completeCfilePath(config, - mangleModuleName(config, filename).AbsoluteFile), ext) - let objFile = completeCfilePath(config, toObjFile(config, cfile)) - if fileExists(objFile): - var cf = Cfile(nimname: m.name.s, cname: cfile, - obj: objFile, - flags: {CfileFlag.Cached}) - addFileToCompile(config, cf) - -when defined(debugDce): - import os, std/packedsets - -proc storeAliveSymsImpl(asymFile: AbsoluteFile; s: seq[int32]) = - var f = rodfiles.create(asymFile.string) - f.storeHeader() - f.storeSection aliveSymsSection - f.storeSeq(s) - close f - -template prepare {.dirty.} = - let asymFile = toRodFile(config, AbsoluteFile toFullPath(config, position.FileIndex), ".alivesyms") - var s = newSeqOfCap[int32](alive[position].len) - for a in items(alive[position]): s.add int32(a) - sort(s) - -proc storeAliveSyms(config: ConfigRef; position: int; alive: AliveSyms) = - prepare() - storeAliveSymsImpl(asymFile, s) - -proc aliveSymsChanged(config: ConfigRef; position: int; alive: AliveSyms): bool = - prepare() - var f2 = rodfiles.open(asymFile.string) - f2.loadHeader() - f2.loadSection aliveSymsSection - var oldData: seq[int32] = @[] - f2.loadSeq(oldData) - f2.close - if f2.err == ok and oldData == s: - result = false - else: - when defined(debugDce): - let oldAsSet = toPackedSet[int32](oldData) - let newAsSet = toPackedSet[int32](s) - echo "set of live symbols changed ", asymFile.changeFileExt("rod"), " ", position, " ", f2.err - echo "in old but not in new ", oldAsSet.difference(newAsSet), " number of entries in old ", oldAsSet.len - echo "in new but not in old ", newAsSet.difference(oldAsSet), " number of entries in new ", newAsSet.len - #if execShellCmd(getAppFilename() & " rod " & quoteShell(asymFile.changeFileExt("rod"))) != 0: - # echo "command failed" - result = true - storeAliveSymsImpl(asymFile, s) - -proc genPackedModule(g: ModuleGraph, i: int; alive: var AliveSyms) = - # case statement here to enforce exhaustive checks. - case g.packed[i].status - of undefined: - discard "nothing to do" - of loading, stored: - assert false - of storing, outdated: - storeAliveSyms(g.config, g.packed[i].module.position, alive) - generateCodeForModule(g, g.packed[i], alive) - closeRodFile(g, g.packed[i].module) - of loaded: - if g.packed[i].loadedButAliveSetChanged: - generateCodeForModule(g, g.packed[i], alive) - else: - addFileToLink(g.config, g.packed[i].module) - replayTypeInfo(g, g.packed[i], FileIndex(i)) - - if g.backend == nil: - g.backend = cgendata.newModuleList(g) - registerInitProcs(BModuleList(g.backend), g.packed[i].module, g.packed[i].fromDisk.backendFlags) - -proc generateCode*(g: ModuleGraph) = - ## The single entry point, generate C(++) code for the entire - ## Nim program aka `ModuleGraph`. - resetForBackend(g) - var alive = computeAliveSyms(g.packed, g.config) - - when false: - for i in 0..= 0: - genPackedModule(g, mainModuleIdx, alive) diff --git a/compiler/ic/dce.nim b/compiler/ic/dce.nim deleted file mode 100644 index 6eb36431ea..0000000000 --- a/compiler/ic/dce.nim +++ /dev/null @@ -1,169 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Dead code elimination (=DCE) for IC. - -import std/[intsets, tables] - -when defined(nimPreviewSlimSystem): - import std/assertions - -import ".." / [ast, options, lineinfos, types] - -import packed_ast, ic, bitabs - -type - AliveSyms* = seq[IntSet] - AliveContext* = object ## Purpose is to fill the 'alive' field. - stack: seq[(int, TOptions, NodePos)] ## A stack for marking symbols as alive. - decoder: PackedDecoder ## We need a PackedDecoder for module ID address translations. - thisModule: int ## The module we're currently analysing for DCE. - alive: AliveSyms ## The final result of our computation. - options: TOptions - compilerProcs: Table[string, (int, int32)] - -proc isExportedToC(c: var AliveContext; g: PackedModuleGraph; symId: int32): bool = - ## "Exported to C" procs are special (these are marked with '.exportc') because these - ## must not be optimized away! - let symPtr = unsafeAddr g[c.thisModule].fromDisk.syms[symId] - let flags = symPtr.flags - # due to a bug/limitation in the lambda lifting, unused inner procs - # are not transformed correctly; issue (#411). However, the whole purpose here - # is to eliminate unused procs. So there is no special logic required for this case. - if sfCompileTime notin flags: - if ({sfExportc, sfCompilerProc} * flags != {}) or - (symPtr.kind == skMethod): - result = true - else: - result = false - # XXX: This used to be a condition to: - # (sfExportc in prc.flags and lfExportLib in prc.loc.flags) or - if sfCompilerProc in flags: - c.compilerProcs[g[c.thisModule].fromDisk.strings[symPtr.name]] = (c.thisModule, symId) - else: - result = false - -template isNotGeneric(n: NodePos): bool = ithSon(tree, n, genericParamsPos).kind == nkEmpty - -proc followLater(c: var AliveContext; g: PackedModuleGraph; module: int; item: int32) = - ## Marks a symbol 'item' as used and later in 'followNow' the symbol's body will - ## be analysed. - if not c.alive[module].containsOrIncl(item): - var body = g[module].fromDisk.syms[item].ast - if body != emptyNodeId: - let opt = g[module].fromDisk.syms[item].options - if g[module].fromDisk.syms[item].kind in routineKinds: - body = NodeId ithSon(g[module].fromDisk.bodies, NodePos body, bodyPos) - c.stack.add((module, opt, NodePos(body))) - - when false: - let nid = g[module].fromDisk.syms[item].name - if nid != LitId(0): - let name = g[module].fromDisk.strings[nid] - if name in ["nimFrame", "callDepthLimitReached"]: - echo "I was called! ", name, " body exists: ", body != emptyNodeId, " ", module, " ", item - -proc requestCompilerProc(c: var AliveContext; g: PackedModuleGraph; name: string) = - let (module, item) = c.compilerProcs[name] - followLater(c, g, module, item) - -proc loadTypeKind(t: PackedItemId; c: AliveContext; g: PackedModuleGraph; toSkip: set[TTypeKind]): TTypeKind = - template kind(t: ItemId): TTypeKind = g[t.module].fromDisk.types[t.item].kind - - var t2 = translateId(t, g, c.thisModule, c.decoder.config) - result = t2.kind - while result in toSkip: - t2 = translateId(g[t2.module].fromDisk.types[t2.item].types[^1], g, t2.module, c.decoder.config) - result = t2.kind - -proc rangeCheckAnalysis(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) = - ## Replicates the logic of `ccgexprs.genRangeChck`. - ## XXX Refactor so that the duplicated logic is avoided. However, for now it's not clear - ## the approach has enough merit. - var dest = loadTypeKind(n.typ, c, g, abstractVar) - if optRangeCheck notin c.options or dest in {tyUInt..tyUInt64}: - discard "no need to generate a check because it was disabled" - else: - let n0t = loadTypeKind(n.firstSon.typ, c, g, {}) - if n0t in {tyUInt, tyUInt64}: - c.requestCompilerProc(g, "raiseRangeErrorNoArgs") - else: - let raiser = - case loadTypeKind(n.typ, c, g, abstractVarRange) - of tyUInt..tyUInt64, tyChar: "raiseRangeErrorU" - of tyFloat..tyFloat128: "raiseRangeErrorF" - else: "raiseRangeErrorI" - c.requestCompilerProc(g, raiser) - -proc aliveCode(c: var AliveContext; g: PackedModuleGraph; tree: PackedTree; n: NodePos) = - ## Marks the symbols we encounter when we traverse the AST at `tree[n]` as alive, unless - ## it is purely in a declarative context (type section etc.). - case n.kind - of nkNone..pred(nkSym), succ(nkSym)..nkNilLit: - discard "ignore non-sym atoms" - of nkSym: - # This symbol is alive and everything its body references. - followLater(c, g, c.thisModule, tree[n].soperand) - of nkModuleRef: - let (n1, n2) = sons2(tree, n) - assert n1.kind == nkNone - assert n2.kind == nkNone - let m = n1.litId - let item = tree[n2].soperand - let otherModule = toFileIndexCached(c.decoder, g, c.thisModule, m).int - followLater(c, g, otherModule, item) - of nkMacroDef, nkTemplateDef, nkTypeSection, nkTypeOfExpr, - nkCommentStmt, nkIncludeStmt, - nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, - nkFromStmt, nkStaticStmt: - discard - of nkVarSection, nkLetSection, nkConstSection: - # XXX ignore the defining local variable name? - for son in sonsReadonly(tree, n): - aliveCode(c, g, tree, son) - of nkChckRangeF, nkChckRange64, nkChckRange: - rangeCheckAnalysis(c, g, tree, n) - of nkProcDef, nkConverterDef, nkMethodDef, nkFuncDef, nkIteratorDef: - if n.firstSon.kind == nkSym and isNotGeneric(n): - let item = tree[n.firstSon].soperand - if isExportedToC(c, g, item): - # This symbol is alive and everything its body references. - followLater(c, g, c.thisModule, item) - else: - for son in sonsReadonly(tree, n): - aliveCode(c, g, tree, son) - -proc followNow(c: var AliveContext; g: PackedModuleGraph) = - ## Mark all entries in the stack. Marking can add more entries - ## to the stack but eventually we have looked at every alive symbol. - while c.stack.len > 0: - let (modId, opt, ast) = c.stack.pop() - c.thisModule = modId - c.options = opt - aliveCode(c, g, g[modId].fromDisk.bodies, ast) - -proc computeAliveSyms*(g: PackedModuleGraph; conf: ConfigRef): AliveSyms = - ## Entry point for our DCE algorithm. - var c = AliveContext(stack: @[], decoder: PackedDecoder(config: conf), - thisModule: -1, alive: newSeq[IntSet](g.len), - options: conf.options) - for i in countdown(len(g)-1, 0): - if g[i].status != undefined: - c.thisModule = i - for p in allNodes(g[i].fromDisk.topLevel): - aliveCode(c, g, g[i].fromDisk.topLevel, p) - - followNow(c, g) - result = move(c.alive) - -proc isAlive*(a: AliveSyms; module: int, item: int32): bool = - ## Backends use this to query if a symbol is `alive` which means - ## we need to produce (C/C++/etc) code for it. - result = a[module].contains(item) - diff --git a/compiler/ic/design.rst b/compiler/ic/design.rst deleted file mode 100644 index b096e3103a..0000000000 --- a/compiler/ic/design.rst +++ /dev/null @@ -1,56 +0,0 @@ -==================================== - Incremental Recompilations -==================================== - -We split the Nim compiler into a frontend and a backend. -The frontend produces a set of `.rod` files. Every `.nim` module -produces its own `.rod` file. - -- The IR must be a faithful representation of the AST in memory. -- The backend can do its own caching but doesn't have to. In the - current implementation the backend also caches its results. - -Advantage of the "set of files" vs the previous global database: -- By construction, we either read from the `.rod` file or from the - `.nim` file, there can be no inconsistency. There can also be no - partial updates. -- No dependency to external packages (SQLite). SQLite simply is too - slow and the old way of serialization was too slow too. We use a - format designed for Nim and expect to base further tools on this - file format. - -References to external modules must be (moduleId, symId) pairs. -The symbol IDs are module specific. This way no global ID increment -mechanism needs to be implemented that we could get wrong. ModuleIds -are rod-file specific too. - - - -Global state ------------- - -There is no global state. - -Rod File Format ---------------- - -It's a simple binary file format. `rodfiles.nim` contains some details. - - -Backend -------- - -Nim programmers have to come to enjoy whole-program dead code elimination, -by default. Since this is a "whole program" optimization, it does break -modularity. However, thanks to the packed AST representation we can perform -this global analysis without having to unpack anything. This is basically -a mark&sweep GC algorithm: - -- Start with the top level statements. Every symbol that is referenced - from a top level statement is not "dead" and needs to be compiled by - the backend. -- Every symbol referenced from a referenced symbol also has to be - compiled. - -Caching logic: Only if the set of alive symbols is different from the -last run, the module has to be regenerated. diff --git a/compiler/ic/ic.nim b/compiler/ic/ic.nim deleted file mode 100644 index 3249482f60..0000000000 --- a/compiler/ic/ic.nim +++ /dev/null @@ -1,1349 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2020 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -import std/[hashes, tables, intsets, monotimes] -import packed_ast, bitabs, rodfiles -import ".." / [ast, idents, lineinfos, msgs, ropes, options, - pathutils, condsyms, packages, modulepaths] -#import ".." / [renderer, astalgo] -from std/os import removeFile, isAbsolute - -import ../../dist/checksums/src/checksums/sha1 - -import iclineinfos - -when defined(nimPreviewSlimSystem): - import std/[syncio, assertions, formatfloat] - -type - PackedConfig* = object - backend: TBackend - selectedGC: TGCMode - cCompiler: TSystemCC - options: TOptions - globalOptions: TGlobalOptions - - ModuleBackendFlag* = enum - HasDatInitProc - HasModuleInitProc - - PackedModule* = object ## the parts of a PackedEncoder that are part of the .rod file - definedSymbols: string - moduleFlags: TSymFlags - includes*: seq[(LitId, string)] # first entry is the module filename itself - imports: seq[LitId] # the modules this module depends on - toReplay*: PackedTree # pragmas and VM specific state to replay. - topLevel*: PackedTree # top level statements - bodies*: PackedTree # other trees. Referenced from typ.n and sym.ast by their position. - #producedGenerics*: Table[GenericKey, SymId] - exports*: seq[(LitId, int32)] - hidden: seq[(LitId, int32)] - reexports: seq[(LitId, PackedItemId)] - compilerProcs*: seq[(LitId, int32)] - converters*, methods*, trmacros*, pureEnums*: seq[int32] - - typeInstCache*: seq[(PackedItemId, PackedItemId)] - procInstCache*: seq[PackedInstantiation] - attachedOps*: seq[(PackedItemId, TTypeAttachedOp, PackedItemId)] - methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)] - enumToStringProcs*: seq[(PackedItemId, PackedItemId)] - methodsPerType*: seq[(PackedItemId, PackedItemId)] - dispatchers*: seq[PackedItemId] - - emittedTypeInfo*: seq[string] - backendFlags*: set[ModuleBackendFlag] - - syms*: OrderedTable[int32, PackedSym] - types*: OrderedTable[int32, PackedType] - strings*: BiTable[string] # we could share these between modules. - numbers*: BiTable[BiggestInt] # we also store floats in here so - # that we can assure that every bit is kept - man*: LineInfoManager - - cfg: PackedConfig - - PackedEncoder* = object - #m*: PackedModule - thisModule*: int32 - lastFile*: FileIndex # remember the last lookup entry. - lastLit*: LitId - filenames*: Table[FileIndex, LitId] - pendingTypes*: seq[PType] - pendingSyms*: seq[PSym] - typeMarker*: IntSet #Table[ItemId, TypeId] # ItemId.item -> TypeId - symMarker*: IntSet #Table[ItemId, SymId] # ItemId.item -> SymId - config*: ConfigRef - -proc toString*(tree: PackedTree; pos: NodePos; m: PackedModule; nesting: int; - result: var string) = - if result.len > 0 and result[^1] notin {' ', '\n'}: - result.add ' ' - - result.add $tree[pos].kind - case tree[pos].kind - of nkEmpty, nkNilLit, nkType: discard - of nkIdent, nkStrLit..nkTripleStrLit: - result.add " " - result.add m.strings[LitId tree[pos].uoperand] - of nkSym: - result.add " " - result.add m.strings[m.syms[tree[pos].soperand].name] - of directIntLit: - result.add " " - result.addInt tree[pos].soperand - of externSIntLit: - result.add " " - result.addInt m.numbers[LitId tree[pos].uoperand] - of externUIntLit: - result.add " " - result.addInt cast[uint64](m.numbers[LitId tree[pos].uoperand]) - of nkFloatLit..nkFloat128Lit: - result.add " " - result.addFloat cast[BiggestFloat](m.numbers[LitId tree[pos].uoperand]) - else: - result.add "(\n" - for i in 1..(nesting+1)*2: result.add ' ' - for child in sonsReadonly(tree, pos): - toString(tree, child, m, nesting + 1, result) - result.add "\n" - for i in 1..nesting*2: result.add ' ' - result.add ")" - #for i in 1..nesting*2: result.add ' ' - -proc toString*(tree: PackedTree; n: NodePos; m: PackedModule): string = - result = "" - toString(tree, n, m, 0, result) - -proc debug*(tree: PackedTree; m: PackedModule) = - stdout.write toString(tree, NodePos 0, m) - -proc isActive*(e: PackedEncoder): bool = e.config != nil -proc disable(e: var PackedEncoder) = e.config = nil - -template primConfigFields(fn: untyped) {.dirty.} = - fn backend - fn selectedGC - fn cCompiler - fn options - fn globalOptions - -proc definedSymbolsAsString(config: ConfigRef): string = - result = newStringOfCap(200) - result.add "config" - for d in definedSymbolNames(config.symbols): - result.add ' ' - result.add d - -proc rememberConfig(c: var PackedEncoder; m: var PackedModule; config: ConfigRef; pc: PackedConfig) = - m.definedSymbols = definedSymbolsAsString(config) - #template rem(x) = - # c.m.cfg.x = config.x - #primConfigFields rem - m.cfg = pc - -const - debugConfigDiff = defined(debugConfigDiff) - -when debugConfigDiff: - import hashes, tables, intsets, sha1, strutils, sets - -proc configIdentical(m: PackedModule; config: ConfigRef): bool = - result = m.definedSymbols == definedSymbolsAsString(config) - when debugConfigDiff: - if not result: - var wordsA = m.definedSymbols.split(Whitespace).toHashSet() - var wordsB = definedSymbolsAsString(config).split(Whitespace).toHashSet() - for c in wordsA - wordsB: - echo "in A but not in B ", c - for c in wordsB - wordsA: - echo "in B but not in A ", c - template eq(x) = - result = result and m.cfg.x == config.x - when debugConfigDiff: - if m.cfg.x != config.x: - echo "B ", m.cfg.x, " ", config.x - primConfigFields eq - -proc rememberStartupConfig*(dest: var PackedConfig, config: ConfigRef) = - template rem(x) = - dest.x = config.x - primConfigFields rem - dest.globalOptions.excl optForceFullMake - -proc hashFileCached(conf: ConfigRef; fileIdx: FileIndex): string = - result = msgs.getHash(conf, fileIdx) - if result.len == 0: - let fullpath = msgs.toFullPath(conf, fileIdx) - result = $secureHashFile(fullpath) - msgs.setHash(conf, fileIdx, result) - -proc toLitId(x: FileIndex; c: var PackedEncoder; m: var PackedModule): LitId = - ## store a file index as a literal - if x == c.lastFile: - result = c.lastLit - else: - result = c.filenames.getOrDefault(x) - if result == LitId(0): - let p = msgs.toFullPath(c.config, x) - result = getOrIncl(m.strings, p) - c.filenames[x] = result - c.lastFile = x - c.lastLit = result - assert result != LitId(0) - -proc toFileIndex*(x: LitId; m: PackedModule; config: ConfigRef): FileIndex = - result = msgs.fileInfoIdx(config, AbsoluteFile m.strings[x]) - -proc includesIdentical(m: var PackedModule; config: ConfigRef): bool = - for it in mitems(m.includes): - if hashFileCached(config, toFileIndex(it[0], m, config)) != it[1]: - return false - result = true - -proc initEncoder*(c: var PackedEncoder; m: var PackedModule; moduleSym: PSym; config: ConfigRef; pc: PackedConfig) = - ## setup a context for serializing to packed ast - c.thisModule = moduleSym.itemId.module - c.config = config - m.moduleFlags = moduleSym.flags - m.bodies = newTreeFrom(m.topLevel) - m.toReplay = newTreeFrom(m.topLevel) - - c.lastFile = FileIndex(-10) - - let thisNimFile = FileIndex c.thisModule - var h = msgs.getHash(config, thisNimFile) - if h.len == 0: - let fullpath = msgs.toFullPath(config, thisNimFile) - if isAbsolute(fullpath): - # For NimScript compiler API support the main Nim file might be from a stream. - h = $secureHashFile(fullpath) - msgs.setHash(config, thisNimFile, h) - m.includes.add((toLitId(thisNimFile, c, m), h)) # the module itself - - rememberConfig(c, m, config, pc) - -proc addIncludeFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = - m.includes.add((toLitId(f, c, m), hashFileCached(c.config, f))) - -proc addImportFileDep*(c: var PackedEncoder; m: var PackedModule; f: FileIndex) = - m.imports.add toLitId(f, c, m) - -proc addHidden*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind != skUnknown - let nameId = getOrIncl(m.strings, s.name.s) - m.hidden.add((nameId, s.itemId.item)) - assert s.itemId.module == c.thisModule - -proc addExported*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind != skUnknown - assert s.itemId.module == c.thisModule - let nameId = getOrIncl(m.strings, s.name.s) - m.exports.add((nameId, s.itemId.item)) - -proc addConverter*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert c.thisModule == s.itemId.module - m.converters.add(s.itemId.item) - -proc addTrmacro*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - m.trmacros.add(s.itemId.item) - -proc addPureEnum*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind == skType - m.pureEnums.add(s.itemId.item) - -proc addMethod*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - m.methods.add s.itemId.item - -proc addReexport*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - assert s.kind != skUnknown - if s.kind == skModule: return - let nameId = getOrIncl(m.strings, s.name.s) - m.reexports.add((nameId, PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), - item: s.itemId.item))) - -proc addCompilerProc*(c: var PackedEncoder; m: var PackedModule; s: PSym) = - let nameId = getOrIncl(m.strings, s.name.s) - m.compilerProcs.add((nameId, s.itemId.item)) - -proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) -proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId -proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId - -proc flush(c: var PackedEncoder; m: var PackedModule) = - ## serialize any pending types or symbols from the context - while true: - if c.pendingTypes.len > 0: - discard storeType(c.pendingTypes.pop, c, m) - elif c.pendingSyms.len > 0: - discard storeSym(c.pendingSyms.pop, c, m) - else: - break - -proc toLitId(x: string; m: var PackedModule): LitId = - ## store a string as a literal - result = getOrIncl(m.strings, x) - -proc toLitId(x: BiggestInt; m: var PackedModule): LitId = - ## store an integer as a literal - result = getOrIncl(m.numbers, x) - -proc toPackedInfo(x: TLineInfo; c: var PackedEncoder; m: var PackedModule): PackedLineInfo = - pack(m.man, toLitId(x.fileIndex, c, m), x.line.int32, x.col.int32) - #PackedLineInfo(line: x.line, col: x.col, file: toLitId(x.fileIndex, c, m)) - -proc safeItemId(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId {.inline.} = - ## given a symbol, produce an ItemId with the correct properties - ## for local or remote symbols, packing the symbol as necessary - if s == nil or s.kind == skPackage: - result = nilItemId - #elif s.itemId.module == c.thisModule: - # result = PackedItemId(module: LitId(0), item: s.itemId.item) - else: - assert int(s.itemId.module) >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), - item: s.itemId.item) - -proc addMissing(c: var PackedEncoder; p: PSym) = - ## consider queuing a symbol for later addition to the packed tree - if p != nil and p.itemId.module == c.thisModule: - if p.itemId.item notin c.symMarker: - if not (sfForward in p.flags and p.kind in routineKinds): - c.pendingSyms.add p - -proc addMissing(c: var PackedEncoder; p: PType) = - ## consider queuing a type for later addition to the packed tree - if p != nil and p.uniqueId.module == c.thisModule: - if p.uniqueId.item notin c.typeMarker: - c.pendingTypes.add p - -template storeNode(dest, src, field) = - var nodeId: NodeId - if src.field != nil: - nodeId = getNodeId(m.bodies) - toPackedNode(src.field, m.bodies, c, m) - else: - nodeId = emptyNodeId - dest.field = nodeId - -proc storeTypeLater(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = - # We store multiple different trees in m.bodies. For this to work out, we - # cannot immediately store types/syms. We enqueue them instead to ensure - # we only write one tree into m.bodies after the other. - if t.isNil: return nilItemId - - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - if t.uniqueId.module == c.thisModule: - # the type belongs to this module, so serialize it here, eventually. - addMissing(c, t) - -proc storeSymLater(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = - if s.isNil: return nilItemId - assert s.itemId.module >= 0 - assert s.itemId.item >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - if s.itemId.module == c.thisModule: - # the sym belongs to this module, so serialize it here, eventually. - addMissing(c, s) - -proc storeType(t: PType; c: var PackedEncoder; m: var PackedModule): PackedItemId = - ## serialize a ptype - if t.isNil: return nilItemId - - assert t.uniqueId.module >= 0 - assert t.uniqueId.item > 0 - result = PackedItemId(module: toLitId(t.uniqueId.module.FileIndex, c, m), item: t.uniqueId.item) - - if t.uniqueId.module == c.thisModule and not c.typeMarker.containsOrIncl(t.uniqueId.item): - #if t.uniqueId.item >= m.types.len: - # setLen m.types, t.uniqueId.item+1 - - var p = PackedType(id: t.uniqueId.item, kind: t.kind, flags: t.flags, callConv: t.callConv, - size: t.size, align: t.align, nonUniqueId: t.itemId.item, - paddingAtEnd: t.paddingAtEnd) - storeNode(p, t, n) - p.typeInst = t.typeInst.storeType(c, m) - if t.kind == tyProc and t.len > 0: - # if kind == tyProc, parameter types are stored in t.n - # and you can access them with `kits` iterator. - # return type is stored in t.sons[0]. - p.types.add t[0].storeType(c, m) - else: - for kid in kids t: - p.types.add kid.storeType(c, m) - c.addMissing t.sym - p.sym = t.sym.safeItemId(c, m) - c.addMissing t.owner - p.owner = t.owner.safeItemId(c, m) - - # fill the reserved slot, nothing else: - m.types[t.uniqueId.item] = p - -proc toPackedLib(l: PLib; c: var PackedEncoder; m: var PackedModule): PackedLib = - ## the plib hangs off the psym via the .annex field - if l.isNil: return - result = PackedLib(kind: l.kind, generated: l.generated, - isOverridden: l.isOverridden, name: toLitId($l.name, m) - ) - storeNode(result, l, path) - -proc storeSym*(s: PSym; c: var PackedEncoder; m: var PackedModule): PackedItemId = - ## serialize a psym - if s.isNil: return nilItemId - - assert s.itemId.module >= 0 - result = PackedItemId(module: toLitId(s.itemId.module.FileIndex, c, m), item: s.itemId.item) - - if s.itemId.module == c.thisModule and not c.symMarker.containsOrIncl(s.itemId.item): - #if s.itemId.item >= m.syms.len: - # setLen m.syms, s.itemId.item+1 - - assert sfForward notin s.flags - - var p = PackedSym(id: s.itemId.item, kind: s.kind, flags: s.flags, info: s.info.toPackedInfo(c, m), magic: s.magic, - position: s.position, offset: s.offset, disamb: s.disamb, options: s.options, - name: s.name.s.toLitId(m)) - - storeNode(p, s, ast) - storeNode(p, s, constraint) - - if s.kind in {skLet, skVar, skField, skForVar}: - c.addMissing s.guard - p.guard = s.guard.safeItemId(c, m) - p.bitsize = s.bitsize - p.alignment = s.alignment - - p.externalName = toLitId(s.loc.snippet, m) - p.locFlags = s.loc.flags - c.addMissing s.typ - p.typ = s.typ.storeType(c, m) - c.addMissing s.owner - p.owner = s.owner.safeItemId(c, m) - p.annex = toPackedLib(s.annex, c, m) - when hasFFI: - p.cname = toLitId(s.cname, m) - p.instantiatedFrom = s.instantiatedFrom.safeItemId(c, m) - - # fill the reserved slot, nothing else: - m.syms[s.itemId.item] = p - -proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = - ## add a remote symbol reference to the tree - let info = n.info.toPackedInfo(c, m) - if n.typ != n.sym.typ: - ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total - info = info, flags = n.flags, - typeId = storeTypeLater(n.typ, c, m)) - else: - ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total - info = info, flags = n.flags) - ir.addNode(kind = nkNone, info = info, - operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32) - ir.addNode(kind = nkNone, info = info, - operand = n.sym.itemId.item) - -proc toPackedNode*(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var PackedModule) = - ## serialize a node into the tree - if n == nil: - ir.addNode(kind = nkNilRodNode, operand = 1, info = NoLineInfo) - return - let info = toPackedInfo(n.info, c, m) - case n.kind - of nkNone, nkEmpty, nkNilLit, nkType: - ir.addNode(kind = n.kind, flags = n.flags, operand = 0, - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkIdent: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.strings, n.ident.s), - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkSym: - if n.sym.itemId.module == c.thisModule: - # it is a symbol that belongs to the module we're currently - # packing: - let id = n.sym.storeSymLater(c, m).item - if n.typ != n.sym.typ: - ir.addNode(kind = nkSym, flags = n.flags, operand = id, - info = info, - typeId = storeTypeLater(n.typ, c, m)) - else: - ir.addNode(kind = nkSym, flags = n.flags, operand = id, - info = info) - else: - # store it as an external module reference: - addModuleRef(n, ir, c, m) - of externIntLit: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.numbers, n.intVal), - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkStrLit..nkTripleStrLit: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.strings, n.strVal), - typeId = storeTypeLater(n.typ, c, m), info = info) - of nkFloatLit..nkFloat128Lit: - ir.addNode(kind = n.kind, flags = n.flags, - operand = int32 getOrIncl(m.numbers, cast[BiggestInt](n.floatVal)), - typeId = storeTypeLater(n.typ, c, m), info = info) - else: - let patchPos = ir.prepare(n.kind, n.flags, - storeTypeLater(n.typ, c, m), info) - for i in 0..= g.len: - g.pm.setLen(si+1) - - if g[si].status == undefined and c.config.cmd == cmdM: - var cachedModules: seq[FileIndex] = @[] - discard needsRecompile(g, c.config, c.cache, FileIndex(si), cachedModules) - for m in cachedModules: - loadToReplayNodes(g, c.config, c.cache, m, g[int m]) - - assert g[si].status in {loaded, storing, stored} - #if not g[si].symsInit: - # g[si].symsInit = true - # setLen g[si].syms, g[si].fromDisk.syms.len - - if g[si].syms.getOrDefault(s.item) == nil: - if g[si].fromDisk.syms[s.item].kind != skModule: - result = symHeaderFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item) - # store it here early on, so that recursions work properly: - g[si].syms[s.item] = result - symBodyFromPacked(c, g, g[si].fromDisk.syms[s.item], si, s.item, result) - else: - result = g[si].module - assert result != nil - g[si].syms[s.item] = result - - else: - result = g[si].syms[s.item] - -proc typeHeaderFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; - t: PackedType; si, item: int32): PType = - result = PType(itemId: ItemId(module: si, item: t.nonUniqueId), kind: t.kind, - flagsImpl: t.flags, sizeImpl: t.size, alignImpl: t.align, - paddingAtEndImpl: t.paddingAtEnd, - uniqueId: ItemId(module: si, item: item), - callConvImpl: t.callConv) - -proc typeBodyFromPacked(c: var PackedDecoder; g: var PackedModuleGraph; - t: PackedType; si, item: int32; result: PType) = - result.sym = loadSym(c, g, si, t.sym) - setOwner(result, loadSym(c, g, si, t.owner)) - when false: - for op, item in pairs t.attachedOps: - result.attachedOps[op] = loadSym(c, g, si, item) - result.typeInst = loadType(c, g, si, t.typeInst) - var sons = newSeq[PType]() - for son in items t.types: - sons.add loadType(c, g, si, son) - result.setSons(sons) - loadAstBody(t, n) - when false: - for gen, id in items t.methods: - result.methods.add((gen, loadSym(c, g, si, id))) - -proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t: PackedItemId): PType = - if t == nilItemId: - result = nil - else: - let si = moduleIndex(c, g, thisModule, t) - assert g[si].status in {loaded, storing, stored} - assert t.item > 0 - - #if not g[si].typesInit: - # g[si].typesInit = true - # setLen g[si].types, g[si].fromDisk.types.len - - if g[si].types.getOrDefault(t.item) == nil: - result = typeHeaderFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item) - # store it here early on, so that recursions work properly: - g[si].types[t.item] = result - typeBodyFromPacked(c, g, g[si].fromDisk.types[t.item], si, t.item, result) - #assert result.itemId.item == t.item, $(result.itemId.item, t.item) - assert result.itemId.item > 0, $(result.itemId.item, t.item) - else: - result = g[si].types[t.item] - assert result.itemId.item > 0, "2" - -proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; m: var LoadedModule) = - m.iface = initTable[PIdent, seq[PackedItemId]]() - m.ifaceHidden = initTable[PIdent, seq[PackedItemId]]() - template impl(iface, e) = - let nameLit = e[0] - let e2 = - when e[1] is PackedItemId: e[1] - else: PackedItemId(module: LitId(0), item: e[1]) - iface.mgetOrPut(cache.getIdent(m.fromDisk.strings[nameLit]), @[]).add(e2) - - for e in m.fromDisk.exports: - m.iface.impl(e) - m.ifaceHidden.impl(e) - for e in m.fromDisk.reexports: - m.iface.impl(e) - m.ifaceHidden.impl(e) - for e in m.fromDisk.hidden: - m.ifaceHidden.impl(e) - - let filename = AbsoluteFile toFullPath(conf, fileIdx) - # We cannot call ``newSym`` here, because we have to circumvent the ID - # mechanism, which we do in order to assign each module a persistent ID. - m.module = PSym(kindImpl: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32), - name: getIdent(cache, splitFile(filename).name), - infoImpl: newLineInfo(fileIdx, 1, 1), - positionImpl: int(fileIdx)) - setOwner(m.module, getPackage(conf, cache, fileIdx)) - m.module.flagsImpl = m.fromDisk.moduleFlags - -proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; m: var LoadedModule) = - m.module.ast = newNode(nkStmtList) - if m.fromDisk.toReplay.len > 0: - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: conf, - cache: cache) - for p in allNodes(m.fromDisk.toReplay): - m.module.ast.add loadNodes(decoder, g, int(fileIdx), m.fromDisk.toReplay, p) - -proc needsRecompile(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; cachedModules: var seq[FileIndex]): bool = - # Does the file belong to the fileIdx need to be recompiled? - let m = int(fileIdx) - if m >= g.len: - g.pm.setLen(m+1) - - case g[m].status - of undefined: - g[m].status = loading - let fullpath = msgs.toFullPath(conf, fileIdx) - let rod = toRodFile(conf, AbsoluteFile fullpath) - let err = loadRodFile(rod, g[m].fromDisk, conf, ignoreConfig = conf.cmd == cmdM) - if err == ok: - if conf.cmd == cmdM: - setupLookupTables(g, conf, cache, fileIdx, g[m]) - cachedModules.add fileIdx - g[m].status = loaded - result = false - else: - result = optForceFullMake in conf.globalOptions - # check its dependencies: - let imp = g[m].fromDisk.imports - for dep in imp: - let fid = toFileIndex(dep, g[m].fromDisk, conf) - # Warning: we need to traverse the full graph, so - # do **not use break here**! - if needsRecompile(g, conf, cache, fid, cachedModules): - result = true - - if not result: - setupLookupTables(g, conf, cache, fileIdx, g[m]) - cachedModules.add fileIdx - g[m].status = loaded - else: - g.pm[m] = LoadedModule(status: outdated, module: g[m].module) - else: - loadError(err, rod, conf) - g[m].status = outdated - result = true - when false: loadError(err, rod, conf) - of loading, loaded: - # For loading: Assume no recompile is required. - result = false - of outdated, storing, stored: - result = true - -proc moduleFromRodFile*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - fileIdx: FileIndex; cachedModules: var seq[FileIndex]): PSym = - ## Returns 'nil' if the module needs to be recompiled. - bench g.depAnalysis: - if needsRecompile(g, conf, cache, fileIdx, cachedModules): - result = nil - else: - result = g[int fileIdx].module - assert result != nil - assert result.position == int(fileIdx) - for m in cachedModules: - loadToReplayNodes(g, conf, cache, m, g[int m]) - -template setupDecoder() {.dirty.} = - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - -proc loadProcBody*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; s: PSym): PNode = - bench g.loadBody: - let mId = s.itemId.module - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - let pos = g[mId].fromDisk.syms[s.itemId.item].ast - assert pos != emptyNodeId - result = loadProcBody(decoder, g, mId, g[mId].fromDisk.bodies, NodePos pos) - -proc loadTypeFromId*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: int; id: PackedItemId): PType = - bench g.loadType: - result = g[module].types.getOrDefault(id.item) - if result == nil: - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - result = loadType(decoder, g, module, id) - -proc loadSymFromId*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: int; id: PackedItemId): PSym = - bench g.loadSym: - result = g[module].syms.getOrDefault(id.item) - if result == nil: - var decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - result = loadSym(decoder, g, module, id) - -proc translateId*(id: PackedItemId; g: PackedModuleGraph; thisModule: int; config: ConfigRef): ItemId = - if id.module == LitId(0): - ItemId(module: thisModule.int32, item: id.item) - else: - ItemId(module: toFileIndex(id.module, g[thisModule].fromDisk, config).int32, item: id.item) - -proc simulateLoadedModule*(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache; - moduleSym: PSym; m: PackedModule) = - # For now only used for heavy debugging. In the future we could use this to reduce the - # compiler's memory consumption. - let idx = moduleSym.position - assert g[idx].status in {storing} - g[idx].status = loaded - assert g[idx].module == moduleSym - setupLookupTables(g, conf, cache, FileIndex(idx), g[idx]) - loadToReplayNodes(g, conf, cache, FileIndex(idx), g[idx]) - -# ---------------- symbol table handling ---------------- - -type - RodIter* = object - decoder: PackedDecoder - values: seq[PackedItemId] - i, module: int - -template interfSelect(a: LoadedModule, importHidden: bool): auto = - var ret = a.iface.addr - if importHidden: ret = a.ifaceHidden.addr - ret[] - -proc initRodIter*(it: var RodIter; config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; - name: PIdent, importHidden: bool): PSym = - it.decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - it.values = g[int module].interfSelect(importHidden).getOrDefault(name) - it.i = 0 - it.module = int(module) - if it.i < it.values.len: - result = loadSym(it.decoder, g, int(module), it.values[it.i]) - inc it.i - else: - result = nil - -proc initRodIterAllSyms*(it: var RodIter; config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; importHidden: bool): PSym = - it.decoder = PackedDecoder( - lastModule: int32(-1), - lastLit: LitId(0), - lastFile: FileIndex(-1), - config: config, - cache: cache) - it.values = @[] - it.module = int(module) - for v in g[int module].interfSelect(importHidden).values: - it.values.add v - it.i = 0 - if it.i < it.values.len: - result = loadSym(it.decoder, g, int(module), it.values[it.i]) - inc it.i - else: - result = nil - -proc nextRodIter*(it: var RodIter; g: var PackedModuleGraph): PSym = - if it.i < it.values.len: - result = loadSym(it.decoder, g, it.module, it.values[it.i]) - inc it.i - else: - result = nil - -iterator interfaceSymbols*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; - name: PIdent, importHidden: bool): PSym = - setupDecoder() - let values = g[int module].interfSelect(importHidden).getOrDefault(name) - for pid in values: - let s = loadSym(decoder, g, int(module), pid) - assert s != nil - yield s - -proc interfaceSymbol*(config: ConfigRef, cache: IdentCache; - g: var PackedModuleGraph; module: FileIndex; - name: PIdent, importHidden: bool): PSym = - setupDecoder() - let values = g[int module].interfSelect(importHidden).getOrDefault(name) - result = loadSym(decoder, g, int(module), values[0]) - -proc idgenFromLoadedModule*(m: LoadedModule): IdGenerator = - IdGenerator(module: m.module.itemId.module, symId: int32 m.fromDisk.syms.len, - typeId: int32 m.fromDisk.types.len) - -proc searchForCompilerproc*(m: LoadedModule; name: string): int32 = - # slow, linear search, but the results are cached: - for it in items(m.fromDisk.compilerProcs): - if m.fromDisk.strings[it[0]] == name: - return it[1] - return -1 - -# ------------------------- .rod file viewer --------------------------------- - -proc rodViewer*(rodfile: AbsoluteFile; config: ConfigRef, cache: IdentCache) = - var m: PackedModule = PackedModule() - let err = loadRodFile(rodfile, m, config, ignoreConfig=true) - if err != ok: - config.quitOrRaise "Error: could not load: " & $rodfile.string & " reason: " & $err - - when false: - echo "exports:" - for ex in m.exports: - echo " ", m.strings[ex[0]], " local ID: ", ex[1] - assert ex[0] == m.syms[ex[1]].name - # ex[1] int32 - - echo "reexports:" - for ex in m.reexports: - echo " ", m.strings[ex[0]] - # reexports*: seq[(LitId, PackedItemId)] - - echo "hidden: " & $m.hidden.len - for ex in m.hidden: - echo " ", m.strings[ex[0]], " local ID: ", ex[1] - - when false: - echo "all symbols" - for i in 0..high(m.syms): - if m.syms[i].name != LitId(0): - echo " ", m.strings[m.syms[i].name], " local ID: ", i, " kind ", m.syms[i].kind - else: - echo " local ID: ", i, " kind ", m.syms[i].kind - - echo "symbols: ", m.syms.len, " types: ", m.types.len, - " top level nodes: ", m.topLevel.len, " other nodes: ", m.bodies.len, - " strings: ", m.strings.len, " numbers: ", m.numbers.len - - echo "SIZES:" - echo "symbols: ", m.syms.len * sizeof(PackedSym), " types: ", m.types.len * sizeof(PackedType), - " top level nodes: ", m.topLevel.len * sizeof(PackedNode), - " other nodes: ", m.bodies.len * sizeof(PackedNode), - " strings: ", sizeOnDisc(m.strings) - when false: - var tt = 0 - var fc = 0 - for x in m.topLevel: - if x.kind == nkSym or x.typeId == nilItemId: inc tt - if x.flags == {}: inc fc - for x in m.bodies: - if x.kind == nkSym or x.typeId == nilItemId: inc tt - if x.flags == {}: inc fc - let total = float(m.topLevel.len + m.bodies.len) - echo "nodes with nil type: ", tt, " in % ", tt.float / total - echo "nodes with empty flags: ", fc.float / total diff --git a/compiler/ic/iclineinfos.nim b/compiler/ic/iclineinfos.nim deleted file mode 100644 index 74a7d971be..0000000000 --- a/compiler/ic/iclineinfos.nim +++ /dev/null @@ -1,84 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2024 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -# For the line information we use 32 bits. They are used as follows: -# Bit 0 (AsideBit): If we have inline line information or not. If not, the -# remaining 31 bits are used as an index into a seq[(LitId, int, int)]. -# -# We use 10 bits for the "file ID", this means a program can consist of as much -# as 1024 different files. (If it uses more files than that, the overflow bit -# would be set.) -# This means we have 21 bits left to encode the (line, col) pair. We use 7 bits for the column -# so 128 is the limit and 14 bits for the line number. -# The packed representation supports files with up to 16384 lines. -# Keep in mind that whenever any limit is reached the AsideBit is set and the real line -# information is kept in a side channel. - -import std / assertions - -const - AsideBit = 1 - FileBits = 10 - LineBits = 14 - ColBits = 7 - FileMax = (1 shl FileBits) - 1 - LineMax = (1 shl LineBits) - 1 - ColMax = (1 shl ColBits) - 1 - -static: - assert AsideBit + FileBits + LineBits + ColBits == 32 - -import .. / ic / [bitabs, rodfiles] # for LitId - -type - PackedLineInfo* = distinct uint32 - - LineInfoManager* = object - aside: seq[(LitId, int32, int32)] - -const - NoLineInfo* = PackedLineInfo(0'u32) - -proc pack*(m: var LineInfoManager; file: LitId; line, col: int32): PackedLineInfo = - if file.uint32 <= FileMax.uint32 and line <= LineMax and col <= ColMax: - let col = if col < 0'i32: 0'u32 else: col.uint32 - let line = if line < 0'i32: 0'u32 else: line.uint32 - # use inline representation: - result = PackedLineInfo((file.uint32 shl 1'u32) or (line shl uint32(AsideBit + FileBits)) or - (col shl uint32(AsideBit + FileBits + LineBits))) - else: - result = PackedLineInfo((m.aside.len shl 1) or AsideBit) - m.aside.add (file, line, col) - -proc unpack*(m: LineInfoManager; i: PackedLineInfo): (LitId, int32, int32) = - let i = i.uint32 - if (i and 1'u32) == 0'u32: - # inline representation: - result = (LitId((i shr 1'u32) and FileMax.uint32), - int32((i shr uint32(AsideBit + FileBits)) and LineMax.uint32), - int32((i shr uint32(AsideBit + FileBits + LineBits)) and ColMax.uint32)) - else: - result = m.aside[int(i shr 1'u32)] - -proc getFileId*(m: LineInfoManager; i: PackedLineInfo): LitId = - result = unpack(m, i)[0] - -proc store*(r: var RodFile; m: LineInfoManager) = storeSeq(r, m.aside) -proc load*(r: var RodFile; m: var LineInfoManager) = loadSeq(r, m.aside) - -when isMainModule: - var m = LineInfoManager(aside: @[]) - for i in 0'i32..<16388'i32: - for col in 0'i32..<100'i32: - let packed = pack(m, LitId(1023), i, col) - let u = unpack(m, packed) - assert u[0] == LitId(1023) - assert u[1] == i - assert u[2] == col - echo m.aside.len diff --git a/compiler/ic/integrity.nim b/compiler/ic/integrity.nim deleted file mode 100644 index 3e8ea25034..0000000000 --- a/compiler/ic/integrity.nim +++ /dev/null @@ -1,155 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2021 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Integrity checking for a set of .rod files. -## The set must cover a complete Nim project. - -import std/[sets, tables] - -when defined(nimPreviewSlimSystem): - import std/assertions - -import ".." / [ast, modulegraphs] -import packed_ast, bitabs, ic - -type - CheckedContext = object - g: ModuleGraph - thisModule: int32 - checkedSyms: HashSet[ItemId] - checkedTypes: HashSet[ItemId] - -proc checkType(c: var CheckedContext; typeId: PackedItemId) -proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) -proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) - -proc checkTypeObj(c: var CheckedContext; typ: PackedType) = - for child in typ.types: - checkType(c, child) - if typ.n != emptyNodeId: - checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos typ.n) - if typ.sym != nilItemId: - checkForeignSym(c, typ.sym) - if typ.owner != nilItemId: - checkForeignSym(c, typ.owner) - checkType(c, typ.typeInst) - -proc checkType(c: var CheckedContext; typeId: PackedItemId) = - if typeId == nilItemId: return - let itemId = translateId(typeId, c.g.packed, c.thisModule, c.g.config) - if not c.checkedTypes.containsOrIncl(itemId): - let oldThisModule = c.thisModule - c.thisModule = itemId.module - checkTypeObj c, c.g.packed[itemId.module].fromDisk.types[itemId.item] - c.thisModule = oldThisModule - -proc checkSym(c: var CheckedContext; s: PackedSym) = - if s.name != LitId(0): - assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId s.name - checkType c, s.typ - if s.ast != emptyNodeId: - checkNode(c, c.g.packed[c.thisModule].fromDisk.bodies, NodePos s.ast) - if s.owner != nilItemId: - checkForeignSym(c, s.owner) - -proc checkLocalSym(c: var CheckedContext; item: int32) = - let itemId = ItemId(module: c.thisModule, item: item) - if not c.checkedSyms.containsOrIncl(itemId): - checkSym c, c.g.packed[c.thisModule].fromDisk.syms[item] - -proc checkForeignSym(c: var CheckedContext; symId: PackedItemId) = - let itemId = translateId(symId, c.g.packed, c.thisModule, c.g.config) - if not c.checkedSyms.containsOrIncl(itemId): - let oldThisModule = c.thisModule - c.thisModule = itemId.module - checkSym c, c.g.packed[itemId.module].fromDisk.syms[itemId.item] - c.thisModule = oldThisModule - -proc checkNode(c: var CheckedContext; tree: PackedTree; n: NodePos) = - let t = findType(tree, n) - if t != nilItemId: - checkType(c, t) - case n.kind - of nkEmpty, nkNilLit, nkType, nkNilRodNode: - discard - of nkIdent: - assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId - of nkSym: - checkLocalSym(c, tree[n].soperand) - of directIntLit: - discard - of externIntLit, nkFloatLit..nkFloat128Lit: - assert c.g.packed[c.thisModule].fromDisk.numbers.hasLitId n.litId - of nkStrLit..nkTripleStrLit: - assert c.g.packed[c.thisModule].fromDisk.strings.hasLitId n.litId - of nkModuleRef: - let (n1, n2) = sons2(tree, n) - assert n1.kind == nkNone - assert n2.kind == nkNone - checkForeignSym(c, PackedItemId(module: n1.litId, item: tree[n2].soperand)) - else: - for n0 in sonsReadonly(tree, n): - checkNode(c, tree, n0) - -proc checkTree(c: var CheckedContext; t: PackedTree) = - for p in allNodes(t): checkNode(c, t, p) - -proc checkLocalSymIds(c: var CheckedContext; m: PackedModule; symIds: seq[int32]) = - for symId in symIds: - assert symId >= 0 and symId < m.syms.len, $symId & " " & $m.syms.len - -proc checkModule(c: var CheckedContext; m: PackedModule) = - # We check that: - # - Every symbol references existing types and symbols. - # - Every tree node references existing types and symbols. - for _, v in pairs(m.syms): - checkLocalSym c, v.id - - checkTree c, m.toReplay - checkTree c, m.topLevel - - for e in m.exports: - #assert e[1] >= 0 and e[1] < m.syms.len - assert e[0] == m.syms[e[1]].name - - for e in m.compilerProcs: - #assert e[1] >= 0 and e[1] < m.syms.len - assert e[0] == m.syms[e[1]].name - - checkLocalSymIds c, m, m.converters - checkLocalSymIds c, m, m.methods - checkLocalSymIds c, m, m.trmacros - checkLocalSymIds c, m, m.pureEnums - #[ - To do: Check all these fields: - - reexports*: seq[(LitId, PackedItemId)] - macroUsages*: seq[(PackedItemId, PackedLineInfo)] - - typeInstCache*: seq[(PackedItemId, PackedItemId)] - procInstCache*: seq[PackedInstantiation] - attachedOps*: seq[(TTypeAttachedOp, PackedItemId, PackedItemId)] - methodsPerGenericType*: seq[(PackedItemId, int, PackedItemId)] - enumToStringProcs*: seq[(PackedItemId, PackedItemId)] - methodsPerType*: seq[(PackedItemId, PackedItemId)] - dispatchers*: seq[PackedItemId] - ]# - -proc checkIntegrity*(g: ModuleGraph) = - var c = CheckedContext(g: g) - for i in 0..= currentCol and col < currentCol+tokenLen: - result = true - else: - result = false - else: - result = false - -proc searchLocalSym(c: var NavContext; s: PackedSym; info: PackedLineInfo): bool = - result = s.name != LitId(0) and - isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[c.thisModule].fromDisk.strings[s.name].len) - -proc searchForeignSym(c: var NavContext; s: ItemId; info: PackedLineInfo): bool = - let name = c.g.packed[s.module].fromDisk.syms[s.item].name - result = name != LitId(0) and - isTracked(c.g.packed[c.thisModule].fromDisk.man, info, c.trackPos, c.g.packed[s.module].fromDisk.strings[name].len) - -const - EmptyItemId = ItemId(module: -1'i32, item: -1'i32) - -proc search(c: var NavContext; tree: PackedTree): ItemId = - # We use the linear representation here directly: - for i in 0..= 0 and tree[n].kind in declarativeNodes - -proc usage(c: var NavContext; info: PackedLineInfo; isDecl: bool) = - let (fileId, line, col) = unpack(c.g.packed[c.thisModule].fromDisk.man, info) - var m = "" - var file = c.g.packed[c.thisModule].fromDisk.strings[fileId] - if c.outputSep == ' ': - file = os.extractFilename file - toLocation(m, file, line, col + ColOffset) - if not c.alreadyEmitted.containsOrIncl(m): - msgWriteln c.g.config, (if isDecl: "def" else: "usage") & c.outputSep & m - -proc list(c: var NavContext; tree: PackedTree; sym: ItemId) = - for i in 0.. nkNilLit - let distance = int32(tree.nodes.len - pos) - assert distance > 0 - tree.nodes[pos].x = toX(k, cast[uint32](distance)) - -proc len*(tree: PackedTree): int {.inline.} = tree.nodes.len - -proc `[]`*(tree: PackedTree; i: NodePos): lent PackedNode {.inline.} = - tree.nodes[i.int] - -template rawSpan(n: PackedNode): int = int(uoperand(n)) - -proc nextChild(tree: PackedTree; pos: var int) {.inline.} = - if tree.nodes[pos].kind > nkNilLit: - assert tree.nodes[pos].uoperand > 0 - inc pos, tree.nodes[pos].rawSpan - else: - inc pos - -iterator sonsReadonly*(tree: PackedTree; n: NodePos): NodePos = - var pos = n.int - assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].rawSpan - inc pos - while pos < last: - yield NodePos pos - nextChild tree, pos - -iterator sons*(dest: var PackedTree; tree: PackedTree; n: NodePos): NodePos = - let patchPos = prepare(dest, tree, n) - for x in sonsReadonly(tree, n): yield x - patch dest, patchPos - -iterator isons*(dest: var PackedTree; tree: PackedTree; - n: NodePos): (int, NodePos) = - var i = 0 - for ch0 in sons(dest, tree, n): - yield (i, ch0) - inc i - -iterator sonsFrom1*(tree: PackedTree; n: NodePos): NodePos = - var pos = n.int - assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].rawSpan - inc pos - if pos < last: - nextChild tree, pos - while pos < last: - yield NodePos pos - nextChild tree, pos - -iterator sonsWithoutLast2*(tree: PackedTree; n: NodePos): NodePos = - var count = 0 - for child in sonsReadonly(tree, n): - inc count - var pos = n.int - assert tree.nodes[pos].kind > nkNilLit - let last = pos + tree.nodes[pos].rawSpan - inc pos - while pos < last and count > 2: - yield NodePos pos - dec count - nextChild tree, pos - -proc parentImpl(tree: PackedTree; n: NodePos): NodePos = - # finding the parent of a node is rather easy: - var pos = n.int - 1 - while pos >= 0 and (isAtom(tree, pos) or (pos + tree.nodes[pos].rawSpan - 1 < n.int)): - dec pos - #assert pos >= 0, "node has no parent" - result = NodePos(pos) - -template parent*(n: NodePos): NodePos = parentImpl(tree, n) - -proc hasXsons*(tree: PackedTree; n: NodePos; x: int): bool = - var count = 0 - if tree.nodes[n.int].kind > nkNilLit: - for child in sonsReadonly(tree, n): inc count - result = count == x - -proc hasAtLeastXsons*(tree: PackedTree; n: NodePos; x: int): bool = - if tree.nodes[n.int].kind > nkNilLit: - var count = 0 - for child in sonsReadonly(tree, n): - inc count - if count >= x: return true - return false - -proc firstSon*(tree: PackedTree; n: NodePos): NodePos {.inline.} = - NodePos(n.int+1) -proc kind*(tree: PackedTree; n: NodePos): TNodeKind {.inline.} = - tree.nodes[n.int].kind -proc litId*(tree: PackedTree; n: NodePos): LitId {.inline.} = - LitId tree.nodes[n.int].uoperand -proc info*(tree: PackedTree; n: NodePos): PackedLineInfo {.inline.} = - tree.nodes[n.int].info - -proc findType*(tree: PackedTree; n: NodePos): PackedItemId = - for x in tree.withTypes: - if x[0] == int32(n): return x[1] - if x[0] > int32(n): return nilItemId - return nilItemId - -proc findFlags*(tree: PackedTree; n: NodePos): TNodeFlags = - for x in tree.withFlags: - if x[0] == int32(n): return x[1] - if x[0] > int32(n): return {} - return {} - -template typ*(n: NodePos): PackedItemId = - tree.findType(n) -template flags*(n: NodePos): TNodeFlags = - tree.findFlags(n) - -template uoperand*(n: NodePos): uint32 = - tree.nodes[n.int].uoperand - -proc span*(tree: PackedTree; pos: int): int {.inline.} = - if isAtom(tree, pos): 1 else: tree.nodes[pos].rawSpan - -proc sons2*(tree: PackedTree; n: NodePos): (NodePos, NodePos) = - assert(not isAtom(tree, n.int)) - let a = n.int+1 - let b = a + span(tree, a) - result = (NodePos a, NodePos b) - -proc sons3*(tree: PackedTree; n: NodePos): (NodePos, NodePos, NodePos) = - assert(not isAtom(tree, n.int)) - let a = n.int+1 - let b = a + span(tree, a) - let c = b + span(tree, b) - result = (NodePos a, NodePos b, NodePos c) - -proc ithSon*(tree: PackedTree; n: NodePos; i: int): NodePos = - result = default(NodePos) - if tree.nodes[n.int].kind > nkNilLit: - var count = 0 - for child in sonsReadonly(tree, n): - if count == i: return child - inc count - assert false, "node has no i-th child" - -when false: - proc `@`*(tree: PackedTree; lit: LitId): lent string {.inline.} = - tree.sh.strings[lit] - -template kind*(n: NodePos): TNodeKind = tree.nodes[n.int].kind -template info*(n: NodePos): PackedLineInfo = tree.nodes[n.int].info -template litId*(n: NodePos): LitId = LitId tree.nodes[n.int].uoperand - -template symId*(n: NodePos): SymId = SymId tree.nodes[n.int].soperand - -proc firstSon*(n: NodePos): NodePos {.inline.} = NodePos(n.int+1) - -const - externIntLit* = {nkCharLit, - nkIntLit, - nkInt8Lit, - nkInt16Lit, - nkInt32Lit, - nkInt64Lit, - nkUIntLit, - nkUInt8Lit, - nkUInt16Lit, - nkUInt32Lit, - nkUInt64Lit} - - externSIntLit* = {nkIntLit, nkInt8Lit, nkInt16Lit, nkInt32Lit, nkInt64Lit} - externUIntLit* = {nkUIntLit, nkUInt8Lit, nkUInt16Lit, nkUInt32Lit, nkUInt64Lit} - directIntLit* = nkNone - -template copyInto*(dest, n, body) = - let patchPos = prepare(dest, tree, n) - body - patch dest, patchPos - -template copyIntoKind*(dest, kind, info, body) = - let patchPos = prepare(dest, kind, info) - body - patch dest, patchPos - -proc getNodeId*(tree: PackedTree): NodeId {.inline.} = NodeId tree.nodes.len - -iterator allNodes*(tree: PackedTree): NodePos = - var p = 0 - while p < tree.len: - yield NodePos(p) - let s = span(tree, p) - inc p, s - -proc toPackedItemId*(item: int32): PackedItemId {.inline.} = - PackedItemId(module: LitId(0), item: item) - -proc load*(f: var RodFile; t: var PackedTree) = - loadSeq f, t.nodes - loadSeq f, t.withFlags - loadSeq f, t.withTypes - -proc store*(f: var RodFile; t: PackedTree) = - storeSeq f, t.nodes - storeSeq f, t.withFlags - storeSeq f, t.withTypes diff --git a/compiler/ic/replayer.nim b/compiler/ic/replayer.nim index b244ec885c..6152ffb48d 100644 --- a/compiler/ic/replayer.nim +++ b/compiler/ic/replayer.nim @@ -19,8 +19,6 @@ import std/tables when defined(nimPreviewSlimSystem): import std/assertions -import packed_ast, ic, bitabs - proc replayStateChanges*(module: PSym; g: ModuleGraph) = let list = module.ast assert list != nil @@ -88,84 +86,3 @@ proc replayStateChanges*(module: PSym; g: ModuleGraph) = g.cacheSeqs[destKey].add val else: internalAssert g.config, false - -proc replayBackendProcs*(g: ModuleGraph; module: int) = - for it in mitems(g.packed[module].fromDisk.attachedOps): - let key = translateId(it[0], g.packed, module, g.config) - let op = it[1] - let tmp = translateId(it[2], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[2]) - g.attachedOps[op][key] = LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.enumToStringProcs): - let key = translateId(it[0], g.packed, module, g.config) - let tmp = translateId(it[1], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[1]) - g.enumToStringProcs[key] = LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.methodsPerType): - let key = translateId(it[0], g.packed, module, g.config) - let tmp = translateId(it[1], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[1]) - g.methodsPerType.mgetOrPut(key, @[]).add LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.dispatchers): - let tmp = translateId(it, g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it) - g.dispatchers.add LazySym(id: symId, sym: nil) - -proc replayGenericCacheInformation*(g: ModuleGraph; module: int) = - ## We remember the generic instantiations a module performed - ## in order to to avoid the code bloat that generic code tends - ## to imply. This is cheaper than deduplication of identical - ## generic instantiations. However, deduplication is more - ## powerful and general and I hope to implement it soon too - ## (famous last words). - assert g.packed[module].status == loaded - for it in g.packed[module].fromDisk.typeInstCache: - let key = translateId(it[0], g.packed, module, g.config) - g.typeInstCache.mgetOrPut(key, @[]).add LazyType(id: FullId(module: module, packed: it[1]), typ: nil) - - for it in mitems(g.packed[module].fromDisk.procInstCache): - let key = translateId(it.key, g.packed, module, g.config) - let sym = translateId(it.sym, g.packed, module, g.config) - var concreteTypes = newSeq[FullId](it.concreteTypes.len) - for i in 0..high(it.concreteTypes): - let tmp = translateId(it.concreteTypes[i], g.packed, module, g.config) - concreteTypes[i] = FullId(module: tmp.module, packed: it.concreteTypes[i]) - - g.procInstCache.mgetOrPut(key, @[]).add LazyInstantiation( - module: module, sym: FullId(module: sym.module, packed: it.sym), - concreteTypes: concreteTypes, inst: nil) - - for it in mitems(g.packed[module].fromDisk.methodsPerGenericType): - let key = translateId(it[0], g.packed, module, g.config) - let col = it[1] - let tmp = translateId(it[2], g.packed, module, g.config) - let symId = FullId(module: tmp.module, packed: it[2]) - g.methodsPerGenericType.mgetOrPut(key, @[]).add (col, LazySym(id: symId, sym: nil)) - - replayBackendProcs(g, module) - - for it in mitems(g.packed[module].fromDisk.methods): - let sym = loadSymFromId(g.config, g.cache, g.packed, module, - PackedItemId(module: LitId(0), item: it)) - methodDef(g, g.idgen, sym) - - when false: - # not used anymore: - for it in mitems(g.packed[module].fromDisk.compilerProcs): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it[1])) - g.lazyCompilerprocs[g.packed[module].fromDisk.sh.strings[it[0]]] = symId - - for it in mitems(g.packed[module].fromDisk.converters): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it)) - g.ifaces[module].converters.add LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.trmacros): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it)) - g.ifaces[module].patterns.add LazySym(id: symId, sym: nil) - - for it in mitems(g.packed[module].fromDisk.pureEnums): - let symId = FullId(module: module, packed: PackedItemId(module: LitId(0), item: it)) - g.ifaces[module].pureEnums.add LazySym(id: symId, sym: nil) diff --git a/compiler/ic/rodfiles.nim b/compiler/ic/rodfiles.nim deleted file mode 100644 index ac995dd2eb..0000000000 --- a/compiler/ic/rodfiles.nim +++ /dev/null @@ -1,283 +0,0 @@ -# -# -# The Nim Compiler -# (c) Copyright 2020 Andreas Rumpf -# -# See the file "copying.txt", included in this -# distribution, for details about the copyright. -# - -## Low level binary format used by the compiler to store and load various AST -## and related data. -## -## NB: this is incredibly low level and if you're interested in how the -## compiler works and less a storage format, you're probably looking for -## the `ic` or `packed_ast` modules to understand the logical format. - -from std/typetraits import supportsCopyMem - -when defined(nimPreviewSlimSystem): - import std/[syncio, assertions] - -import std / tables - -## Overview -## ======== -## `RodFile` represents a Rod File (versioned binary format), and the -## associated data for common interactions such as IO and error tracking -## (`RodFileError`). The file format broken up into sections (`RodSection`) -## and preceded by a header (see: `cookie`). The precise layout, section -## ordering and data following the section are determined by the user. See -## `ic.loadRodFile`. -## -## A basic but "wrong" example of the lifecycle: -## --------------------------------------------- -## 1. `create` or `open` - create a new one or open an existing -## 2. `storeHeader` - header info -## 3. `storePrim` or `storeSeq` - save your stuff -## 4. `close` - and we're done -## -## Now read the bits below to understand what's missing. -## -## ### Issues with the Example -## Missing Sections: -## This is a low level API, so headers and sections need to be stored and -## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` & -## `loadSection`, respectively. -## -## No Error Handling: -## The API is centered around IO and prone to error, each operation checks or -## sets the `RodFile.err` field. A user of this API needs to handle these -## appropriately. -## -## API Notes -## ========= -## -## Valid inputs for Rod files -## -------------------------- -## ASTs, hopes, dreams, and anything as long as it and any children it may have -## support `copyMem`. This means anything that is not a pointer and that does not contain a pointer. At a glance these are: -## * string -## * objects & tuples (fields are recursed) -## * sequences AKA `seq[T]` -## -## Note on error handling style -## ---------------------------- -## A flag based approach is used where operations no-op in case of a -## preexisting error and set the flag if they encounter one. -## -## Misc -## ---- -## * 'Prim' is short for 'primitive', as in a non-sequence type - -type - RodSection* = enum - versionSection - configSection - stringsSection - checkSumsSection - depsSection - numbersSection - exportsSection - hiddenSection - reexportsSection - compilerProcsSection - trmacrosSection - convertersSection - methodsSection - pureEnumsSection - toReplaySection - topLevelSection - bodiesSection - symsSection - typesSection - typeInstCacheSection - procInstCacheSection - attachedOpsSection - methodsPerGenericTypeSection - enumToStringProcsSection - methodsPerTypeSection - dispatchersSection - typeInfoSection # required by the backend - backendFlagsSection - aliveSymsSection # beware, this is stored in a `.alivesyms` file. - sideChannelSection - namespaceSection - symnamesSection - - RodFileError* = enum - ok, tooBig, cannotOpen, ioFailure, wrongHeader, wrongSection, configMismatch, - includeFileChanged - - RodFile* = object - f*: File - currentSection*: RodSection # for error checking - err*: RodFileError # little experiment to see if this works - # better than exceptions. - -const - RodVersion = 2 - defaultCookie = [byte(0), byte('R'), byte('O'), byte('D'), - byte(sizeof(int)*8), byte(system.cpuEndian), byte(0), byte(RodVersion)] - -proc setError(f: var RodFile; err: RodFileError) {.inline.} = - f.err = err - #raise newException(IOError, "IO error") - -proc storePrim*(f: var RodFile; s: string) = - ## Stores a string. - ## The len is prefixed to allow for later retreival. - if f.err != ok: return - if s.len >= high(int32): - setError f, tooBig - return - var lenPrefix = int32(s.len) - if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - if s.len != 0: - if writeBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: - setError f, ioFailure - -proc storePrim*[T](f: var RodFile; x: T) = - ## Stores a non-sequence/string `T`. - ## If `T` doesn't support `copyMem` and is an object or tuple then the fields - ## are written -- the user from context will need to know which `T` to load. - if f.err != ok: return - when supportsCopyMem(T): - if writeBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): - setError f, ioFailure - elif T is tuple: - for y in fields(x): - storePrim(f, y) - elif T is object: - for y in fields(x): - when y is seq: - storeSeq(f, y) - else: - storePrim(f, y) - else: - {.error: "unsupported type for 'storePrim'".} - -proc storeSeq*[T](f: var RodFile; s: seq[T]) = - ## Stores a sequence of `T`s, with the len as a prefix for later retrieval. - if f.err != ok: return - if s.len >= high(int32): - setError f, tooBig - return - var lenPrefix = int32(s.len) - if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - for i in 0..= high(int32): - setError f, tooBig - return - var lenPrefix = int32(s.len) - if writeBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - for _, v in s: - storePrim(f, v) - -proc loadPrim*(f: var RodFile; s: var string) = - ## Read a string, the length was stored as a prefix - if f.err != ok: return - var lenPrefix = int32(0) - if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - s = newString(lenPrefix) - if lenPrefix > 0: - if readBuffer(f.f, unsafeAddr(s[0]), s.len) != s.len: - setError f, ioFailure - -proc loadPrim*[T](f: var RodFile; x: var T) = - ## Load a non-sequence/string `T`. - if f.err != ok: return - when supportsCopyMem(T): - if readBuffer(f.f, unsafeAddr(x), sizeof(x)) != sizeof(x): - setError f, ioFailure - elif T is tuple: - for y in fields(x): - loadPrim(f, y) - elif T is object: - for y in fields(x): - when y is seq: - loadSeq(f, y) - else: - loadPrim(f, y) - else: - {.error: "unsupported type for 'loadPrim'".} - -proc loadSeq*[T](f: var RodFile; s: var seq[T]) = - ## `T` must be compatible with `copyMem`, see `loadPrim` - if f.err != ok: return - var lenPrefix = int32(0) - if readBuffer(f.f, addr lenPrefix, sizeof(lenPrefix)) != sizeof(lenPrefix): - setError f, ioFailure - else: - s = newSeq[T](lenPrefix) - for i in 0.. 0: return # issue #9933 - if conf.symbolFiles == disabledSf: - cgenWriteModules(graph.backend, conf) - else: - if isDefined(conf, "nimIcIntegrityChecks"): - checkIntegrity(graph) - generateCode(graph) - # graph.backend can be nil under IC when nothing changed at all: - if graph.backend != nil: - cgenWriteModules(graph.backend, conf) + cgenWriteModules(graph.backend, conf) if conf.cmd != cmdTcc and graph.backend != nil: extccomp.callCCompiler(conf) # for now we do not support writing out a .json file with the build instructions when HCR is on @@ -241,10 +223,6 @@ proc commandScan(cache: IdentCache, config: ConfigRef) = else: rawMessage(config, errGenerated, "cannot open file: " & f.string) -proc commandView(graph: ModuleGraph) = - let f = toAbsolute(mainCommandArg(graph.config), AbsoluteDir getCurrentDir()).addFileExt(RodExt) - rodViewer(f, graph.config, graph.cache) - const PrintRopeCacheStats = false @@ -342,8 +320,6 @@ proc mainCommand*(graph: ModuleGraph) = case conf.cmd of cmdBackends: compileToBackend() - when BenchIC: - echoTimes graph.packed of cmdTcc: when hasTinyCBackend: extccomp.setCC(conf, "tcc", unknownLineInfo) @@ -461,10 +437,6 @@ proc mainCommand*(graph: ModuleGraph) = of cmdParse: wantMainModule(conf) discard parseFile(conf.projectMainIdx, cache, conf) - of cmdRod: - wantMainModule(conf) - commandView(graph) - #msgWriteln(conf, "Beware: Indentation tokens depend on the parser's state!") of cmdInteractive: commandInteractive(graph) of cmdNimscript: if conf.projectIsCmd or conf.projectIsStdin: discard diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 40415091fc..0aedfa1b26 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -14,7 +14,6 @@ import std/[intsets, tables, hashes, strtabs, os, strutils, parseutils] import ../dist/checksums/src/checksums/md5 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb -import ic / [packed_ast, ic] when not defined(nimKochBootstrap): import ast2nif @@ -28,16 +27,12 @@ when defined(nimPreviewSlimSystem): type SigHash* = distinct MD5Digest - LazySym* = object - id*: FullId - sym*: PSym - Iface* = object ## data we don't want to store directly in the ## ast.PSym type for s.kind == skModule module*: PSym ## module this "Iface" belongs to - converters*: seq[LazySym] - patterns*: seq[LazySym] - pureEnums*: seq[LazySym] + converters*: seq[PSym] + patterns*: seq[PSym] + pureEnums*: seq[PSym] interf: TStrTable interfHidden: TStrTable uniqueName*: Rope @@ -46,20 +41,6 @@ type opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym - FullId* = object - module*: int - packed*: PackedItemId - - LazyType* = object - id*: FullId - typ*: PType - - LazyInstantiation* = object - module*: int - sym*: FullId - concreteTypes*: seq[FullId] - inst*: PInstantiation - PipelinePass* = enum NonePass SemPass @@ -75,21 +56,18 @@ type ModuleGraph* {.acyclic.} = ref object ifaces*: seq[Iface] ## indexed by int32 fileIdx - packed*: PackedModuleGraph - encoders*: seq[PackedEncoder] - typeInstCache*: Table[ItemId, seq[LazyType]] # A symbol's ItemId. - procInstCache*: Table[ItemId, seq[LazyInstantiation]] # A symbol's ItemId. - attachedOps*: array[TTypeAttachedOp, Table[ItemId, LazySym]] # Type ID, destructors, etc. + typeInstCache*: Table[ItemId, seq[PType]] # A symbol's ItemId. + procInstCache*: Table[ItemId, seq[PInstantiation]] # A symbol's ItemId. + attachedOps*: array[TTypeAttachedOp, Table[ItemId, PSym]] # Type ID, destructors, etc. loadedOps: array[TTypeAttachedOp, Table[string, PSym]] # This can later by unified with `attachedOps` once it's stable opsLog*: seq[LogEntry] - methodsPerGenericType*: Table[ItemId, seq[(int, LazySym)]] # Type ID, attached methods + methodsPerGenericType*: Table[ItemId, seq[(int, PSym)]] # Type ID, attached methods memberProcsPerType*: Table[ItemId, seq[PSym]] # Type ID, attached member procs (only c++, virtual,member and ctor so far). initializersPerType*: Table[ItemId, PNode] # Type ID, AST call to the default ctor (c++ only) - enumToStringProcs*: Table[ItemId, LazySym] + enumToStringProcs*: Table[ItemId, PSym] emittedTypeInfo*: Table[string, FileIndex] - startupPackedConfig*: PackedConfig packageSyms*: TStrTable deps*: IntSet # the dependency graph or potentially its transitive closure. importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies @@ -115,8 +93,8 @@ type methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization! bucketTable*: CountTable[ItemId] objectTree*: Table[ItemId, seq[tuple[depth: int, value: PType]]] - methodsPerType*: Table[ItemId, seq[LazySym]] - dispatchers*: seq[LazySym] + methodsPerType*: Table[ItemId, seq[PSym]] + dispatchers*: seq[PSym] systemModule*: PSym sysTypes*: array[TTypeKind, PType] @@ -146,6 +124,7 @@ type procGlobals*: seq[PNode] nifReplayActions*: Table[int32, seq[PNode]] # module position -> replay actions for NIF + cachedMods: IntSet TPassContext* = object of RootObj # the pass's context idgen*: IdGenerator @@ -228,85 +207,43 @@ proc strTableAdds*(g: ModuleGraph, m: PSym, s: PSym) = strTableAdd(semtabAll(g, m), s) proc isCachedModule(g: ModuleGraph; module: int): bool {.inline.} = - result = module < g.packed.len and g.packed[module].status == loaded + result = module in g.cachedMods proc isCachedModule*(g: ModuleGraph; m: PSym): bool {.inline.} = isCachedModule(g, m.position) -proc simulateCachedModule(g: ModuleGraph; moduleSym: PSym; m: PackedModule) = - when false: - echo "simulating ", moduleSym.name.s, " ", moduleSym.position - simulateLoadedModule(g.packed, g.config, g.cache, moduleSym, m) - -proc initEncoder*(g: ModuleGraph; module: PSym) = - let id = module.position - if id >= g.encoders.len: - setLen g.encoders, id+1 - ic.initEncoder(g.encoders[id], - g.packed[id].fromDisk, module, g.config, g.startupPackedConfig) - type ModuleIter* = object - fromRod: bool modIndex: int ti: TIdentIter - rodIt: RodIter importHidden: bool proc initModuleIter*(mi: var ModuleIter; g: ModuleGraph; m: PSym; name: PIdent): PSym = assert m.kind == skModule mi.modIndex = m.position - mi.fromRod = isCachedModule(g, mi.modIndex) mi.importHidden = optImportHidden in m.options - if mi.fromRod: - result = initRodIter(mi.rodIt, g.config, g.cache, g.packed, FileIndex mi.modIndex, name, mi.importHidden) - else: - result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) + result = initIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden), name) proc nextModuleIter*(mi: var ModuleIter; g: ModuleGraph): PSym = - if mi.fromRod: - result = nextRodIter(mi.rodIt, g.packed) - else: - result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) + result = nextIdentIter(mi.ti, g.ifaces[mi.modIndex].interfSelect(mi.importHidden)) iterator allSyms*(g: ModuleGraph; m: PSym): PSym = let importHidden = optImportHidden in m.options - if isCachedModule(g, m): - var rodIt: RodIter = default(RodIter) - var r = initRodIterAllSyms(rodIt, g.config, g.cache, g.packed, FileIndex m.position, importHidden) - while r != nil: - yield r - r = nextRodIter(rodIt, g.packed) - else: - for s in g.ifaces[m.position].interfSelect(importHidden).data: - if s != nil: - yield s + for s in g.ifaces[m.position].interfSelect(importHidden).data: + if s != nil: + yield s proc someSym*(g: ModuleGraph; m: PSym; name: PIdent): PSym = let importHidden = optImportHidden in m.options - if isCachedModule(g, m): - result = interfaceSymbol(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden) - else: - result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) + result = strTableGet(g.ifaces[m.position].interfSelect(importHidden), name) proc someSymAmb*(g: ModuleGraph; m: PSym; name: PIdent; amb: var bool): PSym = let importHidden = optImportHidden in m.options - if isCachedModule(g, m): - result = nil - for s in interfaceSymbols(g.config, g.cache, g.packed, FileIndex(m.position), name, importHidden): - if result == nil: - # set result to the first symbol - result = s - else: - # another symbol found - amb = true - break - else: - var ti: TIdentIter = default(TIdentIter) - result = initIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden), name) - if result != nil and nextIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden)) != nil: - # another symbol exists with same name - amb = true + var ti: TIdentIter = default(TIdentIter) + result = initIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden), name) + if result != nil and nextIdentIter(ti, g.ifaces[m.position].interfSelect(importHidden)) != nil: + # another symbol exists with same name + amb = true proc systemModuleSym*(g: ModuleGraph; name: PIdent): PSym = result = someSym(g, g.systemModule, name) @@ -318,56 +255,24 @@ iterator systemModuleSyms*(g: ModuleGraph; name: PIdent): PSym = yield r r = nextModuleIter(mi, g) -proc resolveType(g: ModuleGraph; t: var LazyType): PType = - result = t.typ - if result == nil and isCachedModule(g, t.id.module): - result = loadTypeFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) - t.typ = result - assert result != nil - -proc resolveSym(g: ModuleGraph; t: var LazySym): PSym = - result = t.sym - if result == nil and isCachedModule(g, t.id.module): - result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) - t.sym = result - assert result != nil - -proc resolveInst(g: ModuleGraph; t: var LazyInstantiation): PInstantiation = - result = t.inst - if result == nil and isCachedModule(g, t.module): - result = PInstantiation(sym: loadSymFromId(g.config, g.cache, g.packed, t.sym.module, t.sym.packed)) - result.concreteTypes = newSeq[PType](t.concreteTypes.len) - for i in 0..high(result.concreteTypes): - result.concreteTypes[i] = loadTypeFromId(g.config, g.cache, g.packed, - t.concreteTypes[i].module, t.concreteTypes[i].packed) - t.inst = result - assert result != nil - -proc resolveAttachedOp*(g: ModuleGraph; t: var LazySym): PSym = - result = t.sym - if result == nil: - result = loadSymFromId(g.config, g.cache, g.packed, t.id.module, t.id.packed) - t.sym = result - assert result != nil - iterator typeInstCacheItems*(g: ModuleGraph; s: PSym): PType = if g.typeInstCache.contains(s.itemId): let x = addr(g.typeInstCache[s.itemId]) for t in mitems(x[]): - yield resolveType(g, t) + yield t iterator procInstCacheItems*(g: ModuleGraph; s: PSym): PInstantiation = if g.procInstCache.contains(s.itemId): let x = addr(g.procInstCache[s.itemId]) for t in mitems(x[]): - yield resolveInst(g, t) + yield t proc getAttachedOp*(g: ModuleGraph; t: PType; op: TTypeAttachedOp): PSym = ## returns the requested attached operation for type `t`. Can return nil ## if no such operation exists. if g.attachedOps[op].contains(t.itemId): - result = resolveAttachedOp(g, g.attachedOps[op][t.itemId]) + result = g.attachedOps[op][t.itemId] elif g.config.cmd in {cmdNifC, cmdM}: # Fall back to key-based lookup for NIF-loaded hooks let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) @@ -388,36 +293,32 @@ proc setAttachedOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module g.opsLog.add LogEntry(kind: HookEntry, op: op, module: ownerModule, key: key, sym: value) g.loadedOps[op][key] = value - g.attachedOps[op][t.itemId] = LazySym(sym: value) + g.attachedOps[op][t.itemId] = value proc setAttachedOp*(g: ModuleGraph; module: int; typeId: ItemId; op: TTypeAttachedOp; value: PSym) = ## Overload that takes ItemId directly, useful for registering hooks from NIF index. - g.attachedOps[op][typeId] = LazySym(sym: value) + g.attachedOps[op][typeId] = value proc setAttachedOpPartial*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = ## we also need to record this to the packed module. - g.attachedOps[op][t.itemId] = LazySym(sym: value) + g.attachedOps[op][t.itemId] = value -proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) = - if g.config.symbolFiles != disabledSf: - assert module < g.encoders.len - assert isActive(g.encoders[module]) - toPackedGeneratedProcDef(value, g.encoders[module], g.packed[module].fromDisk) - #storeAttachedProcDef(t, op, value, g.encoders[module], g.packed[module].fromDisk) +proc completePartialOp*(g: ModuleGraph; module: int; t: PType; op: TTypeAttachedOp; value: PSym) {.inline.} = + discard iterator getDispatchers*(g: ModuleGraph): PSym = for i in g.dispatchers.mitems: - yield resolveSym(g, i) + yield i proc addDispatchers*(g: ModuleGraph, value: PSym) = # TODO: add it for packed modules - g.dispatchers.add LazySym(sym: value) + g.dispatchers.add value -iterator resolveLazySymSeq(g: ModuleGraph, list: var seq[LazySym]): PSym = +iterator resolveLazySymSeq(g: ModuleGraph, list: var seq[PSym]): PSym = for it in list.mitems: - yield resolveSym(g, it) + yield it -proc setMethodsPerType*(g: ModuleGraph; id: ItemId, methods: seq[LazySym]) = +proc setMethodsPerType*(g: ModuleGraph; id: ItemId, methods: seq[PSym]) = # TODO: add it for packed modules g.methodsPerType[id] = methods @@ -428,14 +329,14 @@ proc addNifReplayAction*(g: ModuleGraph; module: int32; n: PNode) = iterator getMethodsPerType*(g: ModuleGraph; t: PType): PSym = if g.methodsPerType.contains(t.itemId): for it in mitems g.methodsPerType[t.itemId]: - yield resolveSym(g, it) + yield it proc getToStringProc*(g: ModuleGraph; t: PType): PSym = - result = resolveSym(g, g.enumToStringProcs[t.itemId]) + result = g.enumToStringProcs[t.itemId] assert result != nil proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) = - g.enumToStringProcs[t.itemId] = LazySym(sym: value) + g.enumToStringProcs[t.itemId] = value let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: value.itemId.module.int g.opsLog.add LogEntry(kind: EnumToStrEntry, module: ownerModule, key: key, sym: value) @@ -443,10 +344,10 @@ proc setToStringProc*(g: ModuleGraph; t: PType; value: PSym) = iterator methodsForGeneric*(g: ModuleGraph; t: PType): (int, PSym) = if g.methodsPerGenericType.contains(t.itemId): for it in mitems g.methodsPerGenericType[t.itemId]: - yield (it[0], resolveSym(g, it[1])) + yield (it[0], it[1]) proc addMethodToGeneric*(g: ModuleGraph; module: int; t: PType; col: int; m: PSym) = - g.methodsPerGenericType.mgetOrPut(t.itemId, @[]).add (col, LazySym(sym: m)) + g.methodsPerGenericType.mgetOrPut(t.itemId, @[]).add (col, m) let key = typeKey(t, g.config, loadTypeCallback, loadSymCallback) let ownerModule = if t.sym != nil: t.sym.itemId.module.int else: module g.opsLog.add LogEntry(kind: MethodEntry, module: ownerModule, key: key, sym: m) @@ -496,20 +397,6 @@ proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = return result return nil - # slow, linear search, but the results are cached: - for module in 0..= 0: - result = loadSymFromId(g.config, g.cache, g.packed, module, toPackedItemId(x)) - if result != nil: - strTableAdd(g.compilerprocs, result) - return result - -proc loadPackedSym*(g: ModuleGraph; s: var LazySym) = - if s.sym == nil: - s.sym = loadSymFromId(g.config, g.cache, g.packed, s.id.module, s.id.packed) - proc `$`*(u: SigHash): string = toBase64a(cast[cstring](unsafeAddr u), sizeof(u)) @@ -596,16 +483,13 @@ proc registerModule*(g: ModuleGraph; m: PSym) = if m.position >= g.ifaces.len: setLen(g.ifaces, m.position + 1) - if m.position >= g.packed.len: - setLen(g.packed.pm, m.position + 1) - if g.ifaces[m.position].module == nil: g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], uniqueName: rope(uniqueModuleName(g.config, m))) initStrTables(g, m) proc registerModuleById*(g: ModuleGraph; m: FileIndex) = - registerModule(g, g.packed[int m].module) + registerModule(g, g.ifaces[int m].module) proc initOperators*(g: ModuleGraph): Operators = # These are safe for IC. @@ -674,49 +558,13 @@ proc resetAllModules*(g: ModuleGraph) = initModuleGraphFields(g) proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym = - result = nil - if fileIdx.int32 >= 0: - if isCachedModule(g, fileIdx.int32): - result = g.packed[fileIdx.int32].module - elif fileIdx.int32 < g.ifaces.len: - result = g.ifaces[fileIdx.int32].module + if fileIdx.int32 >= 0 and fileIdx.int32 < g.ifaces.len: + result = g.ifaces[fileIdx.int32].module + else: + result = nil proc moduleOpenForCodegen*(g: ModuleGraph; m: FileIndex): bool {.inline.} = - if g.config.symbolFiles == disabledSf: - result = true - else: - result = g.packed[m.int32].status notin {undefined, stored, loaded} - -proc rememberEmittedTypeInfo*(g: ModuleGraph; m: FileIndex; ti: string) = - #assert(not isCachedModule(g, m.int32)) - if g.config.symbolFiles != disabledSf: - #assert g.encoders[m.int32].isActive - assert g.packed[m.int32].status != stored - g.packed[m.int32].fromDisk.emittedTypeInfo.add ti - #echo "added typeinfo ", m.int32, " ", ti, " suspicious ", not g.encoders[m.int32].isActive - -proc rememberFlag*(g: ModuleGraph; m: PSym; flag: ModuleBackendFlag) = - if g.config.symbolFiles != disabledSf: - #assert g.encoders[m.int32].isActive - assert g.packed[m.position].status != stored - g.packed[m.position].fromDisk.backendFlags.incl flag - -proc closeRodFile*(g: ModuleGraph; m: PSym) = - if g.config.symbolFiles in {readOnlySf, v2Sf}: - # For stress testing we seek to reload the symbols from memory. This - # way much of the logic is tested but the test is reproducible as it does - # not depend on the hard disk contents! - let mint = m.position - saveRodFile(toRodFile(g.config, AbsoluteFile toFullPath(g.config, FileIndex(mint))), - g.encoders[mint], g.packed[mint].fromDisk) - g.packed[mint].status = stored - - elif g.config.symbolFiles == stressTest: - # debug code, but maybe a good idea for production? Could reduce the compiler's - # memory consumption considerably at the cost of more loads from disk. - let mint = m.position - simulateCachedModule(g, m, g.packed[mint].fromDisk) - g.packed[mint].status = loaded + result = true proc dependsOn(a, b: int): int {.inline.} = (a shl 15) + b @@ -800,19 +648,8 @@ proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool = proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} = result = s.ast[bodyPos] - if result == nil and g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}: - result = loadProcBody(g.config, g.cache, g.packed, s) - s.ast[bodyPos] = result assert result != nil -proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; - cachedModules: var seq[FileIndex]): PSym = - ## Returns 'nil' if the module needs to be recompiled. - if g.config.symbolFiles in {readOnlySf, v2Sf, stressTest}: - result = moduleFromRodFile(g.packed, g.config, g.cache, fileIdx, cachedModules) - else: - result = nil - when not defined(nimKochBootstrap): proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex; flags: set[LoadFlag] = {}): PrecompiledModule = @@ -846,7 +683,7 @@ when not defined(nimKochBootstrap): of HookEntry: g.loadedOps[x.op][x.key] = x.sym of ConverterEntry: - g.ifaces[fileIdx.int].converters.add LazySym(sym: x.sym) + g.ifaces[fileIdx.int].converters.add x.sym of MethodEntry: discard "todo" of EnumToStrEntry: @@ -857,9 +694,10 @@ when not defined(nimKochBootstrap): discard "todo" proc configComplete*(g: ModuleGraph) = - rememberStartupConfig(g.startupPackedConfig, g.config) + #rememberStartupConfig(g.startupPackedConfig, g.config) + discard -proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym, ) = +proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string, fromModule: PSym) = let conf = graph.config let isNimscript = conf.isDefined("nimscript") if (not isNimscript) or hintProcessing in conf.cmdlineNotes: diff --git a/compiler/options.nim b/compiler/options.nim index 3fdc8a99cc..877898219a 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -156,7 +156,6 @@ type cmdCheck # semantic checking for whole project cmdM # only compile a single cmdParse # parse a single file (for debugging) - cmdRod # .rod to some text representation (for debugging) cmdIdeTools # ide tools (e.g. nimsuggest) cmdNimscript # evaluate nimscript cmdDoc0 diff --git a/compiler/passes.nim b/compiler/passes.nim index 5047ed1085..af682f8138 100644 --- a/compiler/passes.nim +++ b/compiler/passes.nim @@ -148,11 +148,6 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator; closeParser(p) if s.kind != llsStdIn: break closePasses(graph, a) - if graph.config.backend notin {backendC, backendCpp, backendObjc}: - # We only write rod files here if no C-like backend is active. - # The C-like backends have been patched to support the IC mechanism. - # They are responsible for closing the rod files. See `cbackend.nim`. - closeRodFile(graph, module) result = true proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fromModule: PSym = nil): PSym = @@ -168,22 +163,10 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags, fr elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) discard processModule(graph, result, idGeneratorFromModule(result), s) if result == nil: - var cachedModules: seq[FileIndex] = @[] - result = moduleFromRodFile(graph, fileIdx, cachedModules) - let filename = AbsoluteFile toFullPath(graph.config, fileIdx) - if result == nil: - result = newModule(graph, fileIdx) - result.incl flags - registerModule(graph, result) - processModuleAux("import") - else: - if sfSystemModule in flags: - graph.systemModule = result - partialInitModule(result, graph, fileIdx, filename) - for m in cachedModules: - registerModuleById(graph, m) - replayStateChanges(graph.packed.pm[m.int].module, graph) - replayGenericCacheInformation(graph, m.int) + result = newModule(graph, fileIdx) + result.incl flags + registerModule(graph, result) + processModuleAux("import") elif graph.isDirty(result): result.excl sfDirty # reset module fields: diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 6c67f1268c..843ed49ad0 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -261,12 +261,6 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator writeNifModule(graph.config, module.position.int32, topLevelStmts, graph.opsLog, replayActions) - if graph.config.backend notin {backendC, backendCpp, backendObjc} and graph.config.cmd != cmdM: - # We only write rod files here if no C-like backend is active. - # The C-like backends have been patched to support the IC mechanism. - # They are responsible for closing the rod files. See `cbackend.nim`. - # cmdM uses NIF files only, not ROD files. - closeRodFile(graph, module) result = true proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags; fromModule: PSym = nil): PSym = @@ -282,7 +276,6 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF elif graph.config.projectIsCmd: s = llStreamOpen(graph.config.cmdInput) discard processPipelineModule(graph, result, idGeneratorFromModule(result), s) if result == nil: - var cachedModules: seq[FileIndex] = @[] when not defined(nimKochBootstrap): # For cmdM: load imports from NIF files (but compile the main module from source) # Skip when withinSystem is true (compiling system.nim itself) @@ -307,9 +300,6 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF if result.ast != nil: replayStateChanges(result, graph) return result # Return early, don't process from source - if result == nil and graph.config.cmd != cmdM: - # Fall back to ROD file loading (not used for cmdM which uses NIF only) - result = moduleFromRodFile(graph, fileIdx, cachedModules) let path = toFullPath(graph.config, fileIdx) let filename = AbsoluteFile path # it could be a stdinfile/cmdfile @@ -328,16 +318,6 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF registerModule(graph, result) processModuleAux("import") partialInitModule(result, graph, fileIdx, filename) - for m in cachedModules: - registerModuleById(graph, m) - if graph.config.cmd == cmdM: - # cmdM uses NIF files - replay from module AST loaded by loadNifModule - let module = graph.getModule(m) - if module != nil and module.ast != nil: - replayStateChanges(module, graph) - else: - replayStateChanges(graph.packed.pm[m.int].module, graph) - replayGenericCacheInformation(graph, m.int) elif graph.isDirty(result): result.excl sfDirty # reset module fields: @@ -397,7 +377,6 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx connectPipelineCallbacks(graph) graph.config.m.systemFileIdx = fileInfoIdx(graph.config, graph.config.libpath / RelativeFile"system.nim") - var cachedModules: seq[FileIndex] = @[] when not defined(nimKochBootstrap): let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx) graph.systemModule = precomp.module diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim index 5f99cae7f8..53d928140b 100644 --- a/compiler/pragmas.nim +++ b/compiler/pragmas.nim @@ -21,8 +21,6 @@ import std/[os, math, strutils] when defined(nimPreviewSlimSystem): import std/assertions -from ic / ic import addCompilerProc - const FirstCallConv* = wNimcall LastCallConv* = wNoconv @@ -767,8 +765,6 @@ proc markCompilerProc(c: PContext; s: PSym) = incl(s, sfCompilerProc) incl(s.flagsImpl, sfUsed) registerCompilerProc(c.graph, s) - if c.config.symbolFiles != disabledSf: - addCompilerProc(c.encoder, c.packedRepr, s) proc deprecatedStmt(c: PContext; outerPragma: PNode) = let pragma = outerPragma[1] diff --git a/compiler/sem.nim b/compiler/sem.nim index 1d2ed2e350..0e9653f231 100644 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -889,8 +889,6 @@ proc semWithPContext*(c: PContext, n: PNode): PNode = else: result = newNodeI(nkEmpty, n.info) #if c.config.cmd == cmdIdeTools: findSuggest(c, n) - storeRodNode(c, result) - proc reportUnusedModules(c: PContext) = if c.config.cmd == cmdM: return diff --git a/compiler/semdata.nim b/compiler/semdata.nim index b1dd28ec4c..a3aae559fd 100644 --- a/compiler/semdata.nim +++ b/compiler/semdata.nim @@ -19,8 +19,6 @@ import magicsys, vmdef, modulegraphs, lineinfos, pathutils, layeredtable, types, lowerings, trees, parampatterns, astalgo -import ic / ic - type TOptionEntry* = object # entries to put on a stack for pragma parsing options*: TOptions @@ -336,28 +334,14 @@ proc newContext*(graph: ModuleGraph; module: PSym): PContext = signatures: initStrTable(), features: graph.config.features ) - if graph.config.symbolFiles != disabledSf: - let id = module.position - if graph.config.cmd != cmdM: - assert graph.packed[id].status in {undefined, outdated} - graph.packed[id].status = storing - graph.packed[id].module = module - initEncoder graph, module - -template packedRepr*(c): untyped = c.graph.packed[c.module.position].fromDisk -template encoder*(c): untyped = c.graph.encoders[c.module.position] proc addIncludeFileDep*(c: PContext; f: FileIndex) = - if c.config.symbolFiles != disabledSf: - addIncludeFileDep(c.encoder, c.packedRepr, f) + discard proc addImportFileDep*(c: PContext; f: FileIndex) = - if c.config.symbolFiles != disabledSf: - addImportFileDep(c.encoder, c.packedRepr, f) + discard proc addPragmaComputation*(c: PContext; n: PNode) = - if c.config.symbolFiles != disabledSf: - addPragmaComputation(c.encoder, c.packedRepr, n) # Also store for NIF-based IC (cmdM mode or optCompress) if optCompress in c.config.globalOptions or c.config.cmd == cmdM: addNifReplayAction(c.graph, c.module.position.int32, n) @@ -368,38 +352,28 @@ proc inclSym(sq: var seq[PSym], s: PSym): bool = sq.add s result = true -proc addConverter*(c: PContext, conv: LazySym) = - assert conv.sym != nil - if inclSym(c.converters, conv.sym): +proc addConverter*(c: PContext, conv: PSym) = + assert conv != nil + if inclSym(c.converters, conv): add(c.graph.ifaces[c.module.position].converters, conv) -proc addConverterDef*(c: PContext, conv: LazySym) = +proc addConverterDef*(c: PContext, conv: PSym) = addConverter(c, conv) - if c.config.symbolFiles != disabledSf: - addConverter(c.encoder, c.packedRepr, conv.sym) -proc addPureEnum*(c: PContext, e: LazySym) = - assert e.sym != nil +proc addPureEnum*(c: PContext, e: PSym) = + assert e != nil add(c.graph.ifaces[c.module.position].pureEnums, e) - if c.config.symbolFiles != disabledSf: - addPureEnum(c.encoder, c.packedRepr, e.sym) -proc addPattern*(c: PContext, p: LazySym) = - assert p.sym != nil - if inclSym(c.patterns, p.sym): +proc addPattern*(c: PContext, p: PSym) = + assert p != nil + if inclSym(c.patterns, p): add(c.graph.ifaces[c.module.position].patterns, p) - if c.config.symbolFiles != disabledSf: - addTrmacro(c.encoder, c.packedRepr, p.sym) proc exportSym*(c: PContext; s: PSym) = strTableAdds(c.graph, c.module, s) - if c.config.symbolFiles != disabledSf: - addExported(c.encoder, c.packedRepr, s) proc reexportSym*(c: PContext; s: PSym) = strTableAdds(c.graph, c.module, s) - if c.config.symbolFiles != disabledSf: - addReexport(c.encoder, c.packedRepr, s) proc newLib*(kind: TLibKind): PLib = result = PLib(kind: kind) #result.syms = initObjectSet() @@ -614,19 +588,11 @@ template addExport*(c: PContext; s: PSym) = ## convenience to export a symbol from the current module addExport(c.graph, c.module, s) -proc storeRodNode*(c: PContext, n: PNode) = - if c.config.symbolFiles != disabledSf: - toPackedNodeTopLevel(n, c.encoder, c.packedRepr) - proc addToGenericProcCache*(c: PContext; s: PSym; inst: PInstantiation) = - c.graph.procInstCache.mgetOrPut(s.itemId, @[]).add LazyInstantiation(module: c.module.position, inst: inst) - if c.config.symbolFiles != disabledSf: - storeInstantiation(c.encoder, c.packedRepr, s, inst) + c.graph.procInstCache.mgetOrPut(s.itemId, @[]).add inst proc addToGenericCache*(c: PContext; s: PSym; inst: PType) = - c.graph.typeInstCache.mgetOrPut(s.itemId, @[]).add LazyType(typ: inst) - if c.config.symbolFiles != disabledSf: - storeTypeInst(c.encoder, c.packedRepr, s, inst) + c.graph.typeInstCache.mgetOrPut(s.itemId, @[]).add inst proc sealRodFile*(c: PContext) = if c.config.symbolFiles != disabledSf: @@ -642,9 +608,8 @@ proc rememberExpansion*(c: PContext; info: TLineInfo; expandedSym: PSym) = ## in the sem'checked AST. This is very bad for IDE-like tooling ## ("find all usages of this template" would not work). We need special ## logic to remember macro/template expansions. This is done here and - ## delegated to the "rod" file mechanism. - if c.config.symbolFiles != disabledSf: - storeExpansion(c.encoder, c.packedRepr, info, expandedSym) + ## delegated to the "NIF" file mechanism. + discard "XXX To implement" const errVarForOutParamNeededX = "for a 'var' type a variable needs to be passed; but '$1' is immutable" diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 8318da5bbb..9605683eae 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -3013,9 +3013,9 @@ proc semExportExcept(c: PContext, n: PNode): PNode = proc semExport(c: PContext, n: PNode): PNode = proc specialSyms(c: PContext; s: PSym) {.inline.} = - if s.kind == skConverter: addConverter(c, LazySym(sym: s)) + if s.kind == skConverter: addConverter(c, s) elif s.kind == skType and s.typ != nil and s.typ.kind == tyEnum and sfPure in s.flags: - addPureEnum(c, LazySym(sym: s)) + addPureEnum(c, s) result = newNodeI(nkExportStmt, n.info) for i in 0..= y.depth: 1 @@ -104,7 +104,7 @@ proc collectVTableDispatchers*(g: ModuleGraph) = for item in g.objectTree[baseType.itemId]: if item.value.itemId notin itemTable: - itemTable[item.value.itemId] = newSeq[LazySym](methodIndexLen) + itemTable[item.value.itemId] = newSeq[PSym](methodIndexLen) var mIndex = 0 # here is the correpsonding index if baseType.itemId notin rootItemIdCount: @@ -114,13 +114,13 @@ proc collectVTableDispatchers*(g: ModuleGraph) = rootItemIdCount.inc(baseType.itemId) for idx in 0..= y.depth: 1 @@ -142,7 +142,7 @@ proc sortVTableDispatchers*(g: ModuleGraph) = for item in g.objectTree[baseType.itemId]: if item.value.itemId notin itemTable: - itemTable[item.value.itemId] = newSeq[LazySym](methodIndexLen) + itemTable[item.value.itemId] = newSeq[PSym](methodIndexLen) var mIndex = 0 # here is the correpsonding index if baseType.itemId notin rootItemIdCount: @@ -152,7 +152,7 @@ proc sortVTableDispatchers*(g: ModuleGraph) = rootItemIdCount.inc(baseType.itemId) for idx in 0..