From a773178e2b846817a0da514e2b9883ecc5d06da0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 1 Dec 2025 22:59:12 +0100 Subject: [PATCH] IC: progress (#25314) --- compiler/ast.nim | 6 +- compiler/ast2nif.nim | 164 +++++++++++++++++++++++++++---------- compiler/astalgo.nim | 101 ----------------------- compiler/astdef.nim | 103 +++++++++++++++++++++++ compiler/ccgexprs.nim | 10 +++ compiler/ccgtypes.nim | 2 +- compiler/cgen.nim | 10 +-- compiler/modulegraphs.nim | 27 ++++++ compiler/pipelines.nim | 14 +++- lib/system.nim | 63 +++++++------- lib/system/arithmetics.nim | 4 + lib/system/dyncalls.nim | 2 +- lib/system/excpt.nim | 4 - 13 files changed, 320 insertions(+), 190 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 7ae2f67765..eebe6f257e 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -33,7 +33,7 @@ template typ*(n: PNode): PType = n.typField when not defined(nimKochBootstrap): - var program {.threadvar.}: DecodeContext + var program* {.threadvar.}: DecodeContext proc setupProgram*(config: ConfigRef; cache: IdentCache) = when not defined(nimKochBootstrap): @@ -736,10 +736,6 @@ proc appendToModule*(m: PSym, n: PNode) = assert m.astImpl.kind == nkStmtList m.astImpl.add(n) -const # for all kind of hash tables: - GrowthFactor* = 2 # must be power of 2, > 0 - StartSize* = 8 # must be power of 2, > 0 - proc copyStrTable*(dest: var TStrTable, src: TStrTable) = dest.counter = src.counter setLen(dest.data, src.data.len) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index dbf388e9d6..cac2743a1e 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -126,6 +126,8 @@ type moduleToNifSuffix: Table[FileIndex, string] locals: HashSet[ItemId] # track proc-local symbols inProc: int + writtenTypes: seq[PType] # types written in this module, to be unloaded later + writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, @@ -238,6 +240,8 @@ proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = elif typ.itemId.module == w.currentModule and typ.state == Complete: typ.state = Sealed writeTypeDef(w, dest, typ) + # Collect for later unloading after entire module is written + w.writtenTypes.add typ else: dest.addSymUse pool.syms.getOrIncl(w.typeToNifSym(typ)), NoLineInfo @@ -291,6 +295,11 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeSym(w, dest, sym.instantiatedFromImpl) dest.addParRi + # Collect for later unloading after entire module is written + if sym.kindImpl notin {skModule, skPackage}: + # do not unload modules + w.writtenSyms.add sym + proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = if sym == nil: dest.addDotToken() @@ -453,14 +462,19 @@ proc writeToplevelNode(w: var Writer; outer, inner: var TokenBuf; n: PNode) = else: writeNode w, outer, n +proc createStmtList(buf: var TokenBuf; info: PackedLineInfo) {.inline.} = + buf.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), info + buf.addDotToken # flags + buf.addDotToken # type + proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var outer = createTokenBuf(300) var inner = createTokenBuf(300) let rootInfo = trLineInfo(w, n.info) - outer.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), rootInfo - inner.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), rootInfo + createStmtList(outer, rootInfo) + createStmtList(inner, rootInfo) w.writeToplevelNode outer, inner, n @@ -472,7 +486,7 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = let d = completeGeneratedFilePath(config, nifFilename).string var dest = createTokenBuf(600) - dest.addParLe pool.tags.getOrIncl(toNifTag(nkStmtList)), rootInfo + createStmtList(dest, rootInfo) dest.add w.deps dest.add outer dest.add inner @@ -481,6 +495,13 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode) = writeFile(dest, d) createIndex(d, false, dest[0].info) + # Unload all written types and symbols from memory after the entire module is written + # This handles cyclic references correctly since everything is written before unloading + for typ in w.writtenTypes: + forcePartial(typ) + for sym in w.writtenSyms: + forcePartial(sym) + # --------------------------- Loader (lazy!) ----------------------------------------------- @@ -548,7 +569,7 @@ type syms: Table[ItemId, (PSym, NifIndexEntry)] mods: seq[NifModule] cache: IdentCache - #moduleToNifSuffix: Table[FileIndex, string] + moduleToNifSuffix: Table[FileIndex, string] proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable @@ -567,7 +588,7 @@ proc moduleId(c: var DecodeContext; suffix: string): FileIndex = result = c.infos.config.registerNifSuffix(suffix, isKnownFile) if not isKnownFile: let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string - let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".idx.nif")).string + let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.idx.nif")).string if result.int >= c.mods.len: c.mods.setLen(result.int + 1) c.mods[result.int] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile)) @@ -580,7 +601,7 @@ proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifInd if result.offset == 0: raiseAssert "symbol has no offset: " & nifName -proc loadNode(c: var DecodeContext; n: var Cursor): PNode +proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string): PNode proc loadTypeStub(c: var DecodeContext; t: SymId): PType = let name = pool.syms[t] @@ -619,10 +640,10 @@ proc loadTypeStub(c: var DecodeContext; n: var Cursor): PType = else: raiseAssert "type expected but got " & $n.kind -proc loadSymStub(c: var DecodeContext; t: SymId): PSym = +proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string): PSym = let symAsStr = pool.syms[t] let sn = parseSymName(symAsStr) - let module = moduleId(c, sn.module) + let module = moduleId(c, if sn.module.len > 0: sn.module else: thisModule) let val = addr c.mods[module.int32].symCounter inc val[] @@ -632,19 +653,20 @@ proc loadSymStub(c: var DecodeContext; t: SymId): PSym = let offs = c.getOffset(module, symAsStr) result = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[id] = (result, offs) + c.moduleToNifSuffix[module] = (if sn.module.len > 0: sn.module else: thisModule) -proc loadSymStub(c: var DecodeContext; n: var Cursor): PSym = +proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string): PSym = if n.kind == DotToken: result = nil inc n elif n.kind == Symbol: let s = n.symId - result = loadSymStub(c, s) + result = loadSymStub(c, s, thisModule) inc n elif n.kind == ParLe and n.tagId == sdefTag: let s = n.firstSon.symId skip n - result = loadSymStub(c, s) + result = loadSymStub(c, s, thisModule) else: raiseAssert "sym expected but got " & $n.kind @@ -700,6 +722,7 @@ proc loadType*(c: var DecodeContext; t: PType) = inc n expect n, SymbolDef # ignore the type's name, we have already used it to create this PType's itemId! + let typesModule = parseSymName(pool.syms[n.symId]).module inc n #loadField t.kind loadField t.flagsImpl @@ -710,9 +733,9 @@ proc loadType*(c: var DecodeContext; t: PType) = loadField t.itemId.item # nonUniqueId t.typeInstImpl = loadTypeStub(c, n) - t.nImpl = loadNode(c, n) - t.ownerFieldImpl = loadSymStub(c, n) - t.symImpl = loadSymStub(c, n) + t.nImpl = loadNode(c, n, typesModule) + t.ownerFieldImpl = loadSymStub(c, n, typesModule) + t.symImpl = loadSymStub(c, n, typesModule) loadLoc c, n, t.locImpl while n.kind != ParRi: @@ -720,7 +743,7 @@ proc loadType*(c: var DecodeContext; t: PType) = skipParRi n -proc loadAnnex(c: var DecodeContext; n: var Cursor): PLib = +proc loadAnnex(c: var DecodeContext; n: var Cursor; thisModule: string): PLib = if n.kind == DotToken: result = nil inc n @@ -732,7 +755,7 @@ proc loadAnnex(c: var DecodeContext; n: var Cursor): PLib = expect n, StringLit result.name = pool.strings[n.litId] inc n - result.path = loadNode(c, n) + result.path = loadNode(c, n, thisModule) skipParRi n else: raiseAssert "`lib/annex` information expected" @@ -741,7 +764,8 @@ proc loadSym*(c: var DecodeContext; s: PSym) = if s.state != Partial: return s.state = Sealed var buf = createTokenBuf(30) - var n = cursorFromIndexEntry(c, s.itemId.module.FileIndex, c.syms[s.itemId][1], buf) + let symsModule = s.itemId.module.FileIndex + var n = cursorFromIndexEntry(c, symsModule, c.syms[s.itemId][1], buf) expect n, ParLe if n.tagId != sdefTag: @@ -772,7 +796,7 @@ proc loadSym*(c: var DecodeContext; s: PSym) = case s.kindImpl of skLet, skVar, skField, skForVar: - s.guardImpl = loadSymStub(c, n) + s.guardImpl = loadSymStub(c, n, c.moduleToNifSuffix[symsModule]) loadField s.bitsizeImpl loadField s.alignmentImpl else: @@ -785,17 +809,18 @@ proc loadSym*(c: var DecodeContext; s: PSym) = else: loadField s.positionImpl s.typImpl = loadTypeStub(c, n) - s.ownerFieldImpl = loadSymStub(c, n) + s.ownerFieldImpl = loadSymStub(c, n, c.moduleToNifSuffix[symsModule]) # We do not store `sym.ast` here but instead set it in the deserializer #writeNode(w, sym.ast) loadLoc c, n, s.locImpl - s.constraintImpl = loadNode(c, n) - s.instantiatedFromImpl = loadSymStub(c, n) + s.constraintImpl = loadNode(c, n, c.moduleToNifSuffix[symsModule]) + s.instantiatedFromImpl = loadSymStub(c, n, c.moduleToNifSuffix[symsModule]) skipParRi n template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNodeKind; body: untyped) = let info = c.infos.oldLineInfo(n.info) + inc n let flags = loadAtom(TNodeFlags, n) result = newNodeI(kind, info) result.flags = flags @@ -803,15 +828,18 @@ template withNode(c: var DecodeContext; n: var Cursor; result: PNode; kind: TNod body skipParRi n -proc loadNode(c: var DecodeContext; n: var Cursor): PNode = +proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string): PNode = result = nil - case n.kind: + case n.kind + of Symbol: + let info = c.infos.oldLineInfo(n.info) + result = newSymNode(c.loadSymStub(n, thisModule), info) of DotToken: result = nil inc n of ParLe: let kind = n.nodeKind - case kind: + case kind of nkNone: # special NIF introduced tag? case pool.tags[n.tagId] @@ -819,28 +847,30 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode = inc n let typ = c.loadTypeStub n let info = c.infos.oldLineInfo(n.info) - result = newSymNode(c.loadSymStub n, info) + result = newSymNode(c.loadSymStub(n, thisModule), info) result.typField = typ skipParRi n of symDefTagName: let name = n.firstSon assert name.kind == SymbolDef - result = newSymNode(c.loadSymStub name.symId, c.infos.oldLineInfo(n.info)) + result = newSymNode(c.loadSymStub(name.symId, thisModule), c.infos.oldLineInfo(n.info)) skip n of typeDefTagName: raiseAssert "`td` tag in invalid context" of "none": result = newNodeI(nkNone, c.infos.oldLineInfo(n.info)) + inc n result.flags = loadAtom(TNodeFlags, n) skipParRi n else: raiseAssert "Unknown NIF tag " & pool.tags[n.tagId] of nkEmpty: result = newNodeI(nkEmpty, c.infos.oldLineInfo(n.info)) - result.flags = loadAtom(TNodeFlags, n) + inc n skipParRi n of nkIdent: let info = c.infos.oldLineInfo(n.info) + inc n let flags = loadAtom(TNodeFlags, n) let typ = c.loadTypeStub n expect n, Ident @@ -850,8 +880,9 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode = result.typField = typ skipParRi n of nkSym: - let info = c.infos.oldLineInfo(n.info) - result = newSymNode(c.loadSymStub n, info) + #let info = c.infos.oldLineInfo(n.info) + #result = newSymNode(c.loadSymStub n, info) + raiseAssert "nkSym should be mapped to a NIF symbol, not a tag" of nkCharLit: c.withNode n, result, kind: expect n, CharLit @@ -897,24 +928,71 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode = else: c.withNode n, result, kind: while n.kind != ParRi: - result.sons.add c.loadNode(n) + result.sons.add c.loadNode(n, thisModule) else: raiseAssert "Not yet implemented " & $n.kind -when false: - proc loadNifModule*(c: var DecodeContext; f: FileIndex): PNode = - let moduleSuffix = moduleSuffix(c.infos.config, f) - let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(moduleSuffix), ".nif").string +proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = + moduleSuffix(toFullPath(conf, f), cast[seq[string]](conf.searchPaths)) - var buf = createTokenBuf(300) - var s = nifstreams.open(modFile) - # XXX We can optimize this here and only load the top level entries! - try: - nifcursors.parse(s, buf, NoLineInfo) - finally: - nifstreams.close(s) - var n = cursorAt(buf, 0) - result = loadNode(c, n) +proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; + nifName: string; entry: NifIndexEntry; thisModule: string): PSym = + ## Loads a symbol from the NIF index entry. + ## Creates a symbol stub and loads its full definition. + result = loadSymStub(c, pool.syms.getOrIncl nifName, thisModule) + +proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; + interf, interfHidden: var TStrTable; thisModule: string) = + ## Populates interface tables from the NIF index structure. + ## Uses the index's public/private tables instead of traversing AST. + let idx = addr c.mods[module.int32].index + + # Add all public symbols to interf (exported interface) and interfHidden + for nifName, entry in idx.public: + if not nifName.startsWith("`t"): + # do not load types, they are not part of an interface but an implementation detail! + #echo "LOADING SYM ", nifName, " ", entry.offset + let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) + if sym != nil: + strTableAdd(interf, sym) + strTableAdd(interfHidden, sym) + + when false: + # Add private symbols to interfHidden only + for nifName, entry in idx.private: + let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) + if sym != nil: + strTableAdd(interfHidden, sym) + +proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = + let suffix = moduleSuffix(conf, f) + result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string + +proc toNifIndexFilename*(conf: ConfigRef; f: FileIndex): string = + let suffix = moduleSuffix(conf, f) + result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.idx.nif").string + +proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable): PNode = + let suffix = moduleSuffix(c.infos.config, f) + let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(suffix), ".nif").string + + # Ensure module index is loaded - moduleId returns the FileIndex for this suffix + let module = moduleId(c, suffix) + + # Populate interface tables from the NIF index structure + # Use the FileIndex returned by moduleId to ensure we access the correct index + populateInterfaceTablesFromIndex(c, module, interf, interfHidden, suffix) + + var buf = createTokenBuf(300) + var s = nifstreams.open(modFile) + discard processDirectives(s.r) + # XXX We can optimize this here and only load the top level entries! + try: + nifcursors.parse(s, buf, NoLineInfo) + finally: + nifstreams.close(s) + var n = cursorAt(buf, 0) + result = loadNode(c, n, suffix) when isMainModule: import std / syncio diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 14dc7c5994..baa852b9e0 100644 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -68,8 +68,6 @@ template mdbg*: bool {.deprecated.} = # --------------------------------------------------------------------------- proc lookupInRecord*(n: PNode, field: PIdent): PSym -proc mustRehash*(length, counter: int): bool -proc nextTry*(h, maxHash: Hash): Hash {.inline.} # ------------- table[int, int] --------------------------------------------- const @@ -216,10 +214,6 @@ proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym = proc hashNode(p: RootRef): Hash = result = hash(cast[pointer](p)) -proc mustRehash(length, counter: int): bool = - assert(length > counter) - result = (length * 2 < counter * 3) or (length - counter < 4) - import std/tables const backrefStyle = "\e[90m" @@ -484,12 +478,6 @@ proc debug(n: PNode; conf: ConfigRef) = this.value(n) echo($this.res) -proc nextTry(h, maxHash: Hash): Hash {.inline.} = - result = ((5 * h) + 1) and maxHash - # For any initial h in range(maxHash), repeating that maxHash times - # generates each int in range(maxHash) exactly once (see any text on - # random-number generation for proof). - proc objectSetContains*(t: TObjectSet, obj: RootRef): bool = # returns true whether n is in t var h: Hash = hashNode(obj) and high(t.data) # start with real hash value @@ -537,95 +525,6 @@ proc objectSetContainsOrIncl*(t: var TObjectSet, obj: RootRef): bool = inc(t.counter) result = false -proc strTableContains*(t: TStrTable, n: PSym): bool = - var h: Hash = n.name.h and high(t.data) # start with real hash value - while t.data[h] != nil: - if (t.data[h] == n): - return true - h = nextTry(h, high(t.data)) - result = false - -proc strTableRawInsert(data: var seq[PSym], n: PSym) = - var h: Hash = n.name.h and high(data) - while data[h] != nil: - if data[h] == n: - # allowed for 'export' feature: - #InternalError(n.info, "StrTableRawInsert: " & n.name.s) - return - h = nextTry(h, high(data)) - assert(data[h] == nil) - data[h] = n - -proc symTabReplaceRaw(data: var seq[PSym], prevSym: PSym, newSym: PSym) = - assert prevSym.name.h == newSym.name.h - var h: Hash = prevSym.name.h and high(data) - while data[h] != nil: - if data[h] == prevSym: - data[h] = newSym - return - h = nextTry(h, high(data)) - assert false - -proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) = - symTabReplaceRaw(t.data, prevSym, newSym) - -proc strTableEnlarge(t: var TStrTable) = - var n: seq[PSym] - newSeq(n, t.data.len * GrowthFactor) - for i in 0..high(t.data): - if t.data[i] != nil: strTableRawInsert(n, t.data[i]) - swap(t.data, n) - -proc strTableAdd*(t: var TStrTable, n: PSym) = - if mustRehash(t.data.len, t.counter): strTableEnlarge(t) - strTableRawInsert(t.data, n) - inc(t.counter) - -proc strTableInclReportConflict*(t: var TStrTable, n: PSym; - onConflictKeepOld = false): PSym = - # if `t` has a conflicting symbol (same identifier as `n`), return it - # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true` - # and a conflict was found. - assert n.name != nil - var h: Hash = n.name.h and high(t.data) - var replaceSlot = -1 - while true: - var it = t.data[h] - if it == nil: break - # Semantic checking can happen multiple times thanks to templates - # and overloading: (var x=@[]; x).mapIt(it). - # So it is possible the very same sym is added multiple - # times to the symbol table which we allow here with the 'it == n' check. - if it.name.id == n.name.id: - if it == n: return nil - replaceSlot = h - h = nextTry(h, high(t.data)) - if replaceSlot >= 0: - result = t.data[replaceSlot] # found it - if not onConflictKeepOld: - t.data[replaceSlot] = n # overwrite it with newer definition! - return result # but return the old one - elif mustRehash(t.data.len, t.counter): - strTableEnlarge(t) - strTableRawInsert(t.data, n) - else: - assert(t.data[h] == nil) - t.data[h] = n - inc(t.counter) - result = nil - -proc strTableIncl*(t: var TStrTable, n: PSym; - onConflictKeepOld = false): bool {.discardable.} = - result = strTableInclReportConflict(t, n, onConflictKeepOld) != nil - -proc strTableGet*(t: TStrTable, name: PIdent): PSym = - var h: Hash = name.h and high(t.data) - while true: - result = t.data[h] - if result == nil: break - if result.name.id == name.id: break - h = nextTry(h, high(t.data)) - type TIdentIter* = object # iterator over all syms with same identifier diff --git a/compiler/astdef.nim b/compiler/astdef.nim index fb32178223..cc0c4c49a6 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -1031,3 +1031,106 @@ proc forcePartial*(t: PType) = t.paddingAtEndImpl = 0'i16 t.locImpl = TLoc() t.typeInstImpl = nil + +const # for all kind of hash tables: + GrowthFactor* = 2 # must be power of 2, > 0 + StartSize* = 8 # must be power of 2, > 0 + +proc nextTry*(h, maxHash: Hash): Hash {.inline.} = + result = ((5 * h) + 1) and maxHash + # For any initial h in range(maxHash), repeating that maxHash times + # generates each int in range(maxHash) exactly once (see any text on + # random-number generation for proof). + +proc mustRehash*(length, counter: int): bool = + assert(length > counter) + result = (length * 2 < counter * 3) or (length - counter < 4) + +proc strTableContains*(t: TStrTable, n: PSym): bool = + var h: Hash = n.name.h and high(t.data) # start with real hash value + while t.data[h] != nil: + if (t.data[h] == n): + return true + h = nextTry(h, high(t.data)) + result = false + +proc strTableRawInsert(data: var seq[PSym], n: PSym) = + var h: Hash = n.name.h and high(data) + while data[h] != nil: + if data[h] == n: + # allowed for 'export' feature: + #InternalError(n.info, "StrTableRawInsert: " & n.name.s) + return + h = nextTry(h, high(data)) + assert(data[h] == nil) + data[h] = n + +proc symTabReplaceRaw(data: var seq[PSym], prevSym: PSym, newSym: PSym) = + assert prevSym.name.h == newSym.name.h + var h: Hash = prevSym.name.h and high(data) + while data[h] != nil: + if data[h] == prevSym: + data[h] = newSym + return + h = nextTry(h, high(data)) + assert false + +proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) = + symTabReplaceRaw(t.data, prevSym, newSym) + +proc strTableEnlarge(t: var TStrTable) = + var n: seq[PSym] + newSeq(n, t.data.len * GrowthFactor) + for i in 0..high(t.data): + if t.data[i] != nil: strTableRawInsert(n, t.data[i]) + swap(t.data, n) + +proc strTableAdd*(t: var TStrTable, n: PSym) = + if mustRehash(t.data.len, t.counter): strTableEnlarge(t) + strTableRawInsert(t.data, n) + inc(t.counter) + +proc strTableInclReportConflict*(t: var TStrTable, n: PSym; + onConflictKeepOld = false): PSym = + # if `t` has a conflicting symbol (same identifier as `n`), return it + # otherwise return `nil`. Incl `n` to `t` unless `onConflictKeepOld = true` + # and a conflict was found. + assert n.name != nil + var h: Hash = n.name.h and high(t.data) + var replaceSlot = -1 + while true: + var it = t.data[h] + if it == nil: break + # Semantic checking can happen multiple times thanks to templates + # and overloading: (var x=@[]; x).mapIt(it). + # So it is possible the very same sym is added multiple + # times to the symbol table which we allow here with the 'it == n' check. + if it.name.id == n.name.id: + if it == n: return nil + replaceSlot = h + h = nextTry(h, high(t.data)) + if replaceSlot >= 0: + result = t.data[replaceSlot] # found it + if not onConflictKeepOld: + t.data[replaceSlot] = n # overwrite it with newer definition! + return result # but return the old one + elif mustRehash(t.data.len, t.counter): + strTableEnlarge(t) + strTableRawInsert(t.data, n) + else: + assert(t.data[h] == nil) + t.data[h] = n + inc(t.counter) + result = nil + +proc strTableIncl*(t: var TStrTable, n: PSym; + onConflictKeepOld = false): bool {.discardable.} = + result = strTableInclReportConflict(t, n, onConflictKeepOld) != nil + +proc strTableGet*(t: TStrTable, name: PIdent): PSym = + var h: Hash = name.h and high(t.data) + while true: + result = t.data[h] + if result == nil: break + if result.name.id == name.id: break + h = nextTry(h, high(t.data)) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index b5ece69ae6..20b1db25e6 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3471,6 +3471,10 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = genProcPrototype(p.module, sym) else: genProc(p.module, sym) + # For cross-module inline procs with optCompress, ensure prototype is emitted + if sym.typ.callConv == ccInline and optCompress in p.config.globalOptions and + sym.itemId.module != p.module.module.position: + genProcPrototype(p.module, sym) if sym.loc.snippet == "" or sym.loc.lode == nil: internalError(p.config, n.info, "expr: proc not init " & sym.name.s) putLocIntoDest(p, d, sym.loc) @@ -3479,6 +3483,12 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = var lit = newBuilder("") genLiteral(p, sym.astdef, sym.typ, lit) putIntoDest(p, d, n, extract(lit), OnStatic) + elif optCompress in p.config.globalOptions: + # With delayed codegen, we need to ensure the definition is generated + # not just the extern header declaration + requestConstImpl(p, sym) + assert((sym.loc.snippet != "") and (sym.loc.t != nil)) + putLocIntoDest(p, d, sym.loc) elif delayedCodegen(p.module): genConstHeader(p.module, p.module, p, sym) assert((sym.loc.snippet != "") and (sym.loc.t != nil)) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 545274b125..76669d41ba 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -84,7 +84,7 @@ proc fillBackendName(m: BModule; s: PSym) = if m.hcrOn: result.add '_' result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config)) - ensureMutable s + backendEnsureMutable s s.locImpl.snippet = result proc fillParamName(m: BModule; s: PSym) = diff --git a/compiler/cgen.nim b/compiler/cgen.nim index e45621210a..a932c180ff 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1508,7 +1508,7 @@ proc genProcNoForward(m: BModule, prc: PSym) = # mangle the inline proc based on the module where it is defined - # not on the first module that uses it if m.module.itemId.module != prc.itemId.module and optCompress in m.config.globalOptions: - let prcCopy = copyInlineProc(prc, m.idgen) + let prcCopy = prc # copyInlineProc(prc, m.idgen) fillProcLoc(m, prcCopy.ast[namePos]) genProcPrototype(m, prcCopy) genProcAux(m, prcCopy) @@ -1518,9 +1518,9 @@ proc genProcNoForward(m: BModule, prc: PSym) = fillProcLoc(m2, prc.ast[namePos]) #elif {sfExportc, sfImportc} * prc.flags == {}: # # reset name to restore consistency in case of hashing collisions: - # echo "resetting ", prc.id, " by ", m.module.name.s - # prc.loc.snippet = nil - # prc.loc.snippet = mangleName(m, prc) + # #echo "resetting ", prc.id, " by ", m.module.name.s + # #prc.loc.snippet = nil + # #prc.loc.snippet = mangleName(m, prc) genProcPrototype(m, prc) genProcAux(m, prc) elif sfImportc notin prc.flags: @@ -2523,7 +2523,7 @@ proc writeModule(m: BModule, pending: bool) = while m.queue.len > 0: let sym = m.queue.pop() - genProcAux(m, sym) + genProcNoForward(m, sym) finishTypeDescriptions(m) if sfMainModule in m.module.flags: diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index fe2131c555..408acd3ed3 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -16,6 +16,8 @@ import ../dist/checksums/src/checksums/md5 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb import ic / [packed_ast, ic] +when not defined(nimKochBootstrap): + import ast2nif when defined(nimPreviewSlimSystem): import std/assertions @@ -741,6 +743,31 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; else: result = nil +when not defined(nimKochBootstrap): + proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex; + cachedModules: var seq[FileIndex]): PSym = + ## Returns 'nil' if the module needs to be recompiled. + ## Loads module from NIF file when optCompress is enabled. + + if not fileExists(toNifFilename(g.config, fileIdx)): + return nil + + # Create module symbol + let filename = AbsoluteFile toFullPath(g.config, fileIdx) + result = PSym( + kindImpl: skModule, + itemId: ItemId(module: int32(fileIdx), item: 0'i32), + name: getIdent(g.cache, splitFile(filename).name), + infoImpl: newLineInfo(fileIdx, 1, 1), + positionImpl: int(fileIdx), + ) + setOwner(result, getPackage(g.config, g.cache, fileIdx)) + + # Register module in graph + registerModule(g, result) + result.astImpl = loadNifModule(ast.program, fileIdx, g.ifaces[fileIdx.int].interf, g.ifaces[fileIdx.int].interfHidden) + cachedModules.add fileIdx + proc configComplete*(g: ModuleGraph) = rememberStartupConfig(g.startupPackedConfig, g.config) diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 0137fde646..f00d0a3196 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -235,7 +235,7 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator raiseAssert "use setPipeLinePass to set a proper PipelinePass" when not defined(nimKochBootstrap): - if optCompress in graph.config.globalOptions: + if optCompress in graph.config.globalOptions and not graph.config.isDefined("nimscript"): topLevelStmts.add finalNode writeNifModule(graph.config, module.position.int32, topLevelStmts) @@ -260,7 +260,13 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF discard processPipelineModule(graph, result, idGeneratorFromModule(result), s) if result == nil: var cachedModules: seq[FileIndex] = @[] - result = moduleFromRodFile(graph, fileIdx, cachedModules) + when not defined(nimKochBootstrap): + # Try loading from NIF file first if optCompress is enabled + if optCompress in graph.config.globalOptions and not graph.config.isDefined("nimscript"): + result = moduleFromNifFile(graph, fileIdx, cachedModules) + if result == nil: + # Fall back to ROD file loading + result = moduleFromRodFile(graph, fileIdx, cachedModules) let path = toFullPath(graph.config, fileIdx) let filename = AbsoluteFile path # it could be a stdinfile/cmdfile @@ -315,10 +321,12 @@ proc connectPipelineCallbacks*(graph: ModuleGraph) = proc compilePipelineSystemModule*(graph: ModuleGraph) = if graph.systemModule == nil: + graph.withinSystem = true connectPipelineCallbacks(graph) graph.config.m.systemFileIdx = fileInfoIdx(graph.config, graph.config.libpath / RelativeFile"system.nim") discard graph.compilePipelineModule(graph.config.m.systemFileIdx, {sfSystemModule}) + graph.withinSystem = false proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) = connectPipelineCallbacks(graph) @@ -335,7 +343,9 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx graph.importStack.add projectFile if projectFile == systemFileIdx: + graph.withinSystem = true discard graph.compilePipelineModule(projectFile, {sfMainModule, sfSystemModule}) + graph.withinSystem = false else: graph.compilePipelineSystemModule() discard graph.compilePipelineModule(projectFile, {sfMainModule}) diff --git a/lib/system.nim b/lib/system.nim index c7667cfba4..ecc14b2ea7 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1620,6 +1620,41 @@ when notJSnotNims: import system/ansi_c import system/memory +when notJSnotNims and defined(nimSeqsV2): + const nimStrVersion {.core.} = 2 + + type + NimStrPayloadBase = object + cap: int + + NimStrPayload {.core.} = object + cap: int + data: UncheckedArray[char] + + NimStringV2 {.core.} = object + len: int + p: ptr NimStrPayload ## can be nil if len == 0. + +when defined(windows): + proc GetLastError(): int32 {.header: "", nodecl.} + const ERROR_BAD_EXE_FORMAT = 193 + +when notJSnotNims: + when defined(nimSeqsV2): + proc nimToCStringConv(s: NimStringV2): cstring {.compilerproc, nonReloadable, inline.} + + when hostOS != "standalone" and hostOS != "any": + type + LibHandle = pointer # private type + ProcAddr = pointer # library loading and loading of procs: + + proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} + proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} + proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} + + proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} + + include "system/dyncalls" {.push stackTrace: off.} @@ -1648,21 +1683,6 @@ when not defined(js) and defined(nimV2): vTable: UncheckedArray[pointer] # vtable for types PNimTypeV2 = ptr TNimTypeV2 -when notJSnotNims and defined(nimSeqsV2): - const nimStrVersion {.core.} = 2 - - type - NimStrPayloadBase = object - cap: int - - NimStrPayload {.core.} = object - cap: int - data: UncheckedArray[char] - - NimStringV2 {.core.} = object - len: int - p: ptr NimStrPayload ## can be nil if len == 0. - when not defined(nimIcIntegrityChecks): import system/exceptions export exceptions @@ -2316,19 +2336,6 @@ when not defined(js): when notJSnotNims: - when hostOS != "standalone" and hostOS != "any": - type - LibHandle = pointer # private type - ProcAddr = pointer # library loading and loading of procs: - - proc nimLoadLibrary(path: string): LibHandle {.compilerproc, hcrInline, nonReloadable.} - proc nimUnloadLibrary(lib: LibHandle) {.compilerproc, hcrInline, nonReloadable.} - proc nimGetProcAddr(lib: LibHandle, name: cstring): ProcAddr {.compilerproc, hcrInline, nonReloadable.} - - proc nimLoadLibraryError(path: string) {.compilerproc, hcrInline, nonReloadable.} - - include "system/dyncalls" - import system/countbits_impl include "system/sets" diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index e229a0f4b4..9d533ce7a8 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -1,3 +1,5 @@ +{.push stack_trace: off.} + proc succ*[T, V: Ordinal](x: T, y: V = 1): T {.magic: "Succ", noSideEffect.} = ## Returns the `y`-th successor (default: 1) of the value `x`. ## @@ -403,3 +405,5 @@ proc `%%`*(x, y: int8): int8 {.inline.} = cast[int8](cast[uint8](x) mod cast[u proc `%%`*(x, y: int16): int16 {.inline.} = cast[int16](cast[uint16](x) mod cast[uint16](y)) proc `%%`*(x, y: int32): int32 {.inline.} = cast[int32](cast[uint32](x) mod cast[uint32](y)) proc `%%`*(x, y: int64): int64 {.inline.} = cast[int64](cast[uint64](x) mod cast[uint64](y)) + +{.pop.} diff --git a/lib/system/dyncalls.nim b/lib/system/dyncalls.nim index 817f7d8a5f..2088e466e3 100644 --- a/lib/system/dyncalls.nim +++ b/lib/system/dyncalls.nim @@ -12,7 +12,7 @@ # However, the interface has been designed to take platform differences into # account and been ported to all major platforms. -{.push stack_trace: off.} +{.push stack_trace: off, checks: off.} const NilLibHandle: LibHandle = nil diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim index 2fb958999f..12552515cc 100644 --- a/lib/system/excpt.nim +++ b/lib/system/excpt.nim @@ -22,10 +22,6 @@ var ## instead of `stdmsg.write` when printing stacktrace. ## Unstable API. -when defined(windows): - proc GetLastError(): int32 {.header: "", nodecl.} - const ERROR_BAD_EXE_FORMAT = 193 - when not defined(windows) or not defined(guiapp): proc writeToStdErr(msg: cstring) = rawWrite(cstderr, msg) proc writeToStdErr(msg: cstring, length: int) =