From bc26f70ca85955bfb82d80813ba6a86a3e188f26 Mon Sep 17 00:00:00 2001 From: araq Date: Fri, 14 Nov 2025 13:40:31 +0100 Subject: [PATCH] progress --- compiler/ast2nif.nim | 92 ++++++++++++++++++++++--------------------- compiler/ccgexprs.nim | 12 +++--- compiler/cgen.nim | 6 +++ compiler/cgendata.nim | 4 ++ compiler/msgs.nim | 10 +++++ 5 files changed, 74 insertions(+), 50 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 5ca1f5f141..1528b9d112 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -13,7 +13,7 @@ import std / [assertions, tables, sets] from std / strutils import startsWith import astdef, idents, msgs, options import lineinfos as astli -import pathutils +import pathutils #, modulegraphs import "../dist/nimony/src/lib" / [bitabs, nifstreams, nifcursors, lineinfos, nifindexes, nifreader] import "../dist/nimony/src/gear2" / modnames @@ -258,6 +258,10 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addParLe sdefTag, trLineInfo(w, sym.infoImpl) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo + if sfExported in sym.flagsImpl: + dest.addIdent "x" + else: + dest.addDotToken if sym.magicImpl == mNone: dest.addDotToken else: @@ -537,48 +541,37 @@ type DecodeContext* = object infos: LineInfoWriter - moduleIds: Table[string, int32] + #moduleIds: Table[string, int32] types: Table[ItemId, (PType, NifIndexEntry)] syms: Table[ItemId, (PSym, NifIndexEntry)] mods: seq[NifModule] cache: IdentCache - moduleToNifSuffix: Table[FileIndex, string] + #moduleToNifSuffix: Table[FileIndex, string] proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache) -proc idToIdx(x: int32): int {.inline.} = - assert x <= -2'i32 - result = -(x+2) - -proc cursorFromIndexEntry(c: var DecodeContext; module: int32; entry: NifIndexEntry; +proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry; buf: var TokenBuf): Cursor = - let m = idToIdx(module) - let s = addr c.mods[m].stream + let s = addr c.mods[module.int32].stream s.r.jumpTo entry.offset var buf = createTokenBuf(30) nifcursors.parse(s[], buf, entry.info) result = cursorAt(buf, 0) -proc moduleId(c: var DecodeContext; suffix: string): int32 = - # We don't know the "real" FileIndex due to our mapping to a short "Module suffix" - # This is not a problem, we use negative `ItemId.module` values here and then - # there is no interference with in-memory-modules. Modulegraphs.nim already uses -1 - # so we start at -2 here. - result = c.moduleIds.getOrDefault(suffix) - if result == 0: - result = -int32(c.moduleIds.len + 2) # negative index! +proc moduleId(c: var DecodeContext; suffix: string): FileIndex = + var isKnownFile = false + result = c.infos.config.registerNifSuffix(suffix, isKnownFile) + if not isKnownFile: let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".idx.nif")).string - c.moduleIds[suffix] = result - c.mods.add NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile)) - assert c.mods.len-1 == idToIdx(result) + if result.int >= c.mods.len: + c.mods.setLen(result.int + 1) + c.mods[result.int] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile)) -proc getOffset(c: var DecodeContext; module: int32; nifName: string): NifIndexEntry = - assert module < 0'i32 - let index = idToIdx(module) - let ii = addr c.mods[index].index +proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = + let ii = addr c.mods[module.int32].index result = ii.public.getOrDefault(nifName) if result.offset == 0: result = ii.private.getOrDefault(nifName) @@ -602,10 +595,10 @@ proc loadTypeStub(c: var DecodeContext; t: SymId): PType = inc i if i < name.len and name[i] == '.': inc i let suffix = name.substr(i) - let id = ItemId(module: moduleId(c, suffix), item: itemId) + let id = ItemId(module: moduleId(c, suffix).int32, item: itemId) result = c.types.getOrDefault(id)[0] if result == nil: - let offs = c.getOffset(id.module, name) + let offs = c.getOffset(id.module.FileIndex, name) result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) c.types[id] = (result, offs) @@ -628,10 +621,10 @@ proc loadSymStub(c: var DecodeContext; t: SymId): PSym = let symAsStr = pool.syms[t] let sn = parseSymName(symAsStr) let module = moduleId(c, sn.module) - let val = addr c.mods[idToIdx(module)].symCounter + let val = addr c.mods[module.int32].symCounter inc val[] - let id = ItemId(module: module, item: val[]) + let id = ItemId(module: module.int32, item: val[]) result = c.syms.getOrDefault(id)[0] if result == nil: let offs = c.getOffset(module, symAsStr) @@ -697,7 +690,7 @@ proc loadType*(c: var DecodeContext; t: PType) = if t.state != Partial: return t.state = Sealed var buf = createTokenBuf(30) - var n = cursorFromIndexEntry(c, t.itemId.module, c.types[t.itemId][1], buf) + var n = cursorFromIndexEntry(c, t.itemId.module.FileIndex, c.types[t.itemId][1], buf) expect n, ParLe if n.tagId != tdefTag: @@ -746,7 +739,7 @@ proc loadSym*(c: var DecodeContext; s: PSym) = if s.state != Partial: return s.state = Sealed var buf = createTokenBuf(30) - var n = cursorFromIndexEntry(c, s.itemId.module, c.syms[s.itemId][1], buf) + var n = cursorFromIndexEntry(c, s.itemId.module.FileIndex, c.syms[s.itemId][1], buf) expect n, ParLe if n.tagId != sdefTag: @@ -755,6 +748,17 @@ proc loadSym*(c: var DecodeContext; s: PSym) = expect n, SymbolDef # ignore the symbol's name, we have already used it to create this PSym instance! inc n + if n.kind == Ident: + if pool.strings[n.litId] == "x": + s.flagsImpl.incl sfExported + inc n + else: + raiseAssert "expected `x` as the export marker" + elif n.kind == DotToken: + inc n + else: + raiseAssert "expected `x` or '.' but got " & $n.kind + loadField s.magicImpl loadField s.flagsImpl loadField s.optionsImpl @@ -895,20 +899,20 @@ proc loadNode(c: var DecodeContext; n: var Cursor): PNode = else: raiseAssert "Not yet implemented " & $n.kind +when false: + proc loadNifModule*(c: var DecodeContext; f: FileIndex): PNode = + let moduleSuffix = moduleSuffix(c.infos.config, f) + let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(moduleSuffix), ".nif").string -proc loadNifModule*(c: var DecodeContext; f: FileIndex): PNode = - let moduleSuffix = modname(c.moduleToNifSuffix, f.int, c.infos.config) - let modFile = toGeneratedFile(c.infos.config, AbsoluteFile(moduleSuffix), ".nif").string - - var buf = createTokenBuf(300) - var s = nifstreams.open(modFile) - # XXX We can optimize this here and only load the top level entries! - try: - nifcursors.parse(s, buf, NoLineInfo) - finally: - nifstreams.close(s) - var n = cursorAt(buf, 0) - result = loadNode(c, n) + var buf = createTokenBuf(300) + var s = nifstreams.open(modFile) + # XXX We can optimize this here and only load the top level entries! + try: + nifcursors.parse(s, buf, NoLineInfo) + finally: + nifstreams.close(s) + var n = cursorAt(buf, 0) + result = loadNode(c, n) when isMainModule: import std / syncio diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 5859abd8b4..27c8f607e7 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3435,7 +3435,7 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) = proc genConstStmt(p: BProc, n: PNode) = # This code is only used in the new DCE implementation. - assert useAliveDataFromDce in p.module.flags + assert delayedCodegen(p.module) let m = p.module for it in n: if it[0].kind == nkSym: @@ -3453,7 +3453,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = var sym = n.sym case sym.kind of skMethod: - if useAliveDataFromDce in p.module.flags or {sfDispatcher, sfForward} * sym.flags != {}: + if delayedCodegen(p.module) or {sfDispatcher, sfForward} * sym.flags != {}: # we cannot produce code for the dispatcher yet: fillProcLoc(p.module, n) genProcPrototype(p.module, sym) @@ -3466,7 +3466,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = if sfCompileTime in sym.flags: localError(p.config, n.info, "request to generate code for .compileTime proc: " & sym.name.s) - if useAliveDataFromDce in p.module.flags and sym.typ.callConv != ccInline: + if delayedCodegen(p.module) and sym.typ.callConv != ccInline: fillProcLoc(p.module, n) genProcPrototype(p.module, sym) else: @@ -3479,7 +3479,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = var lit = newBuilder("") genLiteral(p, sym.astdef, sym.typ, lit) putIntoDest(p, d, n, extract(lit), OnStatic) - elif useAliveDataFromDce in p.module.flags: + elif delayedCodegen(p.module): genConstHeader(p.module, p.module, p, sym) assert((sym.loc.snippet != "") and (sym.loc.t != nil)) putLocIntoDest(p, d, sym.loc) @@ -3611,7 +3611,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkWhileStmt: genWhileStmt(p, n) of nkVarSection, nkLetSection: genVarStmt(p, n) of nkConstSection: - if useAliveDataFromDce in p.module.flags: + if delayedCodegen(p.module): genConstStmt(p, n) else: # enforce addressable consts for exportc let m = p.module @@ -3677,7 +3677,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef: if n[genericParamsPos].kind == nkEmpty: var prc = n[namePos].sym - if useAliveDataFromDce in p.module.flags: + if delayedCodegen(p.module): if p.module.alive.contains(prc.itemId.item) and prc.magic in generatedMagics: genProc(p.module, prc) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 518613c1bd..5a0c793fe2 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1445,6 +1445,7 @@ proc genProcPrototype(m: BModule, sym: PSym) = getModuleDllPath(m, sym), '"' & name & '"') elif not containsOrIncl(m.declaredProtos, sym.id): + m.queue.add(sym) let asPtr = isReloadable(m, sym) var header = newBuilder("") var visibility: DeclVisibility = None @@ -2506,6 +2507,11 @@ proc writeModule(m: BModule, pending: bool) = let cfile = getCFile(m) if moduleHasChanged(m.g.graph, m.module): genInitCode(m) + + while m.queue.len > 0: + let sym = m.queue.pop() + genProcAux(m, sym) + finishTypeDescriptions(m) if sfMainModule in m.module.flags: # generate main file: diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index f9ed9c6fda..b82962898f 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -155,6 +155,7 @@ type forwTypeCache*: TypeCache # cache for forward declarations of types declaredThings*: IntSet # things we have declared in this .c file declaredProtos*: IntSet # prototypes we have declared in this .c file + queue*: seq[PSym] # queue of procs to generate alive*: IntSet # symbol IDs of alive data as computed by `dce.nim` headerFiles*: seq[string] # needed headers to include typeInfoMarker*: TypeCache # needed for generating type information @@ -178,6 +179,9 @@ template config*(m: BModule): ConfigRef = m.g.config template config*(p: BProc): ConfigRef = p.module.g.config template vccAndC*(p: BProc): bool = p.module.config.cCompiler == ccVcc and p.module.config.backend == backendC +proc delayedCodegen*(m: BModule): bool {.inline.} = + useAliveDataFromDce in m.flags or m.config.globalOptions.contains(optCompress) + proc includeHeader*(this: BModule; header: string) = if not this.headerFiles.contains header: this.headerFiles.add header diff --git a/compiler/msgs.nim b/compiler/msgs.nim index c49ca8c9b1..f0e7419f68 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -133,6 +133,16 @@ proc fileInfoIdx*(conf: ConfigRef; filename: RelativeFile): FileIndex = var dummy: bool = false fileInfoIdx(conf, AbsoluteFile expandFilename(filename.string), dummy) +proc registerNifSuffix*(conf: ConfigRef; suffix: string; isKnownFile: var bool): FileIndex = + result = conf.m.filenameToIndexTbl.getOrDefault(suffix, InvalidFileIdx) + if result == InvalidFileIdx: + isKnownFile = false + result = conf.m.fileInfos.len.FileIndex + conf.m.fileInfos.add(newFileInfo(AbsoluteFile suffix, RelativeFile suffix)) + conf.m.filenameToIndexTbl[suffix] = result + else: + isKnownFile = true + proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo = result = TLineInfo(fileIndex: fileInfoIdx) if line < int high(uint16):