From 0b3a0b552775d6570c3b9fea6c91b4e3e2703be5 Mon Sep 17 00:00:00 2001 From: araq Date: Wed, 14 Jan 2026 12:31:44 +0100 Subject: [PATCH] WIP --- compiler/ast2nif.nim | 188 ++++++++++++++++++++++++++++++------------- koch.nim | 2 +- 2 files changed, 131 insertions(+), 59 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index d470dcf60f..2a66fa872c 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -693,6 +693,9 @@ let repDeepCopyTag = registerTag("repdeepcopy") let repEnumToStrTag = registerTag("repenumtostr") let repMethodTag = registerTag("repmethod") #let repClassTag = registerTag("repclass") +let repExportTag = registerTag("repexport") +let repFromExportTag = registerTag("repfromexport") +let repExportExceptTag = registerTag("repexportexcept") let includeTag = registerTag("include") let importTag = registerTag("import") let implTag = registerTag("implementation") @@ -730,6 +733,23 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = of GenericInstEntry: discard "will only be written later to ensure it is materialized" +proc writeExports(w: var Writer; content: var TokenBuf) = + ## Write export info as top-level processing instructions + for modIdx, names in w.exports: + let path = toFullPath(w.infos.config, modIdx) + if names.len == 0: + # Export all from this module: (repexport "path") + content.addParLe repExportTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(path), NoLineInfo) + content.addParRi() + else: + # Export specific symbols: (repfromexport "path" name1 name2 ...) + content.addParLe repFromExportTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(path), NoLineInfo) + for name in names: + content.add identToken(pool.strings.getOrIncl(name), NoLineInfo) + content.addParRi() + proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; opsLog: seq[LogEntry]; replayActions: seq[PNode] = @[]) = @@ -749,6 +769,8 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; for op in opsLog: if op.module == thisModule.int: writeOp(w, content, op) + # Write export info as top-level instructions + writeExports(w, content) var bottom = createTokenBuf(300) w.writeToplevelNode content, bottom, n @@ -784,10 +806,6 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; writeFile(dest, d) - let exportBuf = buildExportBuf(w) - createIndex(d, dest[0].info, false, - IndexSections(exportBuf: exportBuf)) - # --------------------------- Loader (lazy!) ----------------------------------------------- proc nodeKind(n: Cursor): TNodeKind {.inline.} = @@ -842,10 +860,15 @@ proc loadBool(n: var Cursor): bool = raiseAssert "(true)/(false) expected" type + ExportInfo* = object + path: string + kind: NifIndexKind + names: seq[StrId] + NifModule = ref object stream: nifstreams.Stream symCounter: int32 - index: NifIndex + index: Table[string, NifIndexEntry] # Simple embedded index for offsets suffix: string DecodeContext* = object @@ -871,25 +894,61 @@ type LoadFlag* = enum LoadFullAst, AlwaysLoadInterface +proc readEmbeddedIndex(s: var Stream): Table[string, NifIndexEntry] = + ## Reads the simple embedded index (index (kv sym offset)...) from indexStartsAt position. + result = initTable[string, NifIndexEntry]() + let indexPos = indexStartsAt(s.r) + if indexPos <= 0: + return + let contentPos = offset(s.r) # Save position + s.r.jumpTo(indexPos) + + var previousOffset = 0 + var t = next(s) + let exportedTagId = pool.tags.getOrIncl("x") + if t.kind == ParLe and pool.tags[t.tagId] == ".index": + t = next(s) + while t.kind != EofToken and t.kind != ParRi: + if t.kind == ParLe: + let vis = if t.tagId == exportedTagId: Exported else: Hidden + let info = t.info + t = next(s) # skip (kv + var key = "" + if t.kind == Symbol: + key = pool.syms[t.symId] + elif t.kind == Ident: + key = pool.strings[t.litId] + t = next(s) # skip symbol + if t.kind == IntLit: + let offset = int(pool.integers[t.intId]) + previousOffset + result[key] = NifIndexEntry(offset: offset, info: info, vis: vis) + previousOffset = offset + t = next(s) # skip offset + if t.kind == ParRi: + t = next(s) # skip ) + else: + t = next(s) + + s.r.jumpTo(contentPos) # Restore position + proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): FileIndex = var isKnownFile = false result = c.infos.config.registerNifSuffix(suffix, isKnownFile) if not isKnownFile or AlwaysLoadInterface in flags: let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string - let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.idx.nif")).string if not fileExists(modFile): raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile & ". This can happen when loading a module from NIF that references another module " & "whose NIF file hasn't been written yet." - c.mods[result] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile), suffix: suffix) + var stream = nifstreams.open(modFile) + let index = readEmbeddedIndex(stream) + c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix) proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = let ii = addr c.mods[module].index - result = ii.public.getOrDefault(nifName) + result = ii[].getOrDefault(nifName) if result.offset == 0: - result = ii.private.getOrDefault(nifName) - if result.offset == 0: - raiseAssert "symbol has no offset: " & nifName + raiseAssert "symbol has no offset: " & nifName proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PNode @@ -1393,18 +1452,17 @@ proc extractBasename(nifName: string): string = result.add c proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; - interf, interfHidden: var TStrTable; thisModule: string) = + interf, interfHidden: var TStrTable; thisModule: string; + exports: seq[ExportInfo]) = ## Populates interface tables from the NIF index structure. - ## Uses the index's public/private tables instead of traversing AST. + ## Uses the simple embedded index for offsets, exports passed from processTopLevel. - # Move the public table and exports list out to avoid iterator invalidation + # Move the index table out to avoid iterator invalidation # (moduleId can add to c.mods which would invalidate Table iterators) - # We move them back after iteration. - var publicTab = move c.mods[module].index.public - var exportsList = move c.mods[module].index.exports + var indexTab = move c.mods[module].index - # Add all public symbols to interf (exported interface) and interfHidden - for nifName, entry in publicTab: + # Add all symbols to interf (exported interface) and interfHidden + for nifName, entry in indexTab: if not nifName.startsWith("`t"): # do not load types, they are not part of an interface but an implementation detail! #echo "LOADING SYM ", nifName, " ", entry.offset @@ -1413,33 +1471,32 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; strTableAdd(interf, sym) strTableAdd(interfHidden, sym) - # Move public table back - c.mods[module].index.public = move publicTab + # Move index table back + c.mods[module].index = move indexTab # Process exports (re-exports from other modules) - for exp in exportsList: - let (path, kind, names) = exp + for exp in exports: # Convert path to module suffix - let expSuffix = moduleSuffix(path, cast[seq[string]](c.infos.config.searchPaths)) + let expSuffix = moduleSuffix(exp.path, cast[seq[string]](c.infos.config.searchPaths)) # Load the exported module's index let expModule = moduleId(c, expSuffix) - # Move the exported module's public table out to avoid iterator invalidation - var expPublicTab = move c.mods[expModule].index.public + # Move the exported module's index table out to avoid iterator invalidation + var expIndexTab = move c.mods[expModule].index # Build a set of names for filtering var nameSet = initHashSet[string]() - for nameId in names: + for nameId in exp.names: nameSet.incl pool.strings[nameId] # Add symbols based on export kind - for nifName, entry in expPublicTab: + for nifName, entry in expIndexTab: if nifName.startsWith("`t"): continue # skip types let basename = extractBasename(nifName) let shouldInclude = - case kind + case exp.kind of ExportIdx: true # export all of FromexportIdx: basename in nameSet # only specific names of ExportexceptIdx: basename notin nameSet # all except specific names @@ -1448,30 +1505,17 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; if shouldInclude: let sym = loadSymFromIndexEntry(c, expModule, nifName, entry, expSuffix) if sym != nil: - strTableAdd(interf, sym) + if entry.vis == Exported: + strTableAdd(interf, sym) strTableAdd(interfHidden, sym) - # Move exported module's public table back - c.mods[expModule].index.public = move expPublicTab - - # Move exports list back - c.mods[module].index.exports = move exportsList - - when false: - # Add private symbols to interfHidden only - for nifName, entry in idx.private: - let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) - if sym != nil: - strTableAdd(interfHidden, sym) + # Move exported module's index table back + c.mods[expModule].index = move expIndexTab proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string -proc toNifIndexFilename*(conf: ConfigRef; f: FileIndex): string = - let suffix = moduleSuffix(conf, f) - result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.idx.nif").string - proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym = result = c.syms.getOrDefault(symAsStr)[0] if result != nil: @@ -1482,14 +1526,11 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo return nil # Local symbols shouldn't be hooks let module = moduleId(c, sn.module) # Look up the symbol in the module's index - var offs = c.mods[module].index.public.getOrDefault(symAsStr) + var offs = c.mods[module].index.getOrDefault(symAsStr) if offs.offset == 0: - if alsoConsiderPrivate: - offs = c.mods[module].index.private.getOrDefault(symAsStr) - if offs.offset == 0: - return nil - else: - return nil + return nil + if not alsoConsiderPrivate and offs.vis == Hidden: + return nil # Create a stub symbol let val = addr c.mods[module].symCounter inc val[] @@ -1563,6 +1604,7 @@ type topLevel*: PNode # top level statements of the main module deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of logOps*: seq[LogEntry] + exports*: seq[ExportInfo] # Re-export info read from inline instructions module*: PSym # set by modulegraphs.nim! proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = @@ -1627,6 +1669,34 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] = t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module) #elif t.tagId == repClassTag: # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) + elif t.tagId == repExportTag: + t = next(s) # skip (repexport + if t.kind == StringLit: + result.exports.add ExportInfo(path: pool.strings[t.litId], kind: ExportIdx, names: @[]) + t = next(s) + if t.kind == ParRi: t = next(s) + elif t.tagId == repFromExportTag: + t = next(s) # skip (repfromexport + if t.kind == StringLit: + let path = pool.strings[t.litId] + t = next(s) + var names: seq[StrId] = @[] + while t.kind == Ident: + names.add t.litId + t = next(s) + result.exports.add ExportInfo(path: path, kind: FromexportIdx, names: names) + if t.kind == ParRi: t = next(s) + elif t.tagId == repExportExceptTag: + t = next(s) # skip (repexportexcept + if t.kind == StringLit: + let path = pool.strings[t.litId] + t = next(s) + var names: seq[StrId] = @[] + while t.kind == Ident: + names.add t.litId + t = next(s) + result.exports.add ExportInfo(path: path, kind: ExportexceptIdx, names: names) + if t.kind == ParRi: t = next(s) elif t.tagId == includeTag: t = skipTree(s) elif t.tagId == importTag: @@ -1649,14 +1719,11 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] = proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable; flags: set[LoadFlag] = {}): PrecompiledModule = - # Ensure module index is loaded - moduleId returns the FileIndex for this suffix + # Ensure module index is loaded - moduleId returns the FileIndex for this suffix let module = moduleId(c, string(suffix), flags) - # Populate interface tables from the NIF index structure - # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym - populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix)) - # Load the module AST (or just replay actions if loadFullAst is false) + # processTopLevel also collects export instructions let s = addr c.mods[module].stream var t = next(s[]) if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): @@ -1666,6 +1733,11 @@ proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHi else: result = PrecompiledModule(topLevel: newNode(nkStmtList)) + # Populate interface tables from the NIF index structure + # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym + # Use exports collected by processTopLevel + populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix), result.exports) + proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; flags: set[LoadFlag] = {}): PrecompiledModule = let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f)) diff --git a/koch.nim b/koch.nim index 0ab6320ba8..1f193bce40 100644 --- a/koch.nim +++ b/koch.nim @@ -16,7 +16,7 @@ const ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1 SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01" - NimonyStableCommit = "e5acb82d29d5b6a82ec83a6690a6d9ea32cf8976" # unversioned \ + NimonyStableCommit = "deb9b50c573fb55e071825ab55385e293b7216d5" # unversioned \ # Note that Nimony uses Nim as a git submodule but we don't want to install # Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive # is **required** here.