mirrors/Nim
1
0
Fork 0
mirror of https://github.com/nim-lang/Nim.git synced 2026-08-20 05:41:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

WIP

Browse Source
This commit is contained in:
araq
2026-01-14 12:31:44 +01:00
parent 1867f46569
commit 0b3a0b5527
2 changed files with 131 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
compiler/ast2nif.nim
View File

@@ -693,6 +693,9 @@ let repDeepCopyTag = registerTag("repdeepcopy")
let repEnumToStrTag = registerTag("repenumtostr")
let repMethodTag = registerTag("repmethod")
#let repClassTag = registerTag("repclass")
let repExportTag = registerTag("repexport")
let repFromExportTag = registerTag("repfromexport")
let repExportExceptTag = registerTag("repexportexcept")
let includeTag = registerTag("include")
let importTag = registerTag("import")
let implTag = registerTag("implementation")
@@ -730,6 +733,23 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) =
of GenericInstEntry:
discard "will only be written later to ensure it is materialized"
proc writeExports(w: var Writer; content: var TokenBuf) =
## Write export info as top-level processing instructions
for modIdx, names in w.exports:
let path = toFullPath(w.infos.config, modIdx)
if names.len == 0:
# Export all from this module: (repexport "path")
content.addParLe repExportTag, NoLineInfo
content.add strToken(pool.strings.getOrIncl(path), NoLineInfo)
content.addParRi()
else:
# Export specific symbols: (repfromexport "path" name1 name2 ...)
content.addParLe repFromExportTag, NoLineInfo
content.add strToken(pool.strings.getOrIncl(path), NoLineInfo)
for name in names:
content.add identToken(pool.strings.getOrIncl(name), NoLineInfo)
content.addParRi()
proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
opsLog: seq[LogEntry];
replayActions: seq[PNode] = @[]) =
@@ -749,6 +769,8 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
for op in opsLog:
if op.module == thisModule.int:
writeOp(w, content, op)
# Write export info as top-level instructions
writeExports(w, content)
var bottom = createTokenBuf(300)
w.writeToplevelNode content, bottom, n
@@ -784,10 +806,6 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode;
writeFile(dest, d)
let exportBuf = buildExportBuf(w)
createIndex(d, dest[0].info, false,
IndexSections(exportBuf: exportBuf))
# --------------------------- Loader (lazy!) -----------------------------------------------
proc nodeKind(n: Cursor): TNodeKind {.inline.} =
@@ -842,10 +860,15 @@ proc loadBool(n: var Cursor): bool =
raiseAssert "(true)/(false) expected"
type
ExportInfo* = object
path: string
kind: NifIndexKind
names: seq[StrId]
NifModule = ref object
stream: nifstreams.Stream
symCounter: int32
index: NifIndex
index: Table[string, NifIndexEntry] # Simple embedded index for offsets
suffix: string
DecodeContext* = object
@@ -871,25 +894,61 @@ type
LoadFlag* = enum
LoadFullAst, AlwaysLoadInterface
proc readEmbeddedIndex(s: var Stream): Table[string, NifIndexEntry] =
## Reads the simple embedded index (index (kv sym offset)...) from indexStartsAt position.
result = initTable[string, NifIndexEntry]()
let indexPos = indexStartsAt(s.r)
if indexPos <= 0:
return
let contentPos = offset(s.r) # Save position
s.r.jumpTo(indexPos)
var previousOffset = 0
var t = next(s)
let exportedTagId = pool.tags.getOrIncl("x")
if t.kind == ParLe and pool.tags[t.tagId] == ".index":
t = next(s)
while t.kind != EofToken and t.kind != ParRi:
if t.kind == ParLe:
let vis = if t.tagId == exportedTagId: Exported else: Hidden
let info = t.info
t = next(s) # skip (kv
var key = ""
if t.kind == Symbol:
key = pool.syms[t.symId]
elif t.kind == Ident:
key = pool.strings[t.litId]
t = next(s) # skip symbol
if t.kind == IntLit:
let offset = int(pool.integers[t.intId]) + previousOffset
result[key] = NifIndexEntry(offset: offset, info: info, vis: vis)
previousOffset = offset
t = next(s) # skip offset
if t.kind == ParRi:
t = next(s) # skip )
else:
t = next(s)
s.r.jumpTo(contentPos) # Restore position
proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): FileIndex =
var isKnownFile = false
result = c.infos.config.registerNifSuffix(suffix, isKnownFile)
if not isKnownFile or AlwaysLoadInterface in flags:
let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string
let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.idx.nif")).string
if not fileExists(modFile):
raiseAssert "NIF file not found for module suffix '" & suffix & "': " & modFile &
". This can happen when loading a module from NIF that references another module " &
"whose NIF file hasn't been written yet."
c.mods[result] = NifModule(stream: nifstreams.open(modFile), index: readIndex(idxFile), suffix: suffix)
var stream = nifstreams.open(modFile)
let index = readEmbeddedIndex(stream)
c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix)
proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry =
let ii = addr c.mods[module].index
result = ii.public.getOrDefault(nifName)
result = ii[].getOrDefault(nifName)
if result.offset == 0:
result = ii.private.getOrDefault(nifName)
if result.offset == 0:
raiseAssert "symbol has no offset: " & nifName
raiseAssert "symbol has no offset: " & nifName
proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string;
localSyms: var Table[string, PSym]): PNode
@@ -1393,18 +1452,17 @@ proc extractBasename(nifName: string): string =
result.add c
proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex;
interf, interfHidden: var TStrTable; thisModule: string) =
interf, interfHidden: var TStrTable; thisModule: string;
exports: seq[ExportInfo]) =
## Populates interface tables from the NIF index structure.
## Uses the index's public/private tables instead of traversing AST.
## Uses the simple embedded index for offsets, exports passed from processTopLevel.
# Move the public table and exports list out to avoid iterator invalidation
# Move the index table out to avoid iterator invalidation
# (moduleId can add to c.mods which would invalidate Table iterators)
# We move them back after iteration.
var publicTab = move c.mods[module].index.public
var exportsList = move c.mods[module].index.exports
var indexTab = move c.mods[module].index
# Add all public symbols to interf (exported interface) and interfHidden
for nifName, entry in publicTab:
# Add all symbols to interf (exported interface) and interfHidden
for nifName, entry in indexTab:
if not nifName.startsWith("`t"):
# do not load types, they are not part of an interface but an implementation detail!
#echo "LOADING SYM ", nifName, " ", entry.offset
@@ -1413,33 +1471,32 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex;
strTableAdd(interf, sym)
strTableAdd(interfHidden, sym)
# Move public table back
c.mods[module].index.public = move publicTab
# Move index table back
c.mods[module].index = move indexTab
# Process exports (re-exports from other modules)
for exp in exportsList:
let (path, kind, names) = exp
for exp in exports:
# Convert path to module suffix
let expSuffix = moduleSuffix(path, cast[seq[string]](c.infos.config.searchPaths))
let expSuffix = moduleSuffix(exp.path, cast[seq[string]](c.infos.config.searchPaths))
# Load the exported module's index
let expModule = moduleId(c, expSuffix)
# Move the exported module's public table out to avoid iterator invalidation
var expPublicTab = move c.mods[expModule].index.public
# Move the exported module's index table out to avoid iterator invalidation
var expIndexTab = move c.mods[expModule].index
# Build a set of names for filtering
var nameSet = initHashSet[string]()
for nameId in names:
for nameId in exp.names:
nameSet.incl pool.strings[nameId]
# Add symbols based on export kind
for nifName, entry in expPublicTab:
for nifName, entry in expIndexTab:
if nifName.startsWith("`t"):
continue # skip types
let basename = extractBasename(nifName)
let shouldInclude =
case kind
case exp.kind
of ExportIdx: true # export all
of FromexportIdx: basename in nameSet # only specific names
of ExportexceptIdx: basename notin nameSet # all except specific names
@@ -1448,30 +1505,17 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex;
if shouldInclude:
let sym = loadSymFromIndexEntry(c, expModule, nifName, entry, expSuffix)
if sym != nil:
strTableAdd(interf, sym)
if entry.vis == Exported:
strTableAdd(interf, sym)
strTableAdd(interfHidden, sym)
# Move exported module's public table back
c.mods[expModule].index.public = move expPublicTab
# Move exports list back
c.mods[module].index.exports = move exportsList
when false:
# Add private symbols to interfHidden only
for nifName, entry in idx.private:
let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule)
if sym != nil:
strTableAdd(interfHidden, sym)
# Move exported module's index table back
c.mods[expModule].index = move expIndexTab
proc toNifFilename*(conf: ConfigRef; f: FileIndex): string =
let suffix = moduleSuffix(conf, f)
result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string
proc toNifIndexFilename*(conf: ConfigRef; f: FileIndex): string =
let suffix = moduleSuffix(conf, f)
result = toGeneratedFile(conf, AbsoluteFile(suffix), ".s.idx.nif").string
proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: bool): PSym =
result = c.syms.getOrDefault(symAsStr)[0]
if result != nil:
@@ -1482,14 +1526,11 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo
return nil # Local symbols shouldn't be hooks
let module = moduleId(c, sn.module)
# Look up the symbol in the module's index
var offs = c.mods[module].index.public.getOrDefault(symAsStr)
var offs = c.mods[module].index.getOrDefault(symAsStr)
if offs.offset == 0:
if alsoConsiderPrivate:
offs = c.mods[module].index.private.getOrDefault(symAsStr)
if offs.offset == 0:
return nil
else:
return nil
return nil
if not alsoConsiderPrivate and offs.vis == Hidden:
return nil
# Create a stub symbol
let val = addr c.mods[module].symCounter
inc val[]
@@ -1563,6 +1604,7 @@ type
topLevel*: PNode # top level statements of the main module
deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of
logOps*: seq[LogEntry]
exports*: seq[ExportInfo] # Re-export info read from inline instructions
module*: PSym # set by modulegraphs.nim!
proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) =
@@ -1627,6 +1669,34 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] =
t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module)
#elif t.tagId == repClassTag:
# t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module)
elif t.tagId == repExportTag:
t = next(s) # skip (repexport
if t.kind == StringLit:
result.exports.add ExportInfo(path: pool.strings[t.litId], kind: ExportIdx, names: @[])
t = next(s)
if t.kind == ParRi: t = next(s)
elif t.tagId == repFromExportTag:
t = next(s) # skip (repfromexport
if t.kind == StringLit:
let path = pool.strings[t.litId]
t = next(s)
var names: seq[StrId] = @[]
while t.kind == Ident:
names.add t.litId
t = next(s)
result.exports.add ExportInfo(path: path, kind: FromexportIdx, names: names)
if t.kind == ParRi: t = next(s)
elif t.tagId == repExportExceptTag:
t = next(s) # skip (repexportexcept
if t.kind == StringLit:
let path = pool.strings[t.litId]
t = next(s)
var names: seq[StrId] = @[]
while t.kind == Ident:
names.add t.litId
t = next(s)
result.exports.add ExportInfo(path: path, kind: ExportexceptIdx, names: names)
if t.kind == ParRi: t = next(s)
elif t.tagId == includeTag:
t = skipTree(s)
elif t.tagId == importTag:
@@ -1649,14 +1719,11 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] =
proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable;
flags: set[LoadFlag] = {}): PrecompiledModule =
# Ensure module index is loaded - moduleId returns the FileIndex for this suffix
# Ensure module index is loaded - moduleId returns the FileIndex for this suffix
let module = moduleId(c, string(suffix), flags)
# Populate interface tables from the NIF index structure
# Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym
populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix))
# Load the module AST (or just replay actions if loadFullAst is false)
# processTopLevel also collects export instructions
let s = addr c.mods[module].stream
var t = next(s[])
if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList):
@@ -1666,6 +1733,11 @@ proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHi
else:
result = PrecompiledModule(topLevel: newNode(nkStmtList))
# Populate interface tables from the NIF index structure
# Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym
# Use exports collected by processTopLevel
populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix), result.exports)
proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable;
flags: set[LoadFlag] = {}): PrecompiledModule =
let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f))

2
koch.nim
View File

@@ -16,7 +16,7 @@ const
ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1
SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01"
NimonyStableCommit = "e5acb82d29d5b6a82ec83a6690a6d9ea32cf8976" # unversioned \
NimonyStableCommit = "deb9b50c573fb55e071825ab55385e293b7216d5" # unversioned \
# Note that Nimony uses Nim as a git submodule but we don't want to install
# Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive
# is **required** here.