From fa4d79f51994fb33b08480591b32e0c16e317c2b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 7 Dec 2025 13:07:44 +0100 Subject: [PATCH] IC: progress (#25339) --- compiler/ast2nif.nim | 103 +++++++++++++++++++++++++++------- compiler/ccgliterals.nim | 11 ++-- compiler/ccgtypes.nim | 2 +- compiler/main.nim | 1 + compiler/modulegraphs.nim | 31 +++++++++- compiler/nifbackend.nim | 48 ++++++++-------- compiler/nim.nim | 3 +- lib/system/countbits_impl.nim | 6 -- lib/system/sets.nim | 7 +++ 9 files changed, 150 insertions(+), 62 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index d7ed56b87f..b5f229d94a 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -211,7 +211,7 @@ proc toNifSymName(w: var Writer; sym: PSym): string = result = sym.name.s result.add '.' result.addInt sym.disamb - if sym.itemId notin w.locals and sym.kindImpl notin skLocalSymKinds: + if sym.kindImpl notin skLocalSymKinds and sym.itemId notin w.locals: # Global symbol: ident.disamb.moduleSuffix let module = sym.itemId.module result.add '.' @@ -406,13 +406,13 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addParRi # Collect for later unloading after entire module is written - if sym.kindImpl notin {skModule, skPackage}: + if sym.kindImpl notin {skPackage}: # do not unload modules w.writtenSyms.add sym proc shouldWriteSymDef(w: Writer; sym: PSym): bool {.inline.} = # Don't write module/package symbols - they don't have NIF files - if sym.kindImpl in {skModule, skPackage}: + if sym.kindImpl in {skPackage}: return false # Already written - don't write again if sym.state == Sealed: @@ -430,7 +430,7 @@ proc shouldWriteSymDef(w: Writer; sym: PSym): bool {.inline.} = proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = if sym == nil: dest.addDotToken() - elif sym.kindImpl in {skModule, skPackage}: + elif sym.kindImpl in {skPackage}: # Write module/package symbols as dots - they're resolved differently # (by position/FileIndex, not by NIF lookup) dest.addDotToken() @@ -1084,6 +1084,8 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: if s.kindImpl == skModule: expect n, DotToken inc n + var isKnownFile = false + s.positionImpl = int c.infos.config.registerNifSuffix(thisModule, isKnownFile) else: loadField s.positionImpl @@ -1097,11 +1099,10 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: # Load the AST for routine symbols (procs, funcs, etc.) if s.kindImpl in routineKinds: s.astImpl = loadNode(c, n, thisModule, localSyms) + elif n.kind == DotToken: + inc n else: - if n.kind == DotToken: - inc n - else: - raiseAssert "expected '.' for non-routine symbol AST but got " & $n.kind + raiseAssert "expected '.' for non-routine symbol AST but got " & $n.kind loadLoc c, n, s.locImpl s.constraintImpl = loadNode(c, n, thisModule, localSyms) s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms) @@ -1420,10 +1421,37 @@ proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym = disamb: sn.count.int32, state: Partial) c.syms[id] = (result, offs) +proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym = + ## Tries to resolve a compiler proc from a module by checking the NIF index. + ## Returns nil if the symbol doesn't exist. + let suffix = moduleSuffix(c.infos.config, moduleFileIdx) + let symName = name & ".0." & suffix + + # Check if module index is loaded, if not load it + let module = moduleId(c, suffix) + + # Check if symbol exists in the index (check both public and private) + var offs = c.mods[module].index.public.getOrDefault(symName) + if offs.offset == 0: + offs = c.mods[module].index.private.getOrDefault(symName) + if offs.offset == 0: + return nil + + # Create a stub symbol + let val = addr c.mods[module].symCounter + inc val[] + let id = ItemId(module: int32(module), item: val[]) + result = c.syms.getOrDefault(id)[0] + if result == nil: + result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(name), + disamb: 0, state: Partial) + c.syms[id] = (result, offs) + proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; hooks: var Table[nifstreams.SymId, HooksPerType]; converters: var seq[(string, string)]; - classes: var seq[ClassIndexEntry]): PNode = + classes: var seq[ClassIndexEntry]; + loadFullAst: bool = false): PNode = let suffix = moduleSuffix(c.infos.config, f) # Ensure module index is loaded - moduleId returns the FileIndex for this suffix @@ -1440,29 +1468,62 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va # Return classes/methods from the index classes = move c.mods[module].index.classes - # Check for replay actions at the start of the NIF file + # Load the module AST (or just replay actions if loadFullAst is false) result = newNode(nkStmtList) let s = addr c.mods[module].stream s.r.jumpTo 0 # Start from beginning discard processDirectives(s.r) var localSyms = initTable[string, PSym]() - # Read root stmts node var t = next(s[]) if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): t = next(s[]) # skip flags t = next(s[]) # skip type - # Check if first node is a (replay ...) container - if t.kind == ParLe and pool.tags[t.tagId] == "replay": - t = next(s[]) # move past (replay - # Parse all replay actions inside the container - while t.kind != ParRi and t.kind != EofToken: - if t.kind == ParLe: + # Process all top-level statements + while t.kind != ParRi and t.kind != EofToken: + if t.kind == ParLe: + let tag = pool.tags[t.tagId] + if tag == "replay": + # Always load replay actions (macro cache operations) + t = next(s[]) # move past (replay + while t.kind != ParRi and t.kind != EofToken: + if t.kind == ParLe: + var buf = createTokenBuf(50) + nifcursors.parse(s[], buf, t.info) + var cursor = cursorAt(buf, 0) + let replayNode = loadNode(c, cursor, suffix, localSyms) + if replayNode != nil: + result.sons.add replayNode + t = next(s[]) + elif loadFullAst: + # Parse the full statement var buf = createTokenBuf(50) - nifcursors.parse(s[], buf, t.info) + buf.add t # Add the ParLe token we already read + var nested = 1 + while nested > 0: + t = next(s[]) + buf.add t + if t.kind == ParLe: + inc nested + elif t.kind == ParRi: + dec nested + elif t.kind == EofToken: + break var cursor = cursorAt(buf, 0) - let replayNode = loadNode(c, cursor, suffix, localSyms) - if replayNode != nil: - result.sons.add replayNode + let stmtNode = loadNode(c, cursor, suffix, localSyms) + if stmtNode != nil: + result.sons.add stmtNode + else: + # Skip over the statement by counting parentheses + var nested = 1 + while nested > 0: + t = next(s[]) + if t.kind == ParLe: + inc nested + elif t.kind == ParRi: + dec nested + elif t.kind == EofToken: + break + else: t = next(s[]) when isMainModule: diff --git a/compiler/ccgliterals.nim b/compiler/ccgliterals.nim index 84f72017cf..069ed48df7 100644 --- a/compiler/ccgliterals.nim +++ b/compiler/ccgliterals.nim @@ -16,13 +16,10 @@ ## implementation. template detectVersion(field, corename) = - if m.g.field == 0: - let core = getCompilerProc(m.g.graph, corename) - if core == nil or core.kind != skConst: - m.g.field = 1 - else: - m.g.field = toInt(ast.getInt(core.astdef)) - result = m.g.field + if m.g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc, gcHooks}: + result = 2 + else: + result = 1 proc detectStrVersion(m: BModule): int = detectVersion(strVersion, "nimStrVersion") diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 76669d41ba..a2b5e32cb8 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -721,7 +721,7 @@ proc genRecordFieldsAux(m: BModule; n: PNode, # have to recurse via 'getTypeDescAux'. And not doing so prevents problems # with heavily templatized C++ code: if not isImportedCppType(rectype): - let fieldType = field.loc.lode.typ.skipTypes(abstractInst) + let fieldType = field.loc.t.skipTypes(abstractInst) var typ: Rope = "" var isFlexArray = false var initializer = "" diff --git a/compiler/main.nim b/compiler/main.nim index 8aecccc488..d63c7d3fbf 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -447,6 +447,7 @@ proc mainCommand*(graph: ModuleGraph) = of cmdNifC: # Generate C code from NIF files wantMainModule(conf) + setOutFile(conf) commandNifC(graph) of cmdDeps: # Generate .build.nif for nifmake diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index afb98d67c2..b52ee9f4f0 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -435,7 +435,30 @@ proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) = proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = result = nil - if g.config.symbolFiles == disabledSf: return nil + if g.config.symbolFiles == disabledSf: + # For NIF-based compilation, search in loaded NIF modules + when not defined(nimKochBootstrap): + # Only try to resolve from NIF if we're actually using NIF files (cmdNifC) + if g.config.cmd == cmdNifC: + # First try system module (most compilerprocs are there) + let systemFileIdx = g.config.m.systemFileIdx + if systemFileIdx != InvalidFileIdx: + result = tryResolveCompilerProc(ast.program, name, systemFileIdx) + if result != nil: + strTableAdd(g.compilerprocs, result) + return result + + # Try threadpool module (some compilerprocs like FlowVar are there) + # Find threadpool module by searching loaded modules + for moduleIdx in 0.. 0: let fileIdx = stack.pop() - if visited.containsOrIncl(int(fileIdx)): - continue - - # Load module from NIF - let module = moduleFromNifFile(g, fileIdx, cachedModules) - if module == nil: - continue - - modules.add module - - # Add dependencies to stack (they come from cachedModules) - for dep in cachedModules: - if not visited.contains(int(dep)): - stack.add dep - cachedModules.setLen(0) - - result = modules + if not visited.containsOrIncl(int(fileIdx)): + # Only load full AST for main module; others are loaded lazily by codegen + let isMainModule = fileIdx == mainFileIdx + let module = moduleFromNifFile(g, fileIdx, cachedModules, loadFullAst=isMainModule) + if module != nil: + result.add module + # Add dependencies to stack (they come from cachedModules) + for dep in cachedModules: + if not visited.contains(int(dep)): + stack.add dep + cachedModules.setLen(0) proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule = ## Set up a BModule for code generation from a NIF module. @@ -71,8 +65,10 @@ proc generateCodeForModule(g: ModuleGraph; module: PSym) = if module.ast != nil: cgen.genTopLevelStmt(bmod, module.ast) - # Finalize the module - finalCodegenActions(g, bmod, newNodeI(nkStmtList, module.info)) + # Finalize the module (this adds it to modulesClosed) + # Create an empty stmt list as the init body - genInitCode in writeModule will set it up properly + let initStmt = newNodeI(nkStmtList, module.info) + finalCodegenActions(g, bmod, initStmt) # Generate dispatcher methods for disp in getDispatchers(g): @@ -84,6 +80,14 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = # Reset backend state resetForBackend(g) + let mainModule = g.getModule(mainFileIdx) + + # Also ensure system module is set up and generated if it exists + if g.systemModule != nil and g.systemModule != mainModule: + let systemBmod = BModuleList(g.backend).modules[g.systemModule.position] + if systemBmod == nil: + discard setupNifBackendModule(g, g.systemModule) + generateCodeForModule(g, g.systemModule) # Load all modules in dependency order using stack traversal let modules = loadModuleDependencies(g, mainFileIdx) @@ -92,12 +96,12 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = "Cannot load NIF file for main module: " & toFullPath(g.config, mainFileIdx)) return - # Set up backend modules + # Set up backend modules for all modules that need code generation for module in modules: discard setupNifBackendModule(g, module) # Generate code for all modules except main (main goes last) - let mainModule = g.getModule(mainFileIdx) + # This ensures all modules are added to modulesClosed for module in modules: if module != mainModule: generateCodeForModule(g, module) diff --git a/compiler/nim.nim b/compiler/nim.nim index 319844c62d..72302a186e 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -118,8 +118,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if conf.selectedGC == gcUnselected: if conf.backend in {backendC, backendCpp, backendObjc} or (conf.cmd in cmdDocLike and conf.backend != backendJs) or - conf.cmd == cmdGendepend or - conf.cmd == cmdM: + conf.cmd in {cmdGendepend, cmdNifC, cmdDeps, cmdM}: initOrcDefines(conf) mainCommand(graph) diff --git a/lib/system/countbits_impl.nim b/lib/system/countbits_impl.nim index 34969cb328..d16f06ae79 100644 --- a/lib/system/countbits_impl.nim +++ b/lib/system/countbits_impl.nim @@ -85,9 +85,3 @@ func countSetBitsImpl*(x: SomeInteger): int {.inline.} = else: when sizeof(x) <= 4: result = countBitsImpl(x.uint32) else: result = countBitsImpl(x.uint64) - -proc countBits32*(n: uint32): int {.compilerproc, inline.} = - result = countSetBitsImpl(n) - -proc countBits64*(n: uint64): int {.compilerproc, inline.} = - result = countSetBitsImpl(n) diff --git a/lib/system/sets.nim b/lib/system/sets.nim index 97431c2964..d3b054c0ed 100644 --- a/lib/system/sets.nim +++ b/lib/system/sets.nim @@ -10,6 +10,13 @@ # set handling +# IC: compilerprocs now must be defined in system.nim or threadpool.nim! +proc countBits32*(n: uint32): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) + +proc countBits64*(n: uint64): int {.compilerproc, inline.} = + result = countSetBitsImpl(n) + proc cardSetImpl(s: ptr UncheckedArray[uint8], len: int): int {.inline.} = var i = 0 result = 0