From 4d0663096cb4dd41dc4aa18d935df03d3a567f22 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Tue, 9 Jun 2026 20:46:48 +0800 Subject: [PATCH 01/46] Revert "fixes #22122; raise effects for complex expressions" (#25888) Reverts nim-lang/Nim#25845 ```nim case ecode of ECONNABORTED, EPERM, ETIMEDOUT, ENOTCONN: getConnectionAbortedError(ecode) of EMFILE, ENFILE, ENOBUFS, ENOMEM: getTransportTooManyError(ecode) else: (ref TransportOsError)(code: ecode, msg: "(" & $int(ecode) & ") " & osErrorMsg(ecode)) ``` The compiler inserts a hidden conv for the case expression. Perhaps we can skip hidden convs to inspect the types that are actually raised --- compiler/sempass2.nim | 22 +--------- tests/effects/tcase_raises.nim | 74 ---------------------------------- 2 files changed, 1 insertion(+), 95 deletions(-) delete mode 100644 tests/effects/tcase_raises.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 75ad510b1a..7b2be510f9 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -497,26 +497,6 @@ proc addRaiseEffect(a: PEffects, e, comesFrom: PNode) = if not isDefectException(e.typ): throws(a.exc, e, comesFrom) -proc addRaiseEffectsFromExpr(a: PEffects, e, comesFrom: PNode) = - if e.isNil: - return - let x = skipConvCastAndClosure(e) - case x.kind - of nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr: - if x.len > 0: - addRaiseEffectsFromExpr(a, x.lastSon, comesFrom) - of nkIfExpr, nkIfStmt: - for branch in items(x): - if branch.len > 0: - addRaiseEffectsFromExpr(a, branch.lastSon, comesFrom) - of nkCaseStmt: - for i in 1.. 0: - addRaiseEffectsFromExpr(a, branch.lastSon, comesFrom) - else: - addRaiseEffect(a, x, x) - proc addTag(a: PEffects, e, comesFrom: PNode) = var aa = a.tags for i in 0.. Date: Wed, 10 Jun 2026 03:55:30 +0900 Subject: [PATCH 02/46] adds `modifierMode` parameter to typeof (#25815) This PR adds 3 modes to `typeof` to specify how to handle type modifiers `var`, `sink` and `lent`. - typeOfModCompatible Remove or keep type modifiers in the same way as old typeof. That means keep `sink` but remove `var` and `lent`. - typeOfModRemoveModifier Remove type modifiers. - typeOfModKeepModifier Keep type modifiers. Related to https://github.com/nim-lang/Nim/pull/25779 https://github.com/nim-lang/Nim/issues/25786 --- changelog.md | 2 + compiler/semexprs.nim | 4 +- compiler/semmagic.nim | 11 +---- compiler/semtypes.nim | 62 ++++++++++++++++++++++----- lib/system.nim | 12 +++++- tests/system/ttypeof.nim | 90 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 159 insertions(+), 22 deletions(-) create mode 100644 tests/system/ttypeof.nim diff --git a/changelog.md b/changelog.md index ce7596d5b8..1790684126 100644 --- a/changelog.md +++ b/changelog.md @@ -72,6 +72,8 @@ parameter and result types, not just their source-level shape. Use - `std/nre2` is added to replace deprecated NRE. +- `system.typeof` adds a new parameter `modifierMode` to specify how type modifiers are handled. + [//]: # "Changes:" - `std/math` The `^` symbol now supports floating-point as exponent in addition to the Natural type. diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index aa0489cd22..3712109bcb 100644 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -106,7 +106,9 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = result = semExprCheck(c, n, flags) - if result.typ == nil: + if result.typ == nil and efInTypeof in flags: + result.typ = c.voidType + elif result.typ == nil: localError(c.config, n.info, errExprXHasNoType % renderTree(result, {renderNoComments})) result.typ = errorType(c) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 397c08bf67..f5bf97c580 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -43,17 +43,8 @@ proc semAddr(c: PContext; n: PNode): PNode = result.typ = makePtrType(c, x.typ.skipTypes({tySink})) proc semTypeOf(c: PContext; n: PNode): PNode = - var m = BiggestInt 1 # typeOfIter - if n.len == 3: - let mode = semConstExpr(c, n[2]) - if mode.kind != nkIntLit: - localError(c.config, n.info, "typeof: cannot evaluate 'mode' parameter at compile-time") - else: - m = mode.intVal + let typExpr = semTypeOfImpl(c, n) result = newNodeI(nkTypeOfExpr, n.info) - inc c.inTypeofContext - defer: dec c.inTypeofContext # compiles can raise an exception - let typExpr = semExprWithType(c, n[1], if m == 1: {efInTypeof} else: {}) result.add typExpr if typExpr.typ.kind == tyFromExpr: typExpr.typ.incl tfNonConstExpr diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 8009f7293c..e90811ad9a 100644 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -2063,6 +2063,57 @@ proc semStaticType(c: PContext, childNode: PNode, prev: PType): PType = result.rawAddSon(base) result.incl tfHasStatic +proc semTypeOfImpl(c: PContext; n: PNode): PNode = + var m = BiggestInt 1 # typeOfIter + var modifierMode = BiggestInt 0 # CompatibleTypeModifiers + type + TypeOfParams = enum + topMode + topModifier + if n.len in 3 .. 4: + for i in 2 ..< n.len: + var argKind = topMode + var arg: PNode = nil + if n[i].kind == nkExprEqExpr and n[i][0].kind == nkIdent: + # named param + case n[i][0].ident.s + of "mode": argKind = topMode + of "modifierMode": argKind = topModifier + else: + localError(c.config, n.info, "typeof: got unknown parameter name") + arg = n[i][1] + else: + if i == 2: + argKind = topMode + else: + argKind = topModifier + arg = n[i] + case argKind + of topMode: + let mode = semConstExpr(c, arg) + if mode.kind != nkIntLit: + localError(c.config, n.info, "typeof: cannot evaluate 'mode' parameter at compile-time") + else: + m = mode.intVal + of topModifier: + let modMode = semConstExpr(c, arg) + if modMode.kind != nkIntLit: + localError(c.config, n.info, "typeof: cannot evaluate 'modifierMode' parameter at compile-time") + else: + modifierMode = modMode.intVal + + inc c.inTypeofContext + defer: dec c.inTypeofContext # compiles can raise an exception + var typExpr = semExprNoDeref(c, n[1], if m == 1: {efInTypeof} else: {}) + if modifierMode == 0: + # CompatibleTypeModifiers + typExpr.typ = typExpr.typ.skipTypes({tyVar, tyLent}) + elif modifierMode == 1: + # RemoveTypeModifiers + typExpr.typ = typExpr.typ.skipTypes({tyVar, tyLent, tySink}) + + result = typExpr + proc semTypeOf(c: PContext; n: PNode; prev: PType): PType = openScope(c) inc c.inTypeofContext @@ -2083,16 +2134,7 @@ proc semTypeOf(c: PContext; n: PNode; prev: PType): PType = proc semTypeOf2(c: PContext; n: PNode; prev: PType): PType = openScope(c) - var m = BiggestInt 1 # typeOfIter - if n.len == 3: - let mode = semConstExpr(c, n[2]) - if mode.kind != nkIntLit: - localError(c.config, n.info, "typeof: cannot evaluate 'mode' parameter at compile-time") - else: - m = mode.intVal - inc c.inTypeofContext - defer: dec c.inTypeofContext # compiles can raise an exception - let ex = semExprWithType(c, n[1], if m == 1: {efInTypeof} else: {}) + let ex = semTypeOfImpl(c, n) closeScope(c) result = ex.typ if result.kind == tyFromExpr: diff --git a/lib/system.nim b/lib/system.nim index 26232971bd..44385e1772 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -54,7 +54,12 @@ type typeOfProc, ## Prefer the interpretation that means `x` is a proc call. typeOfIter ## Prefer the interpretation that means `x` is an iterator call. -proc typeof*(x: untyped; mode = typeOfIter): typedesc {. + TypeOfModifiers* = enum ## Modes to handle type modifiers `var`, `sink` and `lent`. + CompatibleTypeModifiers, ## Remove or keep type modifiers in the same way as old typeof. That means keep `sink` but remove `var` and `lent`. + RemoveTypeModifiers, ## Remove type modifiers. + KeepTypeModifiers, ## Keep type modifiers. + +proc typeof*(x: untyped; mode = typeOfIter; modifierMode = CompatibleTypeModifiers): typedesc {. magic: "TypeOf", noSideEffect, compileTime.} = ## Builtin `typeof` operation for accessing the type of an expression. ## Since version 0.20.0. @@ -76,6 +81,11 @@ proc typeof*(x: untyped; mode = typeOfIter): typedesc {. # since `typeOfProc` expects a typed expression and `myFoo2()` can # only be used in a `for` context. + proc varParam(x: var int; + y: typeof(x, modifierMode = RemoveTypeModifiers); + z: typeof(x, modifierMode = KeepTypeModifiers)) = discard + doAssert varParam is proc (x: var int; y: int; z: var int) {.nimcall.} + proc `or`*(a, b: typedesc): typedesc {.magic: "TypeTrait", noSideEffect.} ## Constructs an `or` meta class. diff --git a/tests/system/ttypeof.nim b/tests/system/ttypeof.nim new file mode 100644 index 0000000000..93abefafbb --- /dev/null +++ b/tests/system/ttypeof.nim @@ -0,0 +1,90 @@ +static: doAssert typeof(1) is int + +func isVar[T](x: var T): bool = true +func isVar[T](x: T): bool = false + +proc testVarParams1(a: var int; + b: typeof(a); + c: typeof(a, typeOfIter); + d: typeof(a, typeOfIter, CompatibleTypeModifiers); + e: typeof(a, typeOfIter, RemoveTypeModifiers); + f: typeof(a, typeOfIter, KeepTypeModifiers); + g: typeof(a, modifierMode = CompatibleTypeModifiers); + h: typeof(a, modifierMode = RemoveTypeModifiers); + i: typeof(a, modifierMode = KeepTypeModifiers); + ) = + doAssert not isVar(b) + doAssert not isVar(c) + doAssert not isVar(d) + doAssert not isVar(e) + doAssert isVar(f) + doAssert not isVar(g) + doAssert not isVar(h) + doAssert isVar(i) + +static: doAssert testVarParams1 is proc (a: var int; b: int; c: int; d: int; e: int; f: var int; g: int; h: int; i: var int) {.nimcall.} + +block: + var a, f, i: int + testVarParams1(a, 0, 0, 0, 0, f, 0, 0, i) + +# `CompatibleTypeModifiers` and `RemoveTypeModifiers` remove only top `var`, not `var` inside proc type +proc testVarParams2(a: var proc(x: var int): var int; + b: typeof(a); + c: typeof(a, modifierMode = CompatibleTypeModifiers); + d: typeof(a, modifierMode = RemoveTypeModifiers); + e: typeof(a, modifierMode = KeepTypeModifiers)) = + doAssert not isVar(b) + doAssert not isVar(c) + doAssert not isVar(d) + doAssert isVar(e) + +static: doAssert testVarParams2 is proc (a: var proc(x: var int): var int; + b: proc(x: var int): var int; + c: proc(x: var int): var int; + d: proc(x: var int): var int; + e: var proc(x: var int): var int) {.nimcall.} + +block: + var a, e: proc(x: var int): var int = nil + let b, c, d: proc(x: var int): var int = nil + testVarParams2(a, b, c, d, e) + +proc testRet(a: var int): typeof(a) = 0 +static: doAssert testRet is proc (a: var int): int {.nimcall.} +proc testRet2(a: var int): typeof(a, modifierMode = CompatibleTypeModifiers) = 0 +static: doAssert testRet2 is proc (a: var int): int {.nimcall.} +proc testRet3(a: var int): typeof(a, modifierMode = RemoveTypeModifiers) = 0 +static: doAssert testRet3 is proc (a: var int): int {.nimcall.} + +proc fooSink1(a: sink string; + b: typeof(a); + c: typeof(a, modifierMode = CompatibleTypeModifiers); + d: typeof(a, modifierMode = RemoveTypeModifiers); + e: typeof(a, modifierMode = KeepTypeModifiers)) = discard + +static: doAssert fooSink1 is proc (a: sink string; b: sink string; c: sink string; d: string; e: sink string) {.nimcall.} + +proc fooLentRet(a: seq[string]): lent string = a[0] +proc testLentRetComp(a: seq[string]): typeof(fooLentRet(a), modifierMode = CompatibleTypeModifiers) = a[0] +proc testLentRetRemo(a: seq[string]): typeof(fooLentRet(a), modifierMode = RemoveTypeModifiers) = a[0] +proc testLentRetKeep(a: seq[string]): typeof(fooLentRet(a), modifierMode = KeepTypeModifiers) = a[0] + +# workaround # issue 25830 +proc dummyLentProc(a: seq[string]): lent string = a[0] + +static: + doAssert testLentRetComp is proc (a: seq[string]): string {.nimcall.} + doAssert testLentRetRemo is proc (a: seq[string]): string {.nimcall.} + doAssert testLentRetKeep is typeof(dummyLentProc) + +proc voidProc() = discard +static: + doAssert typeof(voidProc()) is void + +type + Foo = typeof(Bar) + Bar = int + +static: + doAssert Foo is int From f5c43ad759a4310aa94327dd095eb79d4f39b08f Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 11 Jun 2026 13:07:10 +0800 Subject: [PATCH 03/46] closes #25885; adds a test case (#25892) closes #25885 --- tests/effects/teffectsmisc.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/effects/teffectsmisc.nim b/tests/effects/teffectsmisc.nim index e2fd2d87db..d3978814c3 100644 --- a/tests/effects/teffectsmisc.nim +++ b/tests/effects/teffectsmisc.nim @@ -57,4 +57,8 @@ block: except IOError as e: raise - f() \ No newline at end of file + f() + +block: + static: doAssert IOError is Exception + proc r(e: ref Exception) {.raises: [IOError].} = raise (ref IOError)(e) From 07685f79e047e431bc33003f8898da2fa387b0f3 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:15:30 +0800 Subject: [PATCH 04/46] implements fallback memfiles on Nintendoswitch (#25891) fix hightlies failures --- lib/pure/memfiles.nim | 289 +++++++++++++++++++++++++----------- tests/stdlib/tmemfiles2.nim | 1 + 2 files changed, 202 insertions(+), 88 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 8e2f61868e..b195e102b4 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -15,12 +15,16 @@ ## It also provides some fast iterators over lines in text files (or ## other "line-like", variable length, delimited records). +const + nimUseFallBack = defined(nintendoswitch) or defined(nimMemfileFallback) + when defined(windows): import std/winlean when defined(nimPreviewSlimSystem): import std/widestrs elif defined(posix): - import std/posix + when not nimUseFallBack: + import std/posix else: {.error: "the memfiles module is not supported on your operating system!".} @@ -29,45 +33,48 @@ import std/oserrors when defined(nimPreviewSlimSystem): import std/[syncio, assertions] +elif nimUseFallBack: + import std/syncio from system/ansi_c import c_memchr proc newEIO(msg: string): ref IOError = result = (ref IOError)(msg: msg) -proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode = - ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1, - ## allocating | freeing space from the file system. This routine returns the - ## last OSErrorCode found rather than raising to support old rollback/clean-up - ## code style. [ Should maybe move to std/osfiles. ] - result = OSErrorCode(0) - if newFileSize < 0 or newFileSize == oldSize: - return result - when defined(windows): - var sizeHigh = int32(newFileSize shr 32) - let sizeLow = int32(newFileSize and 0xffffffff) - let status = setFilePointer(Handle fh, sizeLow, addr(sizeHigh), FILE_BEGIN) - let lastErr = osLastError() - if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or - setEndOfFile(Handle fh) == 0: - result = lastErr - else: - if newFileSize > oldSize: # grow the file - var e: cint = cint(0) # posix_fallocate truncates up when needed. - when declared(posix_fallocate): - while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): - discard - if e == EINVAL or e == EOPNOTSUPP or e == ENOSYS: - # fallback arguable; Most portable BUT allows SEGV - if ftruncate(fh, newFileSize) == -1: +when not nimUseFallBack: + proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode = + ## Set the size of open file pointed to by `fh` to `newFileSize` if != -1, + ## allocating | freeing space from the file system. This routine returns the + ## last OSErrorCode found rather than raising to support old rollback/clean-up + ## code style. [ Should maybe move to std/osfiles. ] + result = OSErrorCode(0) + if newFileSize < 0 or newFileSize == oldSize: + return result + when defined(windows): + var sizeHigh = int32(newFileSize shr 32) + let sizeLow = int32(newFileSize and 0xffffffff) + let status = setFilePointer(Handle fh, sizeLow, addr(sizeHigh), FILE_BEGIN) + let lastErr = osLastError() + if (status == INVALID_SET_FILE_POINTER and lastErr.int32 != NO_ERROR) or + setEndOfFile(Handle fh) == 0: + result = lastErr + else: + if newFileSize > oldSize: # grow the file + var e: cint = cint(0) # posix_fallocate truncates up when needed. + when declared(posix_fallocate): + while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): + discard + if e == EINVAL or e == EOPNOTSUPP or e == ENOSYS: + # fallback arguable; Most portable BUT allows SEGV + if ftruncate(fh, newFileSize) == -1: + result = osLastError() + else: + discard + elif e != 0: + result = osLastError() + else: # shrink the file + if ftruncate(fh.cint, newFileSize) == -1: result = osLastError() - else: - discard - elif e != 0: - result = osLastError() - else: # shrink the file - if ftruncate(fh.cint, newFileSize) == -1: - result = osLastError() type MemFile* = object ## represents a memory mapped file @@ -84,6 +91,89 @@ type else: handle*: cint ## **Caution**: Posix specific public field. flags: cint ## **Caution**: Platform specific private field. + when nimUseFallBack: + backing: string + path: string + readonly: bool + allowRemap: bool + +when nimUseFallBack: + proc fallbackMappedSize(backingLen, mappedSize, offset: int): int = + if mappedSize < -1: + raise newEIO("mappedSize cannot be less than -1") + if offset < 0 or offset > backingLen: + raise newEIO("offset out of bounds") + if mappedSize == -1: + result = backingLen - offset + else: + result = min(mappedSize, backingLen - offset) + + proc setFallbackView(m: var MemFile, mappedSize, offset: int) = + m.size = fallbackMappedSize(m.backing.len, mappedSize, offset) + if m.size > 0: + m.mem = cast[pointer](addr m.backing[offset]) + else: + m.mem = nil + + proc openFallbackMemFile(filename: string, mode: FileMode, mappedSize, + offset, newFileSize: int, + allowRemap: bool): MemFile = + result = MemFile( + handle: -1, + flags: 0, + path: filename, + readonly: mode == fmRead, + allowRemap: allowRemap + ) + if newFileSize != -1: + result.backing = newString(newFileSize) + else: + result.backing = readFile(filename) + setFallbackView(result, mappedSize, offset) + + proc mapMemFallback(m: var MemFile, mode: FileMode, + mappedSize, offset: int): pointer = + if not m.allowRemap: + raise newException(IOError, + "Cannot remap MemFile opened with allowRemap=false") + if mode != fmRead and m.readonly: + raise newEIO("cannot write to read-only mapping") + let size = fallbackMappedSize(m.backing.len, mappedSize, offset) + if size > 0: + result = cast[pointer](addr m.backing[offset]) + else: + result = nil + + proc flushFallback(m: var MemFile) = + if m.readonly or m.path.len == 0: + return + writeFile(m.path, m.backing) + + proc resizeFallback(m: var MemFile, newFileSize: int) = + if m.readonly: + raise newException(IOError, "Cannot resize read-only MemFile") + if not m.allowRemap: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if m.size != m.backing.len: + raise newException(IOError, "Cannot resize partial MemFile") + let oldLen = m.backing.len + m.backing.setLen(newFileSize) + for i in oldLen ..< newFileSize: + m.backing[i] = '\0' + setFallbackView(m, newFileSize, 0) + + proc closeFallback(m: var MemFile) = + if not m.readonly: + flushFallback(m) + m.mem = nil + m.size = 0 + m.handle = -1 + m.flags = 0 + m.backing = "" + m.path = "" + m.readonly = false + m.allowRemap = false proc mapMem*(m: var MemFile, mode: FileMode = fmRead, mappedSize = -1, offset = 0, mapFlags = cint(-1)): pointer = @@ -94,7 +184,7 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead, if mode == fmAppend: raise newEIO("The append mode is not supported.") - var readonly = mode == fmRead + let readonly = mode == fmRead when defined(windows): result = mapViewOfFileEx( m.mapHandle, @@ -105,6 +195,8 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead, nil) if result == nil: raiseOSError(osLastError()) + elif nimUseFallBack: + result = mapMemFallback(m, mode, mappedSize, offset) else: assert mappedSize > 0 @@ -132,6 +224,8 @@ proc unmapMem*(f: var MemFile, p: pointer, size: int) = ## via `mapMem`. when defined(windows): if unmapViewOfFile(p) == 0: raiseOSError(osLastError()) + elif nimUseFallBack: + discard else: if munmap(p, size) != 0: raiseOSError(osLastError()) @@ -178,7 +272,7 @@ proc open*(filename: string, mode: FileMode = fmRead, raise newEIO("The append mode is not supported.") assert newFileSize == -1 or mode != fmRead - var readonly = mode == fmRead + let readonly = mode == fmRead template rollback = result.mem = nil @@ -252,7 +346,10 @@ proc open*(filename: string, mode: FileMode = fmRead, if closeHandle(result.fHandle) != 0: result.fHandle = INVALID_HANDLE_VALUE - else: + elif nimUseFallBack: + result = openFallbackMemFile(filename, mode, mappedSize, offset, + newFileSize, allowRemap) + elif defined(posix): template fail(errCode: OSErrorCode, msg: string) = rollback() if result.handle != -1: discard close(result.handle) @@ -309,6 +406,8 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = lastErr = osLastError() if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode: raiseOSError(lastErr) + elif nimUseFallBack: + flushFallback(f) else: for i in 1..attempts: res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0 @@ -318,59 +417,71 @@ proc flush*(f: var MemFile; attempts: Natural = 3) = if lastErr != EBUSY.OSErrorCode: raiseOSError(lastErr, "error flushing mapping") -proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = - ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS - ## supports it, file space is reserved to ensure room for new virtual pages. - ## Caller should wait often enough for `flush` to finish to limit use of - ## system RAM for write buffering, perhaps just prior to this call. - ## **Note**: this assumes the entire file is mapped read-write at offset 0. - ## Also, the value of `.mem` will probably change. - if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? - raise newException(IOError, "Cannot resize MemFile to < 1 byte") - when defined(windows): - if not f.wasOpened: - raise newException(IOError, "Cannot resize unopened MemFile") - if f.fHandle == INVALID_HANDLE_VALUE: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map - raiseOSError(osLastError()) - if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. - if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); - e != 0.OSErrorCode): raiseOSError(e) - f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) - if f.mapHandle == 0: # Re-do map - raiseOSError(osLastError()) - let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, - 0, 0, WinSizeT(newFileSize), nil) - if m != nil: - f.mem = m +when nimUseFallBack: + proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + resizeFallback(f, newFileSize) +else: + proc resize*(f: var MemFile, newFileSize: int) {.raises: [IOError, OSError].} = + ## Resize & re-map the file underlying an `allowRemap MemFile`. If the OS/FS + ## supports it, file space is reserved to ensure room for new virtual pages. + ## Caller should wait often enough for `flush` to finish to limit use of + ## system RAM for write buffering, perhaps just prior to this call. + ## **Note**: this assumes the entire file is mapped read-write at offset 0. + ## Also, the value of `.mem` will probably change. + if newFileSize < 1: # Q: include system/bitmasks & use PageSize ? + raise newException(IOError, "Cannot resize MemFile to < 1 byte") + when defined(windows): + if not f.wasOpened: + raise newException(IOError, "Cannot resize unopened MemFile") + if f.fHandle == INVALID_HANDLE_VALUE: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if unmapViewOfFile(f.mem) == 0 or closeHandle(f.mapHandle) == 0: # Un-do map + raiseOSError(osLastError()) + if newFileSize != f.size: # Seek to size & `setEndOfFile` => allocated. + if (let e = setFileSize(f.fHandle.FileHandle, newFileSize); + e != 0.OSErrorCode): raiseOSError(e) + f.mapHandle = createFileMappingW(f.fHandle, nil, PAGE_READWRITE, 0,0,nil) + if f.mapHandle == 0: # Re-do map + raiseOSError(osLastError()) + let m = mapViewOfFileEx(f.mapHandle, FILE_MAP_READ or FILE_MAP_WRITE, + 0, 0, WinSizeT(newFileSize), nil) + if m != nil: + f.mem = m + f.size = newFileSize + else: + raiseOSError(osLastError()) + elif defined(posix): + if f.handle == -1: + raise newException(IOError, + "Cannot resize MemFile opened with allowRemap=false") + if newFileSize != f.size: + let e = setFileSize(f.handle.FileHandle, newFileSize, f.size) + if e != 0.OSErrorCode: raiseOSError(e) + when defined(linux): #Maybe NetBSD, too? + # On Linux this can be over 100 times faster than a munmap,mmap cycle. + proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): + pointer {.importc: "mremap", header: "".} + let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + else: + if munmap(f.mem, f.size) != 0: + raiseOSError(osLastError()) + let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, + f.flags, f.handle, 0) + if newAddr == cast[pointer](MAP_FAILED): + raiseOSError(osLastError()) + f.mem = newAddr f.size = newFileSize - else: - raiseOSError(osLastError()) - elif defined(posix): - if f.handle == -1: - raise newException(IOError, - "Cannot resize MemFile opened with allowRemap=false") - if newFileSize != f.size: - let e = setFileSize(f.handle.FileHandle, newFileSize, f.size) - if e != 0.OSErrorCode: raiseOSError(e) - when defined(linux): #Maybe NetBSD, too? - # On Linux this can be over 100 times faster than a munmap,mmap cycle. - proc mremap(old: pointer; oldSize, newSize: csize_t; flags: cint): - pointer {.importc: "mremap", header: "".} - let newAddr = mremap(f.mem, csize_t(f.size), csize_t(newFileSize), 1.cint) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - else: - if munmap(f.mem, f.size) != 0: - raiseOSError(osLastError()) - let newAddr = mmap(nil, newFileSize, PROT_READ or PROT_WRITE, - f.flags, f.handle, 0) - if newAddr == cast[pointer](MAP_FAILED): - raiseOSError(osLastError()) - f.mem = newAddr - f.size = newFileSize proc close*(f: var MemFile) = ## closes the memory mapped file `f`. All changes are written back to the @@ -389,6 +500,8 @@ proc close*(f: var MemFile) = f.fHandle = INVALID_HANDLE_VALUE if error: lastErr = osLastError() + elif nimUseFallBack: + closeFallback(f) else: error = munmap(f.mem, f.size) != 0 lastErr = osLastError() diff --git a/tests/stdlib/tmemfiles2.nim b/tests/stdlib/tmemfiles2.nim index c79f85ebfb..1c6d605286 100644 --- a/tests/stdlib/tmemfiles2.nim +++ b/tests/stdlib/tmemfiles2.nim @@ -1,4 +1,5 @@ discard """ + matrix: "; -d:nimMemfileFallback" disabled: "Windows" output: '''Full read size: 20 Half read size: 10 Data: Hello''' From 0448557bfec17f42acdccea2ad6a157ce620ce0d Mon Sep 17 00:00:00 2001 From: Ryan McConnell Date: Thu, 11 Jun 2026 02:17:35 -0400 Subject: [PATCH 05/46] fix 25778; concept coerces incompatible types (#25781) I don't like it, but seems like this is correct. Concept type classes have to behave like other "named" type classes and participate in "bind once" mechanics or require some weird semantics. As a side note I'm pretty sure the `tuple` example in the manual explaining this is either wrong now or has regressed, but I don't think it matters because I doubt anyone thinks about this feature much. #25778 --- compiler/sigmatch.nim | 4 +++ .../conceptv2negative/tconsistentparams.nim | 28 +++++++++++++++++++ tests/concepts/tconceptsv2.nim | 14 ++++++++++ 3 files changed, 46 insertions(+) create mode 100644 tests/concepts/conceptv2negative/tconsistentparams.nim diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index cb79823af5..9526d37290 100644 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1168,6 +1168,10 @@ proc enterConceptMatch(c: var TCandidate; f,a: PType, flags: TTypeRelFlags): TTy if concpt.kind != tyConcept: container = concpt concpt = container.reduceToBase + # considerPreviousT-like behavior + let prev = lookup(c.bindings, concpt) + if prev != nil: + return typeRel(c, prev, a, flags) if trDontBind in flags: conceptFlags.incl mfDontBind if trCheckGeneric in flags: diff --git a/tests/concepts/conceptv2negative/tconsistentparams.nim b/tests/concepts/conceptv2negative/tconsistentparams.nim new file mode 100644 index 0000000000..c1a0c1290d --- /dev/null +++ b/tests/concepts/conceptv2negative/tconsistentparams.nim @@ -0,0 +1,28 @@ +discard """ +action: "reject" +errormsg: "type mismatch" +""" + +type + Dollarable = concept + proc `$`(x: Self): string + +proc checkEqual(x, y: Dollarable) = + if x != y: + echo $x + echo $y + +type + StateFlags = enum + sfMatch + sfSoft + + MatchKind = enum + NoFurtherMatch + NoMatch + Match + AllFurtherMatch + +proc `==`(a: set[StateFlags]; b: MatchKind): bool = true + +checkEqual({sfMatch, sfSoft}, Match) diff --git a/tests/concepts/tconceptsv2.nim b/tests/concepts/tconceptsv2.nim index d861c51c75..13f8211f5b 100644 --- a/tests/concepts/tconceptsv2.nim +++ b/tests/concepts/tconceptsv2.nim @@ -14,6 +14,8 @@ b c 1 2 +5 +test ''' """ import conceptsv2_helper @@ -600,3 +602,15 @@ block: let test = MemMapFileStream() spring(test) + +# explicit negative "bind once" + +type + Dollarable = concept + proc `$`(x: Self): string + +proc checkEqual2[T: Dollarable; S: Dollarable](x: T, y: S) = + echo $x + echo $y + +checkEqual2(5, "test") From eaa4b342beb8827a9b30acf431afd2fa7bb2814d Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Jun 2026 10:33:51 +0200 Subject: [PATCH 06/46] system: remove unused exception raising code (#25894) ...that otherwise causes an unnecessary raise effect on writeWindows / echoBinSafe --- lib/system.nim | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 44385e1772..94fe96ea93 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -3131,10 +3131,7 @@ when notJSnotNims: not defined(nuttx) and hostOS != "any" - proc raiseEIO(msg: string) {.noinline, noreturn.} = - raise newException(IOError, msg) - - proc echoBinSafe(args: openArray[string]) {.compilerproc.} = + proc echoBinSafe(args: openArray[string]) {.compilerproc, raises: [].} = when defined(androidNDK): # When running nim in android app, stdout goes nowhere, so echo gets ignored # To redirect echo to the android logcat, use -d:androidNDK @@ -3156,7 +3153,7 @@ when notJSnotNims: for s in args: when defined(windows): # equivalent to syncio.writeWindows - proc writeWindows(f: CFilePtr; s: string; doRaise = false) = + proc writeWindows(f: CFilePtr; s: string) = # Don't ask why but the 'printf' family of function is the only thing # that writes utf-8 strings reliably on Windows. At least on my Win 10 # machine. We also enable `setConsoleOutputCP(65001)` now by default. @@ -3167,13 +3164,11 @@ when notJSnotNims: if s[i] == '\0': let w = c_fputc('\0', f) if w != 0: - if doRaise: raiseEIO("cannot write string to file") break inc i else: let w = c_fprintf(f, "%s", unsafeAddr s[i]) if w <= 0: - if doRaise: raiseEIO("cannot write string to file") break inc i, w writeWindows(cstdout, s) From c620adcfce2200ee3baf90279afcd657e7371491 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Jun 2026 14:10:42 +0200 Subject: [PATCH 07/46] astyaml: formatting fixes (#25897) fix missing indent and newlines here and there --- compiler/astyaml.nim | 58 ++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/compiler/astyaml.nim b/compiler/astyaml.nim index b0fa2bfb28..260d830b11 100644 --- a/compiler/astyaml.nim +++ b/compiler/astyaml.nim @@ -43,13 +43,13 @@ proc flagsToStr[T](flags: set[T]): string = proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): string = result = "[" result.addYamlString(toFilename(conf, info)) - result.addf ", $1, $2]", [toLinenumber(info), toColumn(info)] + result.addf ", $1, $2]", toLinenumber(info), toColumn(info) -proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; indent, maxRecDepth: int) -proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; indent, maxRecDepth: int) -proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; indent, maxRecDepth: int) +proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; nl: bool, indent, maxRecDepth: int) +proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; nl: bool, indent, maxRecDepth: int) +proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; nl: bool, indent, maxRecDepth: int) -proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; indent: int; maxRecDepth: int) = +proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; nl: bool, indent: int; maxRecDepth: int) = if n == nil: res.add("null") elif containsOrIncl(marker, n.id): @@ -57,10 +57,12 @@ proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; else: let istr = spaces(indent * 4) + if nl: + res.addf("\n$1", istr) res.addf("kind: $1", [makeYamlString($n.kind)]) res.addf("\n$1name: $2", [istr, makeYamlString(n.name.s)]) res.addf("\n$1typ: ", [istr]) - res.typeToYamlAux(conf, n.typ, marker, indent + 1, maxRecDepth - 1) + res.typeToYamlAux(conf, n.typ, marker, true, indent + 1, maxRecDepth - 1) if conf != nil: # if we don't pass the config, we probably don't care about the line info res.addf("\n$1info: $2", [istr, lineInfoToStr(conf, n.info)]) @@ -68,7 +70,7 @@ proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; res.addf("\n$1flags: $2", [istr, flagsToStr(n.flags)]) res.addf("\n$1magic: $2", [istr, makeYamlString($n.magic)]) res.addf("\n$1ast: ", [istr]) - res.treeToYamlAux(conf, n.ast, marker, indent + 1, maxRecDepth - 1) + res.treeToYamlAux(conf, n.ast, marker, true, indent + 1, maxRecDepth - 1) res.addf("\n$1options: $2", [istr, flagsToStr(n.options)]) res.addf("\n$1position: $2", [istr, $n.position]) res.addf("\n$1k: $2", [istr, makeYamlString($n.loc.k)]) @@ -76,53 +78,57 @@ proc symToYamlAux(res: var string; conf: ConfigRef; n: PSym; marker: var IntSet; if card(n.loc.flags) > 0: res.addf("\n$1flags: $2", [istr, makeYamlString($n.loc.flags)]) res.addf("\n$1snippet: $2", [istr, n.loc.snippet]) - res.addf("\n$1lode: $2", [istr]) - res.treeToYamlAux(conf, n.loc.lode, marker, indent + 1, maxRecDepth - 1) + res.addf("\n$1lode: ", [istr]) + res.treeToYamlAux(conf, n.loc.lode, marker, true, indent + 1, maxRecDepth - 1) -proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; indent: int; maxRecDepth: int) = +proc typeToYamlAux(res: var string; conf: ConfigRef; n: PType; marker: var IntSet; nl: bool, indent: int; maxRecDepth: int) = if n == nil: res.add("null") elif containsOrIncl(marker, n.id): res.addf "\"$1 @$2\"" % [$n.kind, strutils.toHex(cast[uint](n), sizeof(n) * 2)] else: let istr = spaces(indent * 4) + if nl: + res.addf("\n$1", istr) res.addf("kind: $2", [istr, makeYamlString($n.kind)]) - res.addf("\n$1sym: ") - res.symToYamlAux(conf, n.sym, marker, indent + 1, maxRecDepth - 1) - res.addf("\n$1n: ") - res.treeToYamlAux(conf, n.n, marker, indent + 1, maxRecDepth - 1) + res.addf("\n$1sym: ", istr) + res.symToYamlAux(conf, n.sym, marker, true, indent + 1, maxRecDepth - 1) + res.addf("\n$1n: ", istr) + res.treeToYamlAux(conf, n.n, marker, true, indent + 1, maxRecDepth - 1) if card(n.flags) > 0: res.addf("\n$1flags: $2", [istr, flagsToStr(n.flags)]) res.addf("\n$1callconv: $2", [istr, makeYamlString($n.callConv)]) res.addf("\n$1size: $2", [istr, $(n.size)]) res.addf("\n$1align: $2", [istr, $(n.align)]) if n.hasElementType: - res.addf("\n$1sons:") + res.addf("\n$1sons:", istr) for a in n.kids: - res.addf("\n - ") - res.typeToYamlAux(conf, a, marker, indent + 1, maxRecDepth - 1) + res.addf("\n$1 - ", istr) + res.typeToYamlAux(conf, a, marker, false, indent + 1, maxRecDepth - 1) -proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; indent: int; +proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSet; nl: bool, indent: int; maxRecDepth: int) = if n == nil: res.add("null") else: var istr = spaces(indent * 4) + if nl: + res.addf("\n$1", istr) res.addf("kind: $1" % [makeYamlString($n.kind)]) if maxRecDepth != 0: if conf != nil: res.addf("\n$1info: $2", [istr, lineInfoToStr(conf, n.info)]) case n.kind - of nkCharLit .. nkInt64Lit: + of nkCharLit .. nkUInt64Lit: res.addf("\n$1intVal: $2", [istr, $(n.intVal)]) - of nkFloatLit, nkFloat32Lit, nkFloat64Lit: + of nkFloatLit .. nkFloat128Lit: res.addf("\n$1floatVal: $2", [istr, n.floatVal.toStrMaxPrecision]) of nkStrLit .. nkTripleStrLit: res.addf("\n$1strVal: $2", [istr, makeYamlString(n.strVal)]) of nkSym: res.addf("\n$1sym: ", [istr]) - res.symToYamlAux(conf, n.sym, marker, indent + 1, maxRecDepth) + res.symToYamlAux(conf, n.sym, marker, true, indent + 1, maxRecDepth) of nkIdent: if n.ident != nil: res.addf("\n$1ident: $2", [istr, makeYamlString(n.ident.s)]) @@ -133,22 +139,22 @@ proc treeToYamlAux(res: var string; conf: ConfigRef; n: PNode; marker: var IntSe res.addf("\n$1sons: ", [istr]) for i in 0 ..< n.len: res.addf("\n$1 - ", [istr]) - res.treeToYamlAux(conf, n[i], marker, indent + 1, maxRecDepth - 1) + res.treeToYamlAux(conf, n[i], marker, false, indent + 1, maxRecDepth - 1) if n.typ != nil: res.addf("\n$1typ: ", [istr]) - res.typeToYamlAux(conf, n.typ, marker, indent + 1, maxRecDepth) + res.typeToYamlAux(conf, n.typ, marker, true, indent + 1, maxRecDepth) proc treeToYaml*(conf: ConfigRef; n: PNode; indent: int = 0; maxRecDepth: int = -1): string = var marker = initIntSet() result = newStringOfCap(1024) - result.treeToYamlAux(conf, n, marker, indent, maxRecDepth) + result.treeToYamlAux(conf, n, marker, false, indent, maxRecDepth) proc typeToYaml*(conf: ConfigRef; n: PType; indent: int = 0; maxRecDepth: int = -1): string = var marker = initIntSet() result = newStringOfCap(1024) - result.typeToYamlAux(conf, n, marker, indent, maxRecDepth) + result.typeToYamlAux(conf, n, marker, false, indent, maxRecDepth) proc symToYaml*(conf: ConfigRef; n: PSym; indent: int = 0; maxRecDepth: int = -1): string = var marker = initIntSet() result = newStringOfCap(1024) - result.symToYamlAux(conf, n, marker, indent, maxRecDepth) + result.symToYamlAux(conf, n, marker, false, indent, maxRecDepth) From 13d152a4d18d0e28264d1ddbf2e17b6237d9e059 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Jun 2026 16:12:28 +0200 Subject: [PATCH 08/46] fix state array constant types (#25893) else there's a mismatch in the AST for the bracket constructor --- compiler/closureiters.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 96734ca8fa..6eeffb4099 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -252,7 +252,8 @@ proc newCurExcAccess(ctx: var Ctx): PNode = ctx.newEnvVarAccess(ctx.curExcSym) proc newStateLabel(ctx: Ctx): PNode = - ctx.g.newIntLit(TLineInfo(), 0) + result = nkIntLit.newIntNode(0) + result.typ = getSysType(ctx.g, TLineInfo(), tyInt16) proc newState(ctx: var Ctx, n: PNode, inlinable: bool, label: PNode): PNode = # Creates a new state, adds it to the context From 13760525199d7d96659016ff864a2f3e9ed35674 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Jun 2026 16:13:24 +0200 Subject: [PATCH 09/46] memalloc: fix forward declarations (#25895) None of them have side effects / all are gcsafe --- lib/system/memalloc.nim | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/system/memalloc.nim b/lib/system/memalloc.nim index ed0de06c19..0739eb6f62 100644 --- a/lib/system/memalloc.nim +++ b/lib/system/memalloc.nim @@ -1,13 +1,13 @@ when notJSnotNims: - proc zeroMem*(p: pointer, size: Natural) {.inline, noSideEffect, - tags: [], raises: [], enforceNoRaises.} + proc zeroMem*(p: pointer, size: Natural) {.inline, gcsafe, + tags: [], raises: [], enforceNoRaises, noSideEffect.} ## Overwrites the contents of the memory at `p` with the value 0. ## ## Exactly `size` bytes will be overwritten. Like any procedure ## dealing with raw memory this is **unsafe**. proc copyMem*(dest, source: pointer, size: Natural) {.inline, gcsafe, - tags: [], raises: [], enforceNoRaises.} + tags: [], raises: [], enforceNoRaises, noSideEffect.} ## Copies the contents from the memory at `source` to the memory ## at `dest`. ## Exactly `size` bytes will be copied. The memory @@ -15,7 +15,7 @@ when notJSnotNims: ## memory this is **unsafe**. proc moveMem*(dest, source: pointer, size: Natural) {.inline, gcsafe, - tags: [], raises: [], enforceNoRaises.} + tags: [], raises: [], enforceNoRaises, noSideEffect.} ## Copies the contents from the memory at `source` to the memory ## at `dest`. ## @@ -24,8 +24,8 @@ when notJSnotNims: ## and is thus somewhat more safe than `copyMem`. Like any procedure ## dealing with raw memory this is still **unsafe**, though. - proc equalMem*(a, b: pointer, size: Natural): bool {.inline, noSideEffect, - tags: [], raises: [], enforceNoRaises.} + proc equalMem*(a, b: pointer, size: Natural): bool {.inline, gcsafe, + tags: [], raises: [], enforceNoRaises, noSideEffect.} ## Compares the memory blocks `a` and `b`. `size` bytes will ## be compared. ## @@ -33,8 +33,8 @@ when notJSnotNims: ## otherwise. Like any procedure dealing with raw memory this is ## **unsafe**. - proc cmpMem*(a, b: pointer, size: Natural): int {.inline, noSideEffect, - tags: [], raises: [], enforceNoRaises.} + proc cmpMem*(a, b: pointer, size: Natural): int {.inline, gcsafe, + tags: [], raises: [], enforceNoRaises, noSideEffect.} ## Compares the memory blocks `a` and `b`. `size` bytes will ## be compared. ## From 7fa006c4e51de6c44164cc53a059fdc494fb6ac5 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 11 Jun 2026 20:24:48 +0200 Subject: [PATCH 10/46] fix invalid join (#25896) can't join a thread that wasn't started (causes random crashes) --- tests/generics/t22305.nim | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tests/generics/t22305.nim b/tests/generics/t22305.nim index 6158ee3f1b..967932a05a 100644 --- a/tests/generics/t22305.nim +++ b/tests/generics/t22305.nim @@ -14,12 +14,6 @@ type WorkProc[A, B] = proc(a: A): Option[B] {.nimcall.} proc worker[TArg](p: TArg) {.thread, nimcall.} = discard -proc readFilesThread() = - type TArg[A, B] = - tuple[r: ptr Channel[Option[A]], w: ptr Channel[Option[B]], p: WorkProc[A, B]] - - var readThread: Thread[TArg[int, SharedBuf]] - proc readFilesAd() {.async.} = var readChan: Channel[Option[int]] @@ -29,8 +23,6 @@ proc readFilesAd() {.async.} = var readThread: Thread[TArg[int, SharedBuf]] let test = await (addr readChan).recv() - joinThread(readThread) - waitFor readFilesAd() type From b44d373b7df1611b19594357b90002b1ac90df37 Mon Sep 17 00:00:00 2001 From: WyattBlue Date: Thu, 11 Jun 2026 17:49:50 -0400 Subject: [PATCH 11/46] adds wasm64 (Memory64) as a first-class target (#25900) This pull request allows setting `--cpu:wasm64`, allowing wasm64 as a first class target. This avoids having to set `-cpu:riscv64` as a workaround. Sane defaults for the emscripten toolchain are also provided. --- compiler/options.nim | 1 + compiler/platform.nim | 5 +++-- config/nim.cfg | 13 +++++++++++++ lib/system/platforms.nim | 4 +++- tools/nim.zsh-completion | 2 +- 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 472ce0b4c2..b8213594c9 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -662,6 +662,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool = of "x86": result = conf.target.targetCPU == cpuI386 of "itanium": result = conf.target.targetCPU == cpuIa64 of "x8664": result = conf.target.targetCPU == cpuAmd64 + of "wasm": result = conf.target.targetCPU in {cpuWasm32, cpuWasm64} of "posix", "unix": result = conf.target.targetOS in {osLinux, osMorphos, osSkyos, osIrix, osPalmos, osQnx, osAtari, osAix, diff --git a/compiler/platform.nim b/compiler/platform.nim index 4b99cd8936..62c8e3e90f 100644 --- a/compiler/platform.nim +++ b/compiler/platform.nim @@ -211,7 +211,7 @@ type cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips, cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430, cpuSparc64, cpuS390x, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, - cpuEsp, cpuWasm32, cpuE2k, cpuLoongArch64 + cpuEsp, cpuWasm32, cpuE2k, cpuLoongArch64, cpuWasm64 type TInfoCPU* = tuple[name: string, intSize: int, endian: Endianness, @@ -249,7 +249,8 @@ const (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32), (name: "e2k", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), - (name: "loongarch64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] + (name: "loongarch64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64), + (name: "wasm64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64)] type Target* = object diff --git a/config/nim.cfg b/config/nim.cfg index 038c3c9bec..6a259321fb 100644 --- a/config/nim.cfg +++ b/config/nim.cfg @@ -168,6 +168,19 @@ nimblepath="$home/.nimble/pkgs/" switch_gcc.cpp.options.always = "-g -Wall -O2 -ffunction-sections -march=armv8-a -mtune=cortex-a57 -mtp=soft -fPIE -D__SWITCH__ -fno-rtti -fno-exceptions -std=gnu++11" @end +# Emscripten toolchain for WebAssembly (wasm32, or wasm64/Memory64). +@if emscripten: + cc = clang + clang.exe = "emcc" + clang.linkerexe = "emcc" + clang.cpp.exe = "emcc" + clang.cpp.linkerexe = "emcc" + @if wasm64: + passC = "-sMEMORY64=1" + passL = "-sMEMORY64=1" + @end +@end + # Configuration for the Intel C/C++ compiler: @if windows: icl.options.speed = "/Ox /arch:SSE2" diff --git a/lib/system/platforms.nim b/lib/system/platforms.nim index 0619f3fcaa..17df7188f5 100644 --- a/lib/system/platforms.nim +++ b/lib/system/platforms.nim @@ -40,7 +40,8 @@ type wasm32, ## WASM, 32-bit e2k, ## MCST Elbrus 2000 loongarch64, ## LoongArch 64-bit processor - s390x ## IBM Z + s390x, ## IBM Z + wasm64 ## WASM, 64-bit OsPlatform* {.pure.} = enum ## the OS this program will run on. none, dos, windows, os2, linux, morphos, skyos, solaris, @@ -101,5 +102,6 @@ const elif defined(e2k): CpuPlatform.e2k elif defined(loongarch64): CpuPlatform.loongarch64 elif defined(s390x): CpuPlatform.s390x + elif defined(wasm64): CpuPlatform.wasm64 else: CpuPlatform.none ## the CPU this program will run on. diff --git a/tools/nim.zsh-completion b/tools/nim.zsh-completion index 2300cea033..0c34c660a2 100644 --- a/tools/nim.zsh-completion +++ b/tools/nim.zsh-completion @@ -78,7 +78,7 @@ _nim() { '--opt\:-[optimization mode]:x:(none speed size)' '--debugger\:native[use native debugger (gdb)]' '--app\:-[generate this type of app (lib=dynamic)]:x:(console gui lib staticlib)' - '--cpu\:-[target architecture]:x:(alpha amd64 arm arm64 avr e2k esp hppa i386 ia64 js loongarch64 m68k mips mipsel mips64 mips64el msp430 nimvm powerpc powerpc64 powerpc64el riscv32 riscv64 sparc sparc64 s390x vm wasm32)' + '--cpu\:-[target architecture]:x:(alpha amd64 arm arm64 avr e2k esp hppa i386 ia64 js loongarch64 m68k mips mipsel mips64 mips64el msp430 nimvm powerpc powerpc64 powerpc64el riscv32 riscv64 sparc sparc64 s390x vm wasm32 wasm64)' '--gc\:-[memory management algorithm to use (default\: refc)]:x:(refc arc orc markAndSweep boehm go regions none)' '--os\:-[operating system to compile for]:x:(AIX Amiga Android Any Atari DOS DragonFly FreeBSD FreeRTOS Genode Haiku iOS Irix JS Linux MacOS MacOSX MorphOS NetBSD Netware NimVM NintendoSwitch OS2 OpenBSD PalmOS Standalone QNX SkyOS Solaris VxWorks Windows)' '--panics\:-[turn panics into process termination (default\: off)]:x:(off on)' From 9db9b8ce5783ced38bda00f296a98b97d82e530b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 13 Jun 2026 11:06:51 +0800 Subject: [PATCH 12/46] adds regression tests (#25906) closes #22842, closes #21252, closes #19312, closes #16956, closes #16416, closes #14913, closes #13296, closes #12424, closes #10902, closes #9892, closes #9617 --- tests/arc/t19312.nim | 19 +++++++++++++++++++ tests/async/t16416.nim | 17 +++++++++++++++++ tests/concepts/t14913.nim | 23 +++++++++++++++++++++++ tests/destructor/t9617.nim | 20 ++++++++++++++++++++ tests/errmsgs/t16956.nim | 11 +++++++++++ tests/generics/t21252.nim | 35 +++++++++++++++++++++++++++++++++++ tests/macros/t10902.nim | 31 +++++++++++++++++++++++++++++++ tests/macros/t13296.nim | 21 +++++++++++++++++++++ tests/macros/t9892.nim | 16 ++++++++++++++++ tests/pragmas/t12424.nim | 8 ++++++++ tests/typerel/t22842.nim | 11 +++++++++++ 11 files changed, 212 insertions(+) create mode 100644 tests/arc/t19312.nim create mode 100644 tests/async/t16416.nim create mode 100644 tests/concepts/t14913.nim create mode 100644 tests/destructor/t9617.nim create mode 100644 tests/errmsgs/t16956.nim create mode 100644 tests/generics/t21252.nim create mode 100644 tests/macros/t10902.nim create mode 100644 tests/macros/t13296.nim create mode 100644 tests/macros/t9892.nim create mode 100644 tests/pragmas/t12424.nim create mode 100644 tests/typerel/t22842.nim diff --git a/tests/arc/t19312.nim b/tests/arc/t19312.nim new file mode 100644 index 0000000000..cded439ec7 --- /dev/null +++ b/tests/arc/t19312.nim @@ -0,0 +1,19 @@ +discard """ + matrix: "--mm:orc" + output: '''(val: 1) +(val: 1)''' +""" +# Issue #19312: copied ref object is converted to nil if not used in declaration module under ARC/ORC +# https://github.com/nim-lang/Nim/issues/19312 + +type + Wrapper* = object + val: int + RefWrapper* = ref Wrapper + +let + a* = RefWrapper(val: 1) + b* = a + +echo b[] +echo a[] diff --git a/tests/async/t16416.nim b/tests/async/t16416.nim new file mode 100644 index 0000000000..90ac695ffb --- /dev/null +++ b/tests/async/t16416.nim @@ -0,0 +1,17 @@ +discard """ + output: '''done''' +""" +# Issue #16416: Can't call closure iterator from inside an async function +# https://github.com/nim-lang/Nim/issues/16416 + +import asyncdispatch + +iterator x(): int {.closure.} = + yield 1 + +proc y() {.async.} = + for z in x(): + discard + +waitFor y() +echo "done" diff --git a/tests/concepts/t14913.nim b/tests/concepts/t14913.nim new file mode 100644 index 0000000000..3ef67b9f4c --- /dev/null +++ b/tests/concepts/t14913.nim @@ -0,0 +1,23 @@ +discard """ + output: '''done''' +""" +# Issue #14913: Compiler crash when using a default parameter value for a parameter whose type is a concept +# https://github.com/nim-lang/Nim/issues/14913 + +type + State = object + MoreState = object + StringRecord = concept x, type T + for k, v in fieldPairs(x): + k is string + v is string + StateStrings = object + a, b: string + +proc combine(a: State, b: MoreState): StateStrings = discard + +proc whoops[T: StringRecord](a: State, b: MoreState, c: T = a.combine(b)) = + discard + +whoops(State(), MoreState()) +echo "done" diff --git a/tests/destructor/t9617.nim b/tests/destructor/t9617.nim new file mode 100644 index 0000000000..260d458b24 --- /dev/null +++ b/tests/destructor/t9617.nim @@ -0,0 +1,20 @@ +discard """ + output: '''done''' +""" +# Issue #9617: Compiler error with sequences of destructible types +# https://github.com/nim-lang/Nim/issues/9617 + +type + Foo* = object + + Bar = ref object + s: seq[Foo] + +proc `=destroy`*(self: var Foo) = echo "hi" + +proc test(b: Bar) = + for i in b.s: + discard + +test(Bar()) +echo "done" diff --git a/tests/errmsgs/t16956.nim b/tests/errmsgs/t16956.nim new file mode 100644 index 0000000000..25e3e4cec8 --- /dev/null +++ b/tests/errmsgs/t16956.nim @@ -0,0 +1,11 @@ +discard """ + action: "reject" + errormsg: "invalid type: 'iterator (a: int, b: int, step: Positive): int{.inline, noSideEffect, gcsafe.}' for const" + line: 9 +""" +# Issue #16956: Error: not unused depending on unrelated code changes +# https://github.com/nim-lang/Nim/issues/16956 + +const f2 = case true + of true: countup[int] + of false: countdown[int] diff --git a/tests/generics/t21252.nim b/tests/generics/t21252.nim new file mode 100644 index 0000000000..5240280d72 --- /dev/null +++ b/tests/generics/t21252.nim @@ -0,0 +1,35 @@ +discard """ + output: '''done''' +""" +# Issue #21252: Compiler SIGSEGV when not instantiating generic proc correctly +# https://github.com/nim-lang/Nim/issues/21252 + +type + Addr = object + layerIdx: int + +type Msg0 = object + address: Addr + selSample: tuple[inArrays: seq[seq[float64]], target: seq[float64], gradientStrength: float64] + +type WeightUpdate = object + address: Addr + +proc workerThread[ + layer0StimulusWidth: static int + ]() = + discard + +proc z*[ + layer0StimulusWidth: static int, + nUnitsPerLayer: static seq[int], + targetLen: static int + ]() = + workerThread[layer0StimulusWidth]() + +when isMainModule: + const layer0StimulusWidth: int = 5*29 + const nUnitsPerLayer: seq[int] = @[50, 5] + const targetLen: int = 5 + z[layer0StimulusWidth, nUnitsPerLayer, targetLen]() + echo "done" diff --git a/tests/macros/t10902.nim b/tests/macros/t10902.nim new file mode 100644 index 0000000000..13a0f3f817 --- /dev/null +++ b/tests/macros/t10902.nim @@ -0,0 +1,31 @@ +discard """ + output: '''done''' +""" +# Issue #10902: cannot instantiate T when generating AST from macro +# https://github.com/nim-lang/Nim/issues/10902 + +import macros + +type + Base[T] = ref object + +macro genCloneProc(typeWithGenArg: untyped): untyped = + result = newProc( + ident "clone", [ + typeWithGenArg, + newIdentDefs( + ident "self", + typeWithGenArg, + ) + ], + newStmtList( + newNimNode(nnkDiscardStmt).add(newEmptyNode()) + ) + ) + let genericParamIdent = typeWithGenArg[1] + result[2] = newNimNode(nnkGenericParams) + result[2].add(newIdentDefs(genericParamIdent, newEmptyNode())) + +genCloneProc(Base[T]) + +echo "done" diff --git a/tests/macros/t13296.nim b/tests/macros/t13296.nim new file mode 100644 index 0000000000..e630f4dde9 --- /dev/null +++ b/tests/macros/t13296.nim @@ -0,0 +1,21 @@ +discard """ + output: '''done''' +""" +# Issue #13296: Error: not unused with a macro +# https://github.com/nim-lang/Nim/issues/13296 + +import macros +macro dType(body: untyped) = + if body.kind == nnkCall: + var typ = newNimNode(nnkStmtList) + typ.add quote do: + discard + elif body.kind == nnkTypeSection: + result = newStmtList( + body + ) + +dType: + echo "hi" + +echo "done" diff --git a/tests/macros/t9892.nim b/tests/macros/t9892.nim new file mode 100644 index 0000000000..f67e1d4c35 --- /dev/null +++ b/tests/macros/t9892.nim @@ -0,0 +1,16 @@ +discard """ + output: '''1''' +""" +# Issue #9892: Incorrect "Error: not unused" from else branch in macro +# https://github.com/nim-lang/Nim/issues/9892 + +import macros + +macro foo(x: typed): untyped = + result = newNimNode(nnkStmtListExpr) + if x.kind == nnkStmtListExpr: + result.add x + else: + result = x + +echo foo(1) diff --git a/tests/pragmas/t12424.nim b/tests/pragmas/t12424.nim new file mode 100644 index 0000000000..1a2e0ebdcf --- /dev/null +++ b/tests/pragmas/t12424.nim @@ -0,0 +1,8 @@ +discard """ + nimout: '''t12424.nim(8, 10) Warning: This is a test warning from user code [User]''' +""" +# Issue #12424: Warning and Hint Pragmas do not print to console when declared from a std lib module +# https://github.com/nim-lang/Nim/issues/12424 +# This test verifies that warning pragmas in user code work correctly. + +{.warning: "This is a test warning from user code".} diff --git a/tests/typerel/t22842.nim b/tests/typerel/t22842.nim new file mode 100644 index 0000000000..917aee875f --- /dev/null +++ b/tests/typerel/t22842.nim @@ -0,0 +1,11 @@ +discard """ + output: '''done''' +""" +# Issue #22842: internal error: getTypeDescAux(tyAnything) with auto in proc type +# https://github.com/nim-lang/Nim/issues/22842 + +proc register(cb: proc (e: auto): void) = discard + +register(proc (e: int) = echo e) + +echo "done" From 0f751695e4fb27c36ddc2048ea26c729e33b22f6 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 13 Jun 2026 13:03:28 +0800 Subject: [PATCH 13/46] fixes #25903 and #25904; add closure iterators with error handling (#25905) fixes #25903 fixes #25904 `nkExceptBranch` can have variable structure depending on the exception types and it should handle the last node of the `nkExceptBranch` --- compiler/closureiters.nim | 5 +---- tests/closure/tclosure_issues.nim | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 6eeffb4099..f96685cbbd 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -593,10 +593,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode = let branch = n[i] case branch.kind of nkExceptBranch: - if branch[0].kind == nkType: - branch[1] = ctx.convertExprBodyToAsgn(branch[1], tmp) - else: - branch[0] = ctx.convertExprBodyToAsgn(branch[0], tmp) + branch[^1] = ctx.convertExprBodyToAsgn(branch[^1], tmp) of nkFinally: discard else: diff --git a/tests/closure/tclosure_issues.nim b/tests/closure/tclosure_issues.nim index b1a2d7c6b6..75dc6ada79 100644 --- a/tests/closure/tclosure_issues.nim +++ b/tests/closure/tclosure_issues.nim @@ -80,3 +80,20 @@ block tissue7104: sp do (): inc i echo "ok ", i + +block: # bug #25903 + iterator g: int {.closure.} = + discard try: + yield 0 + 0 + except IOError, OSError: + 0 + let _ = g + +block: # bug #25904 + iterator w: int {.closure.} = + discard try: 0 + except IOError, OSError: + yield 0 + 0 + let _ = w From 67707a54b5df274372fc8c43f1fe8887aa14870b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 13 Jun 2026 13:04:17 +0800 Subject: [PATCH 14/46] fixes #18367 and #21222; using quote inside static block (#25907) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes #18367 fixes #21222 1. In vmdef.nim:304, newCtx now sets templInstCounter: new int when it builds TCtx. 2. vm.nim:1490 — During VM execution of templates, c.templInstCounter is passed to evalTemplate 3. evaltempl.nim:204 — instID: instID[] dereferences the ref int 4. With a nil templInstCounter, this would crash 5. The same initialization already exists on the semantic side in sem.nim:787, so this change makes the VM path consistent with the rest of the compiler. --- compiler/vmdef.nim | 2 +- tests/vm/tquote.nim | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) create mode 100644 tests/vm/tquote.nim diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index a3ac120f99..8310062dbb 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -308,7 +308,7 @@ proc newCtx*(module: PSym; cache: IdentCache; g: ModuleGraph; idgen: IdGenerator callDepth: g.config.maxCallDepthVM, comesFromHeuristic: unknownLineInfo, callbacks: @[], callbackIndex: initTable[string, int](), errorFlag: "", cache: cache, config: g.config, graph: g, idgen: idgen, - contstantTab: initNodeTable(true)) + contstantTab: initNodeTable(true), templInstCounter: new int) proc refresh*(c: PCtx, module: PSym; idgen: IdGenerator) = c.module = module diff --git a/tests/vm/tquote.nim b/tests/vm/tquote.nim new file mode 100644 index 0000000000..3b4c520519 --- /dev/null +++ b/tests/vm/tquote.nim @@ -0,0 +1,13 @@ +discard """ + joinable: false +""" + +import std/macros + +static: + discard quote: + a and b + +var x {.compileTime.} : NimNode = + quote do: + echo "xxx" \ No newline at end of file From 8ad1d106ec9ca41fcdc997eb3041853c5ffd9a1d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 13 Jun 2026 16:17:44 +0800 Subject: [PATCH 15/46] fixes #25885; incompleteStruct ignored without importc (#25898) fixes #25885 --- compiler/sizealignoffsetimpl.nim | 7 ++++--- compiler/vmgen.nim | 17 ++++++++++++----- doc/manual.md | 3 +++ tests/misc/tsizeof_incompleteStruct.nim | 12 ++++++++++++ 4 files changed, 31 insertions(+), 8 deletions(-) create mode 100644 tests/misc/tsizeof_incompleteStruct.nim diff --git a/compiler/sizealignoffsetimpl.nim b/compiler/sizealignoffsetimpl.nim index 1dd481ec0b..fecc6fdd22 100644 --- a/compiler/sizealignoffsetimpl.nim +++ b/compiler/sizealignoffsetimpl.nim @@ -394,9 +394,10 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) = accum.offset = 1 computeObjectOffsetsFoldFunction(conf, typ.n, false, accum) let paddingAtEnd = int16(accum.finish()) - if typ.sym != nil and - typ.sym.flags * {sfCompilerProc, sfImportc} == {sfImportc} and - tfCompleteStruct notin typ.flags: + if (typ.sym != nil and + typ.sym.flags * {sfCompilerProc, sfImportc} == {sfImportc} and + tfCompleteStruct notin typ.flags) or + tfIncompleteStruct in typ.flags: typ.size = szUnknownSize typ.align = szUnknownSize typ.paddingAtEnd = szUnknownSize diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index dd8b8365ca..56458f077d 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -786,8 +786,12 @@ proc genBinaryABCD(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode) = c.freeTemp(tmp2) c.freeTemp(tmp3) -template sizeOfLikeMsg(name): string = - "'$1' requires '.importc' types to be '.completeStruct'" % [name] +template sizeOfLikeMsg(name, incompleteStruct): string = + block: + if incompleteStruct: + "'$1' cannot be used with '.incompleteStruct' types" % [name] + else: + "'$1' requires '.importc' types to be '.completeStruct'" % [name] proc genNarrow(c: PCtx; n: PNode; dest: TDest) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) @@ -1476,11 +1480,14 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}, m: TMag else: globalError(c.config, n.info, "expandToAst requires a call expression") of mSizeOf: - globalError(c.config, n.info, sizeOfLikeMsg("sizeof")) + let arg = n[1].typ.skipTypes({tyTypeDesc}) + globalError(c.config, n.info, sizeOfLikeMsg("sizeof", tfIncompleteStruct in arg.flags)) of mAlignOf: - globalError(c.config, n.info, sizeOfLikeMsg("alignof")) + let arg = n[1].typ.skipTypes({tyTypeDesc}) + globalError(c.config, n.info, sizeOfLikeMsg("alignof", tfIncompleteStruct in arg.flags)) of mOffsetOf: - globalError(c.config, n.info, sizeOfLikeMsg("offsetof")) + let arg = n[1].typ.skipTypes({tyTypeDesc}) + globalError(c.config, n.info, sizeOfLikeMsg("offsetof", tfIncompleteStruct in arg.flags)) of mRunnableExamples: discard "just ignore any call to runnableExamples" of mDestroy, mTrace: discard "ignore calls to the default destructor" diff --git a/doc/manual.md b/doc/manual.md index 0970d0f5b8..046ef33c4b 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -7996,6 +7996,9 @@ underlying C `struct`:c: in a `sizeof` expression: pure, incompleteStruct.} = object ``` +Attempting to use `sizeof` on an `incompleteStruct` type at compile-time +will error with "'sizeof' cannot be used with '.incompleteStruct' types". + CompleteStruct pragma --------------------- diff --git a/tests/misc/tsizeof_incompleteStruct.nim b/tests/misc/tsizeof_incompleteStruct.nim new file mode 100644 index 0000000000..f3256ccc82 --- /dev/null +++ b/tests/misc/tsizeof_incompleteStruct.nim @@ -0,0 +1,12 @@ +discard """ +errormsg: "'sizeof' cannot be used with '.incompleteStruct' types" +line: 10 +""" + +type + MyStruct {.incompleteStruct.} = object + field: int + +const i = sizeof(MyStruct) + +echo i From 587f90a81678508c1a20469dad1a9ea842590b40 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Sat, 13 Jun 2026 16:17:59 +0800 Subject: [PATCH 16/46] fixes #22122; Unclear error message for raise of a complex expression (#25899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fixes #22122 The commit fixes a bug in Nim's effects checker where raise statements with case/if expressions (commonly from template expansion) failed to track exception types from individual branches. Problem: addRaiseEffect only saw the outermost expression. When a template like getTransportError(err) expanded to a case expression raising 3 different exception types, the compiler only registered the top-level call — missing the branch-level exceptions. Fix (2 files): - compiler/sempass2.nim: Added skipHiddenConv to strip implicit type coercion nodes (nkHiddenStdConv/nkHiddenSubConv) that hide the control flow structure. Added addRaiseEffectsFromExpr that recursively walks into case/if/block/stmtlist expressions to find raise effects in each branch body. Changed the nkRaiseStmt handler to use this new function. - tests/effects/tcase_raises.nim: Test with templates that expand to case expressions raising different exception types, verified via {.raises: [].} pragma. --- compiler/sempass2.nim | 29 ++++++++++++- tests/effects/tcase_raises.nim | 74 ++++++++++++++++++++++++++++++++++ 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 tests/effects/tcase_raises.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 7b2be510f9..0d32930472 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -497,6 +497,33 @@ proc addRaiseEffect(a: PEffects, e, comesFrom: PNode) = if not isDefectException(e.typ): throws(a.exc, e, comesFrom) +proc skipHiddenConv(n: PNode): PNode = + result = n + while true: + case result.kind + of nkHiddenStdConv, nkHiddenSubConv: + result = result[1] + else: break + +proc addRaiseEffectsFromExpr(a: PEffects, e, comesFrom: PNode) = + if e.isNil: + return + case e.kind + of nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr: + if e.len > 0: + addRaiseEffectsFromExpr(a, e.lastSon.skipHiddenConv, comesFrom) + of nkIfExpr, nkIfStmt: + for branch in items(e): + if branch.len > 0: + addRaiseEffectsFromExpr(a, branch.lastSon.skipHiddenConv, comesFrom) + of nkCaseStmt: + for i in 1.. 0: + addRaiseEffectsFromExpr(a, branch.lastSon.skipHiddenConv, comesFrom) + else: + addRaiseEffect(a, e, comesFrom) + proc addTag(a: PEffects, e, comesFrom: PNode) = var aa = a.tags for i in 0.. Date: Sat, 13 Jun 2026 05:54:19 -0500 Subject: [PATCH 17/46] docs: correct the Delegating bind statements example (fixes #19240) (#25890) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #19240. The Manual's "Delegating bind statements" example didn't compile (module B didn't import A, type `O` wasn't exported, and `x: T` couldn't bind to `var O`), and once those were fixed it compiled *without* the `bind` statement — so it didn't demonstrate delegating bind at all. This replaces it with a minimal example that genuinely requires `bind init`: `module main` imports A and B but not C, so `init` is not in scope at the final instantiation of `genericA`; the open `mixin` symbol fails to resolve without `bind init` forwarding it from module B. Verified to fail without `bind` and compile with `bind` under Nim 2.2.10. --- Disclosure: I work with Claude as a co-processor. I understand what I'm submitting and I verified the example against the compiler myself. If you prefer human-only contributions, just say so and I'll close without friction. --- doc/manual.md | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/doc/manual.md b/doc/manual.md index 046ef33c4b..5732947133 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -6123,40 +6123,48 @@ instantiations cross multiple different modules: ```nim # module A + type O* = object + proc genericA*[T](x: T) = mixin init init(x) ``` + ```nim + # module C + import A + + proc init*(x: O) = discard + ``` ```nim - import C - # module B + import A, C + proc genericB*[T](x: T) = - # Without the `bind init` statement C's init proc is - # not available when `genericB` is instantiated: + # Without the `bind init` statement, C's `init` proc is not + # available when `genericA` is instantiated through `genericB` + # from `module main`, which does not import C: bind init genericA(x) ``` - ```nim - # module C - type O = object - proc init*(x: var O) = discard - ``` - ```nim # module main - import B, C + import A, B - genericB O() + genericB(O()) ``` -In module B has an `init` proc from module C in its scope that is not -taken into account when `genericB` is instantiated which leads to the -instantiation of `genericA`. The solution is to `forward`:idx: these -symbols by a `bind` statement inside `genericB`. +Because `genericA` uses `mixin init`, `init` is an open symbol that is +resolved when `genericA` is instantiated. Here `genericA` is instantiated +through `genericB`, whose final instantiation happens in `module main`. +Since `module main` does not import `module C`, `init` is not in scope at +that point, and the instantiation fails with ``undeclared identifier: 'init'``. +The `bind init` statement inside `genericB` forwards the `init` symbol that +is visible in `module B` into the instantiation of `genericA`, which makes +the example compile. This `bind`, which re-exposes a symbol to a nested +generic instantiation, is a `delegating bind`:idx:. Templates From 9d7c0cc68369a5040dc969a6ca6f69f005ee6cf0 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 13 Jun 2026 19:27:22 +0200 Subject: [PATCH 18/46] SSO: add readRawDataStable across all string implementations (#25909) Companion to readRawData whose pointer stays valid across moves/copies of the string. Under --strings:sso it promotes a small inline string to its heap representation; under refc/v2 the data is already heap-resident so it aliases readRawData. Uniform `var string` signature on every backend so code can prepare for --strings:sso without `when declared`. --- changelog.md | 9 +++ lib/system.nim | 7 +++ lib/system/strs_v2.nim | 10 +++ lib/system/strs_v3.nim | 27 +++++++++ tests/system/treadrawdatastable.nim | 94 +++++++++++++++++++++++++++++ 5 files changed, 147 insertions(+) create mode 100644 tests/system/treadrawdatastable.nim diff --git a/changelog.md b/changelog.md index 1790684126..5f1ff21778 100644 --- a/changelog.md +++ b/changelog.md @@ -43,6 +43,15 @@ parameter and result types, not just their source-level shape. Use [//]: # "Additions:" +- Added `system.readRawDataStable`, a companion to `readRawData` that returns a + raw `ptr UncheckedArray[char]` into a string's character data which stays valid + across moves and copies of the string value. It is available under every string + implementation (refc, ARC/ORC and `--strings:sso`) with the same signature, so + code can pin an interior buffer pointer today and be ready for `--strings:sso` + without `when declared` guards. Under `--strings:sso` it promotes a small inline + string to its heap representation first; under the other implementations the data + is already heap-resident, so it is equivalent to `readRawData`. + - `setutils.symmetricDifference` along with its operator version `` setutils.`-+-` `` and in-place version `setutils.toggle` have been added to more efficiently calculate the symmetric difference of bitsets. diff --git a/lib/system.nim b/lib/system.nim index 94fe96ea93..313f90969c 100644 --- a/lib/system.nim +++ b/lib/system.nim @@ -1723,11 +1723,18 @@ when not (notJSnotNims and defined(nimSeqsV2)): let ns = cast[NimString](s) if ns == nil: nil else: cast[ptr UncheckedArray[char]](addr ns.data[start]) + template readRawDataStable*(s: var string; start = 0): ptr UncheckedArray[char] = + ## Same as `readRawData` here: the data lives in a heap `NimStringDesc` at a + ## stable address, so the pointer already survives moves of `s`. Takes `s` by + ## `var` to match the `--strings:sso` version, so code can prepare for that + ## upgrade without `when declared` guards. + readRawData(s, start) else: # JS/nimscript: callers are guarded by whenNotVmJsNims/when not defined(js) proc beginStore*(s: var string; newLen: int; start = 0): ptr UncheckedArray[char] {.inline, noSideEffect, raises: [], tags: [].} = nil proc endStore*(s: var string) {.inline, noSideEffect, raises: [], tags: [].} = discard template readRawData*(s: string; start = 0): ptr UncheckedArray[char] = nil + template readRawDataStable*(s: var string; start = 0): ptr UncheckedArray[char] = nil when not defined(js): template newSeqImpl(T, len) = diff --git a/lib/system/strs_v2.nim b/lib/system/strs_v2.nim index 640e8eeb3f..b0a6264cf8 100644 --- a/lib/system/strs_v2.nim +++ b/lib/system/strs_v2.nim @@ -261,4 +261,14 @@ template readRawData*(s: string; start = 0): ptr UncheckedArray[char] = ## Template ensures no copy of `s`; ptr is valid while `s` is alive. rawDataImpl(cast[ptr NimStringV2](unsafeAddr s), start) +template readRawDataStable*(s: var string; start = 0): ptr UncheckedArray[char] = + ## Like `readRawData`, but the returned pointer additionally survives moves and + ## copies of `s` (while `s` stays alive and is not reassigned). For this string + ## implementation the char data already lives in a heap payload at an address + ## independent of the `string` value itself, so no promotion is needed and this + ## is identical to `readRawData`. Takes `s` by `var` to match the `--strings:sso` + ## version (which promotes a small inline string to the heap), so code written + ## against `readRawDataStable` compiles unchanged under either implementation. + rawDataImpl(cast[ptr NimStringV2](addr s), start) + {.pop.} diff --git a/lib/system/strs_v3.nim b/lib/system/strs_v3.nim index 58462ba522..65c0fa88c9 100644 --- a/lib/system/strs_v3.nim +++ b/lib/system/strs_v3.nim @@ -770,6 +770,33 @@ template readRawData*(s: string; start = 0): ptr UncheckedArray[char] = ## Template ensures no copy of `s` is made; ptr is valid while `s` is alive. rawDataImpl(cast[ptr SmallString](unsafeAddr s), start) +proc readRawDataStable*(s: var string; start = 0): ptr UncheckedArray[char] {.inline.} = + ## Like `readRawData`, but the returned pointer stays valid across moves and + ## copies of `s` (as long as `s` stays alive and is not reassigned). A + ## short/medium string keeps its chars *inline* in the string object, so a + ## plain `readRawData` pointer dangles the moment the object is moved; this + ## promotes `s` to its heap (long) representation first, whose payload address + ## is independent of where the string object itself lives. Use this whenever an + ## interior pointer must outlive the current scope of the owning string (e.g. + ## a cursor cached alongside the buffer it points into). + let ss = cast[ptr SmallString](addr s) + let slen = ssLen(ss[]) + if slen > 0 and slen <= PayloadSize: + # Promote inline/medium to a long heap block so the payload lives at a + # stable address. Mirrors the short/medium -> long transition in `add`. + let newCap = max(slen, resize(slen)) + let p = cast[ptr LongString](alloc(LongStringDataOffset + newCap + 1)) + p.rc = 1 + p.fullLen = slen + p.capImpl = newCap + copyMem(addr p.data[0], inlinePtr(ss[]), slen) + p.data[slen] = '\0' + ss[].more = p + setSSLen(ss[], HeapSlen) + # Hot-prefix cache (bytes 1..AlwaysAvail) already mirrors data[0..AlwaysAvail-1] + # because setSSLen only rewrote byte 0; the inline chars are untouched. + rawDataImpl(ss, start) + # These take `string` (tyString) so the codegen uses them directly, bypassing # strmantle.nim's versions which go through nimStrLen/nimStrAtMutV3 compilerproc calls. proc cmpStrings(a, b: string): int {.compilerproc, inline.} = diff --git a/tests/system/treadrawdatastable.nim b/tests/system/treadrawdatastable.nim new file mode 100644 index 0000000000..0d4aaee54b --- /dev/null +++ b/tests/system/treadrawdatastable.nim @@ -0,0 +1,94 @@ +discard """ + matrix: "--mm:refc; --mm:orc; --mm:orc --strings:sso; --backend:cpp --mm:orc; --backend:js --mm:orc" + output: "OK" +""" + +# Tests for `readRawDataStable` and the SSO static-long-string promotion path. +# `readRawDataStable` is available under every string implementation (refc / v2 / +# v3-sso / js) with the same signature, so the code below compiles unchanged on +# all backends -- the point being that users can prepare for `--strings:sso` +# without `when declared` hacks. + +import std/assertions + +const hasNativeSso = defined(nimsso) and + (defined(gcArc) or defined(gcAtomicArc) or defined(gcOrc) or defined(gcYrc)) + +type + Reader = object + buf: string + p: ptr UncheckedArray[char] + +proc openFromBuffer(buf: sink string): Reader = + # `result` (and thus `buf`) is moved into the caller on return. A plain + # `readRawData` pointer into a small SSO string would dangle after that move; + # `readRawDataStable` pins the buffer to a stable address first. + result = Reader(buf: buf) + result.p = readRawDataStable(result.buf) + +proc testStable() = + when not defined(js): # raw pointers are a degenerate nil no-op on the JS backend + block: # short buffer (kept inline under SSO) survives the move + var r = openFromBuffer("hello") + doAssert r.buf == "hello" + doAssert r.p[0] == 'h' + doAssert r.p[4] == 'o' + # Stable pointer == the live buffer's raw data after the move. + doAssert cast[uint](r.p) == cast[uint](readRawData(r.buf)) + block: # medium buffer (len 12: inline overlay under SSO) + var r = openFromBuffer("hello world!") + doAssert r.p[11] == '!' + block: # already-long buffer: returned as-is (already heap-resident) + var r = openFromBuffer("this is a fairly long string buffer") + doAssert r.p[0] == 't' + doAssert r.p[34] == 'r' + block: # empty string: API is callable (the data pointer is implementation-defined) + var e = "" + discard readRawDataStable(e) + else: + # On JS the API exists and is callable (returns nil) so call sites are portable. + var s = "hello" + discard readRawDataStable(s) + +proc testStaticLongPromotion() = + # Regression for the static-long -> heap promotion: when a string literal + # longer than the inline payload (PayloadSize = 14 under SSO) is first + # mutated, the new heap block must be filled from the full static payload, + # not from the 7-byte inline hot-prefix cache. Reading from the cache copied + # 7 valid chars and then ran off into the `more` pointer bytes -- the bug that + # corrupted .nif index files on Windows bootstrap (see Nimony tstatic_long_add). + # The assertion holds on every backend; only SSO ever risked the corruption. + var content = "(.nif27)\n(index\n" # len 16 + let expected = "(.nif27)\n(index\n" + content.add 'X' # triggers static-long -> heap promotion + doAssert content.len == 17 + doAssert content == expected & "X" + for i in 0 ..< expected.len: + doAssert content[i] == expected[i] + +when hasNativeSso: + # A few SSO-tier-boundary sanity checks (short / medium / long, COW, shrink). + proc testSsoTiers() = + var a = "(.nif27)\n(index\n" # static long + let b = "(.nif27)\n(index\n" + doAssert a == b + a.add 'Z' + doAssert a == "(.nif27)\n(index\nZ" + + var c = "abcdefghijklmnop" # static long, len 16 + var d = c # COW share + d[0] = 'X' + doAssert c == "abcdefghijklmnop" # original untouched + doAssert d == "Xbcdefghijklmnop" + + var e = "abcdefghijklmnop" + e.setLen 3 # shrink below the inline cache size + doAssert e == "abc" + doAssert e.len == 3 +else: + proc testSsoTiers() = discard + +testStable() +testStaticLongPromotion() +testSsoTiers() +echo "OK" From 7171e6f01f846a511a5fad8d1ab24baaee66e308 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sun, 14 Jun 2026 22:35:06 +0200 Subject: [PATCH 19/46] IC: progress (#25879) --- compiler/ast.nim | 86 ++- compiler/ast2nif.nim | 853 ++++++++++++++++++++++++--- compiler/astdef.nim | 20 +- compiler/ccgcalls.nim | 12 +- compiler/ccgexprs.nim | 34 +- compiler/ccgstmts.nim | 7 +- compiler/ccgtypes.nim | 127 +++- compiler/ccgutils.nim | 7 +- compiler/cgen.nim | 275 ++++++++- compiler/cgendata.nim | 17 + compiler/cgmeth.nim | 2 + compiler/cnif.nim | 726 +++++++++++++++++++++++ compiler/commands.nim | 52 +- compiler/deps.nim | 829 ++++++++++++++++++++++---- compiler/enumtostr.nim | 1 + compiler/ic/replayer.nim | 34 ++ compiler/icconfig.nim | 127 ++++ compiler/itemids.nim | 98 +++ compiler/lambdalifting.nim | 18 +- compiler/liftdestructors.nim | 22 +- compiler/lookups.nim | 3 + compiler/lowerings.nim | 6 +- compiler/main.nim | 14 +- compiler/mangleutils.nim | 28 +- compiler/modulegraphs.nim | 369 +++++++++++- compiler/modules.nim | 2 +- compiler/msgs.nim | 3 + compiler/nifbackend.nim | 479 +++++++++++++-- compiler/nim.nim | 9 +- compiler/nimconf.nim | 8 +- compiler/options.nim | 66 +++ compiler/parser.nim | 5 +- compiler/pipelines.nim | 61 +- compiler/pipelineutils.nim | 2 +- compiler/semcall.nim | 19 +- compiler/semdata.nim | 18 +- compiler/semexprs.nim | 58 ++ compiler/semfold.nim | 13 +- compiler/seminst.nim | 59 +- compiler/semstmts.nim | 8 + compiler/semtypes.nim | 18 +- compiler/semtypinst.nim | 55 +- compiler/sighashes.nim | 61 ++ compiler/sigmatch.nim | 13 +- compiler/typekeys.nim | 138 ++++- compiler/vm.nim | 6 +- compiler/vmgen.nim | 10 +- compiler/vmops.nim | 4 +- doc/ic.md | 412 +++++++++---- koch.nim | 55 +- lib/system/mmdisp.nim | 5 +- tests/codegen/titaniummangle_nim.nim | 2 +- tests/generics/mopensymdot.nim | 18 + tests/generics/topensymdot.nim | 12 + tests/js/tcodegendeclproc.nim | 2 +- 55 files changed, 4843 insertions(+), 545 deletions(-) create mode 100644 compiler/cnif.nim create mode 100644 compiler/icconfig.nim create mode 100644 compiler/itemids.nim create mode 100644 tests/generics/mopensymdot.nim create mode 100644 tests/generics/topensymdot.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index b04a102b20..068b3c4f86 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -36,6 +36,13 @@ proc setupProgram*(config: ConfigRef; cache: IdentCache) = when not defined(nimKochBootstrap): program = createDecodeContext(config, cache) +proc setIcMainModule*(fileIdx: FileIndex) = + ## Tells the IC loader which module is being compiled fresh, so that + ## re-exports of that module's symbols by dependencies are not loaded as + ## duplicate stubs. + when not defined(nimKochBootstrap): + ast2nif.setMainModule(program, fileIdx) + template loadSym(s: PSym) = ## Loads a symbol from NIF file if it's in Partial state. when not defined(nimKochBootstrap): @@ -70,6 +77,16 @@ proc backendEnsureMutable*(t: PType) {.inline.} = # ^ IC review this later if t.state == Partial: loadType(t) +proc unsealForTransform*(t: PType) {.inline.} = + ## The transformer/lambda lifting also run inside `nim m` when the VM + ## compiles a LOADED routine (macro evaluation, `getImpl`). Their mutations + ## are process-local — transformed bodies are never written back to a NIF — + ## so downgrade the loaded type to mutable, mirroring the `cmdNifC` loader + ## which loads everything `Complete` for exactly this reason (see + ## `ast2nif.loadedState`). + if t.state == Partial: loadType(t) + if t.state == Sealed: t.state = Complete + proc owner*(s: PSym): PSym {.inline.} = if s.state == Partial: loadSym(s) result = s.ownerFieldImpl @@ -221,7 +238,10 @@ proc position*(s: PSym): int {.inline.} = result = s.positionImpl proc `position=`*(s: PSym, val: int) {.inline.} = - assert s.state != Sealed + # No `Sealed` guard: the VM reuses `position` as a register slot while compiling + # a macro for execution (see `vmgen.genGenericParams`), which under IC may be a + # macro loaded from a NIF file. The macro is run, not code-generated, so this + # scratch mutation is harmless. if s.state == Partial: loadSym(s) s.positionImpl = val @@ -445,9 +465,13 @@ var gconfig {.threadvar.}: Gconfig proc setUseIc*(useIc: bool) = gconfig.useIc = useIc proc comment*(n: PNode): string = - if nfHasComment in n.flags and not gconfig.useIc: - # IC doesn't track comments, see `packed_ast`, so this could fail - result = gconfig.comments[n.nodeId] + if nfHasComment in n.flags: + # NIF-based IC doesn't serialize comments, but the comment table is keyed by + # the node's address (`nodeId`), which is unique among live nodes; a loaded + # node that carries `nfHasComment` simply has no entry here (its comment was + # set in another process), so `getOrDefault` safely returns "" for it while + # in-process VM macro nodes (e.g. newCommentStmtNode) still round-trip. + result = gconfig.comments.getOrDefault(n.nodeId) else: result = "" @@ -478,13 +502,6 @@ proc getPIdent*(a: PNode): PIdent {.inline.} = of nkOpenSymChoice, nkClosedSymChoice, nkOpenSym: a.sons[0].sym.name else: nil -const - moduleShift = when defined(cpu32): 20 else: 24 - -template toId*(a: ItemId): int = - let x = a - (x.module.int shl moduleShift) + x.item.int - template id*(a: PType | PSym): int = toId(a.itemId) type @@ -493,28 +510,44 @@ type symId*: int32 typeId*: int32 sealed*: bool + backendMinted*: bool disambTable*: CountTable[PIdent] -const - PackageModuleId* = -3'i32 - proc idGeneratorFromModule*(m: PSym): IdGenerator = assert m.kind == skModule result = IdGenerator(module: m.itemId.module, symId: m.itemId.item, typeId: 0, disambTable: initCountTable[PIdent]()) result.disambTable.inc m.name +proc idGeneratorForBackend*(m: PSym): IdGenerator = + ## Like `idGeneratorFromModule`, but for IC codegen (`nim nifc`): symbols and + ## types minted fresh during codegen (transf labels/temps, lifted hooks, type + ## copies) must not collide with the itemIds the NIF loader synthesizes for + ## lazily-loaded symbols/types of the same module — those come from a + ## per-module load-order counter that keeps running while codegen mints its + ## own ids. A collision corrupts itemId-keyed tables, e.g. `transf`'s inline + ## iterator mapping then substitutes a random loaded sym (a call's callee) + ## with a `:tmp` block label. Backend-minted ids carry a marker bit in the + ## module half (see `itemids.backendItemId`), so the two id spaces are + ## disjoint by construction. + assert m.kind == skModule + result = IdGenerator(module: m.itemId.module, symId: 0, typeId: 0, + backendMinted: true, disambTable: initCountTable[PIdent]()) + result.disambTable.inc m.name + proc idGeneratorForPackage*(nextIdWillBe: int32): IdGenerator = result = IdGenerator(module: PackageModuleId, symId: nextIdWillBe - 1'i32, typeId: 0, disambTable: initCountTable[PIdent]()) proc nextSymId(x: IdGenerator): ItemId {.inline.} = assert(not x.sealed) inc x.symId - result = ItemId(module: x.module, item: x.symId) + result = if x.backendMinted: backendItemId(x.module, x.symId) + else: itemId(x.module, x.symId) proc nextTypeId*(x: IdGenerator): ItemId {.inline.} = assert(not x.sealed) inc x.typeId - result = ItemId(module: x.module, item: x.typeId) + result = if x.backendMinted: backendItemId(x.module, x.typeId) + else: itemId(x.module, x.typeId) when false: proc nextId*(x: IdGenerator): ItemId {.inline.} = @@ -1043,6 +1076,11 @@ proc newType*(kind: TTypeKind; idgen: IdGenerator; owner: PSym; son: sink PType if result.itemId.module == 55 and result.itemId.item == 2: echo "KNID ", kind writeStackTrace() + when defined(icDbg): + if kind == tyOpenArray: + echo "NEWTYPE openArray id=", id.module, ".", id.item, + " owner=", (if owner != nil: owner.name.s else: "nil") + echo getStackTrace() proc setSons*(dest: PType; sons: sink seq[PType]) {.inline.} = assert dest.kind != tyProc or sons.len <= 1 @@ -1105,10 +1143,19 @@ proc copyType*(t: PType, idgen: IdGenerator, owner: PSym): PType = assignType(result, t) result.symImpl = t.sym # backend-info should not be copied -proc exactReplica*(t: PType): PType = +proc exactReplica*(t: PType; idgen: IdGenerator): PType = + ## Replica that KEEPS `itemId` — the generic-param binding tables + ## (`LayeredIdTable`) key on it, so the copy must keep matching its + ## original — but mints a FRESH `uniqueId`: uniqueId is the SERIALIZATION + ## identity (NIF type names key on it) and must be unique per instance. + ## Replicas sharing the original's uniqueId serialized as duplicate defs + ## under one NIF name; the loader collapsed them into a single type, + ## losing their flag differences (use-site `tfUnresolved` typedescs) or + ## their structure (meta instance bodies shadowing a generic's canonical + ## body). result = PType(kind: t.kind, ownerFieldImpl: t.owner, sizeImpl: defaultSize, alignImpl: defaultAlignment, itemId: t.itemId, - uniqueId: t.uniqueId) + uniqueId: nextTypeId(idgen)) assignType(result, t) result.symImpl = t.sym # backend-info should not be copied @@ -1271,6 +1318,9 @@ proc transitionNoneToSym*(n: PNode) = transitionNodeKindCommon(nkSym) template transitionSymKindCommon*(k: TSymKind) = + # Under IC the symbol may still be an unloaded stub (`skStub`); materialise it + # first so its kind-specific fields (read below as `obj.*`) actually exist. + if s.state == Partial: loadSym(s) let obj {.inject.} = s[] s[] = TSym(kindImpl: k, itemId: obj.itemId, magicImpl: obj.magicImpl, typImpl: obj.typImpl, name: obj.name, infoImpl: obj.infoImpl, ownerFieldImpl: obj.ownerFieldImpl, flagsImpl: obj.flagsImpl, astImpl: obj.astImpl, diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 8847af32b7..c4bab071ee 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -10,8 +10,11 @@ ## AST to NIF bridge. import std / [assertions, tables, sets] -from std / strutils import startsWith +from std / strutils import startsWith, endsWith, contains from std / os import fileExists +from std / syncio import readFile +from std / algorithm import sort +import "../dist/checksums/src/checksums" / sha1 import astdef, idents, msgs, options import lineinfos as astli import pathutils #, modulegraphs @@ -23,6 +26,9 @@ import typekeys import ic / [enum2nif] proc typeToNifSym(typ: PType; config: ConfigRef): string = + # NOTE: uniqueId is the serialization identity and is unique per instance — + # `exactReplica` keeps only itemId shared with its original (see ast.nim) + assert not typ.uniqueId.isBackendMinted result = "`t" result.addInt ord(typ.kind) result.add '.' @@ -30,6 +36,16 @@ proc typeToNifSym(typ: PType; config: ConfigRef): string = result.add '.' result.add modname(typ.uniqueId.module, config) +proc icNifTypeName*(typ: PType; config: ConfigRef): string = + ## The serialized NIF name of a type, recorded next to RTTI data + ## definitions in the cnif artifact so a later run can re-demand the + ## typeinfo when a reused TU still references it (the def-retention + ## check). Backend-minted types have no NIF name. + if typ != nil and not typ.uniqueId.isBackendMinted: + result = typeToNifSym(typ, config) + else: + result = "" + proc toHookIndexEntry*(config: ConfigRef; typeId: ItemId; hookSym: PSym): HookIndexEntry = ## Converts a type ItemId and hook symbol to a HookIndexEntry for the NIF index. let typeSymName = "`t" & $typeId.item & "." & cachedModuleSuffix(config, typeId.module.FileIndex) @@ -144,7 +160,7 @@ const symDefTagName = "sd" typeDefTagName = "td" -let +var sdefTag = registerTag(symDefTagName) tdefTag = registerTag(typeDefTagName) hiddenTypeTag = registerTag(hiddenTypeTagName) @@ -161,19 +177,37 @@ type #writtenSyms: seq[PSym] # symbols written in this module, to be unloaded later writtenPackages: HashSet[string] -const - # Symbol kinds that are always local to a proc and should never have module suffix - skLocalSymKinds = {skParam, skForVar, skResult, skTemp} - proc isLocalSym(sym: PSym): bool {.inline.} = - sym.kindImpl in skLocalSymKinds or - (sym.kindImpl in {skVar, skLet} and {sfGlobal, sfThread} * sym.flagsImpl == {} and - (sym.ownerFieldImpl == nil or sym.ownerFieldImpl.kindImpl != skModule)) + ## Every symbol is emitted as a *global* (module-suffixed) name so that its + ## `sdef` gets an index entry and is resolvable by index lookup even when + ## referenced from a different index entry than the one that physically + ## contains the definition. This matters for symbols shared across entries: + ## generic params of a forward declaration vs its implementation, and proc-type + ## params shared between an enclosing proc and a nested object's proc-type + ## field. The per-module `disamb` counter keeps `name.disamb.module` unique, so + ## globalising cannot cause clashes. This trades index size for correctness; + ## size/speed can be optimised later. + false + +const + PkgMarker = "`pkg" + ## Appended to the ident of `skPackage` symbols in NIF names. A package sym + ## has no module of its own: it is written once into every module NIF that + ## references it, named with that module's suffix and its own (independent) + ## disamb counter. Without the marker it can collide with a module-level + ## symbol of the same name and disamb — e.g. extccomp's `compiler` template + ## vs the `compiler` package — and the module sym's owner then resolves to + ## the wrong symbol on load, producing a cyclic owner chain that hangs every + ## owner-walk (sighashes.hashSym etc.). Backtick cannot appear in a Nim + ## identifier, mirroring the "`t" namespace used by `typeToNifSym`. proc toNifSymName(w: var Writer; sym: PSym): string = ## Generate NIF name for a symbol: local names are `ident.disamb`, ## global names are `ident.disamb.moduleSuffix` + assert not sym.itemId.isBackendMinted result = sym.name.s + if sym.kindImpl == skPackage: + result.add PkgMarker result.add '.' result.addInt sym.disamb if not isLocalSym(sym) and sym.itemId notin w.locals: @@ -183,8 +217,11 @@ proc toNifSymName(w: var Writer; sym: PSym): string = result.add modname(module, w.infos.config) -proc globalName(sym: PSym; config: ConfigRef): string = +proc globalName*(sym: PSym; config: ConfigRef): string = result = sym.name.s + if sym.kindImpl == skPackage: + # stubs store the clean name; the NIF index is keyed by the marked one + result.add PkgMarker result.add '.' result.addInt sym.disamb result.add '.' @@ -221,6 +258,18 @@ proc parseSymName*(s: string): ParsedSymName = dec i return ParsedSymName(name: s, module: "") +proc stubKindAndName(cache: IdentCache; rawName: string): (TSymKind, PIdent) = + ## The user-visible name of a symbol stub must NOT keep NIF-only name + ## decorations: the `PkgMarker` of package symbols would otherwise leak into + ## every reader of `name.s` that runs before the stub is fully loaded + ## (e.g. vmgen's callback keys built from owner chains). The marker also + ## tells us the symbol kind up front, which `globalName` uses to rebuild + ## the marked NIF name for the index lookup. + if rawName.endsWith(PkgMarker): + (skPackage, cache.getIdent(rawName[0 ..< rawName.len - PkgMarker.len])) + else: + (skStub, cache.getIdent(rawName)) + template buildTree(dest: var TokenBuf; tag: TagId; body: untyped) = dest.addParLe tag body @@ -262,6 +311,15 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = dest.addIntLit typ.alignImpl dest.addIntLit typ.paddingAtEndImpl dest.addIntLit typ.itemId.item # nonUniqueId + # `exactReplica` keeps the canonical type's itemId (binding-table key) + # while minting a fresh uniqueId (the NIF name): when the two halves + # name different modules, the loader cannot reconstruct itemId.module + # from the type's name — serialize it explicitly + if typ.itemId.module != typ.uniqueId.module and + not typ.itemId.isBackendMinted: + dest.addStrLit modname(typ.itemId.module, w.infos.config) + else: + dest.addDotToken writeType(w, dest, typ.typeInstImpl) #if typ.kind in {tyProc, tyIterator} and typ.nImpl != nil and typ.nImpl.kind != nkFormalParams: @@ -281,7 +339,14 @@ proc writeTypeDef(w: var Writer; dest: var TokenBuf; typ: PType) = proc writeType(w: var Writer; dest: var TokenBuf; typ: PType) = if typ == nil: dest.addDotToken() - elif typ.itemId.module == w.currentModule and typ.state == Complete: + elif typ.uniqueId.module == w.currentModule and typ.state == Complete: + # Ownership for serialization is decided by `uniqueId`, not `itemId`: the NIF + # name (`typeToNifSym`) and the loader (`createTypeStub`) both key off + # `uniqueId`, so the module that *created* the type (uniqueId.module) must be + # the one that emits its definition. `itemId.module` can be reassigned and + # diverge from `uniqueId.module`; gating on it filed the def in the wrong + # module (or nowhere), leaving dangling references (e.g. `symbol has no + # offset` for a `pointer` type whose itemId.module drifted away). typ.state = Sealed writeTypeDef(w, dest, typ) else: @@ -301,28 +366,19 @@ proc writeLib(w: var Writer; dest: var TokenBuf; lib: PLib) = dest.addStrLit lib.name writeNode w, dest, lib.path -proc collectGenericParams(w: var Writer; n: PNode) = - ## Pre-collect generic param symbols into w.locals before writing the type. - ## This ensures generic params get consistent short names, and their sdefs - ## are written in the type (where lazy loading can find them). - if n == nil: return - case n.kind - of nkSym: - if n.sym != nil and w.inProc > 0: - w.locals.incl(n.sym.itemId) - of nkIdentDefs, nkVarTuple: - for i in 0 ..< max(0, n.len - 2): - collectGenericParams(w, n[i]) - of nkGenericParams: - for child in n: - collectGenericParams(w, child) - else: - discard - proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = dest.addParLe sdefTag, trLineInfo(w, sym.infoImpl) dest.addSymDef pool.syms.getOrIncl(w.toNifSymName(sym)), NoLineInfo - if {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}: + # The `x` marker means "importable as a bare identifier into an importer's + # scope". Object fields carry `sfExported` (so they are visible via `obj.field` + # across modules) but must NOT become bare-importable: otherwise an exported + # field name (e.g. `HSlice.a`, whose type is a generic param `T`) leaks into + # module scope and a template's open/mixin symbol of the same name resolves to + # the field instead of a local, producing "type mismatch: got 'T'". Fields are + # still indexed (for `obj.field` resolution via the loaded object type); they + # are merely not advertised as importable. `skEnumField` stays importable — + # enum values are legitimately usable as bare identifiers. + if sym.kindImpl != skField and {sfExported, sfFromGeneric} * sym.flagsImpl == {sfExported}: dest.addIdent "x" else: dest.addDotToken @@ -353,14 +409,12 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeLib(w, dest, sym.annexImpl) - # For routine symbols, pre-collect generic params into w.locals before writing - # the type. This ensures they get consistent short names, and their sdefs are - # written in the type where lazy loading can find them via extractLocalSymsFromTree. - if sym.kindImpl in routineKinds and sym.astImpl != nil and sym.astImpl.len > genericParamsPos: - inc w.inProc - collectGenericParams(w, sym.astImpl[genericParamsPos]) - dec w.inProc - + # Generic params are written as *global* symbols (with a module suffix) so that + # they get their own index entries and can be looked up lazily. This matters for + # generic routines that have a separate forward declaration and implementation: + # the two share the same generic param symbols, but each is serialized as its own + # index entry. If the params were local, a reference from the implementation's + # entry could not resolve the sdef emitted in the forward declaration's entry. writeType(w, dest, sym.typImpl) writeSym(w, dest, sym.ownerFieldImpl) # Store the AST for routine symbols and constants @@ -403,11 +457,24 @@ proc writeSym(w: var Writer; dest: var TokenBuf; sym: PSym) = proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = if sym == nil: dest.addDotToken() - elif shouldWriteSymDef(w, sym): + return + # Compare lazy-aware, not the raw field: a sym node loaded from a NIF carries + # `typField == nil` plus `nfLazyType`, meaning "my type is the symbol's + # type". Comparing `typField` directly would re-serialize such a node as + # `(ht . sym)` — an explicitly nil node type — and the next loader gets a + # nil-typed node *without* the lazy fallback (semfold & friends crash on + # `n.typ == nil`). Only a genuinely nil node type keeps the explicit form. + # (ast.nim's `typ` accessor is not importable here; replicate its fallback. + # For a still-Partial sym `typImpl` is nil, which also compares equal below + # and yields the plain SymUse form — exactly the lazy round-trip we want.) + var nodeTyp = n.typField + if nodeTyp == nil and nfLazyType in n.flags: + nodeTyp = sym.typImpl + if shouldWriteSymDef(w, sym): sym.state = Sealed - if n.typField != n.sym.typImpl: + if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, trLineInfo(w, n.info): - writeType(w, dest, n.typField) + writeType(w, dest, nodeTyp) writeSymDef(w, dest, sym) else: writeSymDef(w, dest, sym) @@ -415,9 +482,9 @@ proc writeSymNode(w: var Writer; dest: var TokenBuf; n: PNode; sym: PSym) = # NIF has direct support for symbol references so we don't need to use a tag here, # unlike what we do for types! let info = trLineInfo(w, n.info) - if n.typField != n.sym.typImpl: + if nodeTyp != n.sym.typImpl: dest.buildTree hiddenTypeTag, info: - writeType(w, dest, n.typField) + writeType(w, dest, nodeTyp) dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info else: dest.addSymUse pool.syms.getOrIncl(w.toNifSymName(sym)), info @@ -433,9 +500,10 @@ template withNode(w: var Writer; dest: var TokenBuf; n: PNode; body: untyped) = dest.addParRi proc addLocalSym(w: var Writer; n: PNode) = - ## Add symbol from a node to locals set if it's a symbol node - if n != nil and n.kind == nkSym and n.sym != nil and w.inProc > 0: - w.locals.incl(n.sym.itemId) + ## Previously forced proc-local symbols to be written without a module suffix. + ## All symbols are now emitted as global (see `isLocalSym`), so `w.locals` is + ## intentionally left empty. + discard proc addLocalSyms(w: var Writer; n: PNode) = case n.kind @@ -467,12 +535,14 @@ proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = proc trImport(w: var Writer; n: PNode) = for child in n: - if child.kind == nkSym: + if child.kind == nkSym and child.sym.kindImpl == skModule: + # a non-module sym appears for an `import v` inside an unexpanded + # template body (e.g. stew/importops' `when compiles((; import v))`): + # not a dependency edge, the import resolves at the expansion site w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) w.deps.addDotToken # flags w.deps.addDotToken # type let s = child.sym - assert s.kindImpl == skModule let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex) w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi @@ -495,21 +565,51 @@ proc trExport(w: var Writer; n: PNode) = w.deps.addSymUse pool.syms.getOrIncl(w.toNifSymName(s)), NoLineInfo w.deps.addParRi -let replayTag = registerTag("replay") -let repConverterTag = registerTag("repconverter") -let repDestroyTag = registerTag("repdestroy") -let repWasMovedTag = registerTag("repwasmoved") -let repCopyTag = registerTag("repcopy") -let repSinkTag = registerTag("repsink") -let repDupTag = registerTag("repdup") -let repTraceTag = registerTag("reptrace") -let repDeepCopyTag = registerTag("repdeepcopy") -let repEnumToStrTag = registerTag("repenumtostr") -let repMethodTag = registerTag("repmethod") -#let repClassTag = registerTag("repclass") -let includeTag = registerTag("include") -let importTag = registerTag("import") -let implTag = registerTag("implementation") +var replayTag = registerTag("replay") +var repConverterTag = registerTag("repconverter") +var repDestroyTag = registerTag("repdestroy") +var repWasMovedTag = registerTag("repwasmoved") +var repCopyTag = registerTag("repcopy") +var repSinkTag = registerTag("repsink") +var repDupTag = registerTag("repdup") +var repTraceTag = registerTag("reptrace") +var repDeepCopyTag = registerTag("repdeepcopy") +var repEnumToStrTag = registerTag("repenumtostr") +var repMethodTag = registerTag("repmethod") +#var repClassTag = registerTag("repclass") +var includeTag = registerTag("include") +var importTag = registerTag("import") +var implTag = registerTag("implementation") +var reexpModTag = registerTag("reexpmod") + +proc registerNifAstTags*() = + ## (Re)registers ast2nif's NIF tags explicitly. The top-level `registerTag` + ## initializers above depend on `nifstreams.pool` having been initialized + ## FIRST (`pool = createLiterals(TagData)` in nifstreams' module init) — an + ## inter-module init-order requirement. The IC-built compiler currently emits + ## module init calls in a different order, so the initializers registered + ## into a pool that was subsequently replaced: the tag ids then denoted + ## builtin tags (`replay` came out as `deref`, `repdestroy` as `pat`, ...) + ## and every written NIF was silently corrupted. Called from `nim.nim` + ## before any command runs; idempotent (`getOrIncl` by name). + sdefTag = registerTag(symDefTagName) + tdefTag = registerTag(typeDefTagName) + hiddenTypeTag = registerTag(hiddenTypeTagName) + replayTag = registerTag("replay") + repConverterTag = registerTag("repconverter") + repDestroyTag = registerTag("repdestroy") + repWasMovedTag = registerTag("repwasmoved") + repCopyTag = registerTag("repcopy") + repSinkTag = registerTag("repsink") + repDupTag = registerTag("repdup") + repTraceTag = registerTag("reptrace") + repDeepCopyTag = registerTag("repdeepcopy") + repEnumToStrTag = registerTag("repenumtostr") + repMethodTag = registerTag("repmethod") + includeTag = registerTag("include") + importTag = registerTag("import") + implTag = registerTag("implementation") + reexpModTag = registerTag("reexpmod") proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n == nil: @@ -589,13 +689,28 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = if n[namePos].kind == nkSym: ast = n[namePos].sym.astImpl if ast == nil: ast = n - else: skipParams = true + else: + # params can only be recovered from `sym.typ.n` if the routine + # was actually semchecked. A routine nested in a TEMPLATE body + # (e.g. faststreams' `proc consumer(bytesVar: openArray[byte]) + # {.gensym.}` inside `consumeOutputs`) has a sym but a nil type — + # its params exist only in the AST; dropping them broke the + # template-param substitution at expansion ("undeclared + # identifier" for the injected name). + skipParams = n[namePos].sym.typImpl != nil w.withNode dest, ast: for i in 0 ..< ast.len: if i == paramsPos and skipParams: - # Parameter are redundant with s.typ.n and even dangerous as for generic instances - # we do not adapt the symbols properly - addDotToken(dest) + # Parameters are redundant with s.typ.n (and re-emitting their syms + # is dangerous for generic instances — we do not adapt the symbols + # properly). Emit an `nkEmpty` placeholder rather than a dot token: + # a dot loads back as a `nil` son, but ast children must be real + # nodes — the loaded routine ast is walked by passes (lambdalifting, + # liftdestructors, transf) that dereference `ast[paramsPos]`, and + # `nkEmpty` is the canonical empty slot. The actual params are + # recovered from `sym.typ.n` where needed. + dest.addParLe pool.tags.getOrIncl(toNifTag(nkEmpty)), NoLineInfo + dest.addParRi else: writeNode(w, dest, ast[i], forAst) dec w.inProc @@ -697,7 +812,10 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) content.addParRi() of MethodEntry: - discard "to implement" + content.addParLe repMethodTag, NoLineInfo + content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) + content.add symToken(pool.syms.getOrIncl(w.toNifSymName(op.sym)), NoLineInfo) + content.addParRi() of EnumToStrEntry: content.addParLe repEnumToStrTag, NoLineInfo content.add strToken(pool.strings.getOrIncl(op.key), NoLineInfo) @@ -706,9 +824,385 @@ proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = of GenericInstEntry: discard "will only be written later to ensure it is materialized" +# --------------------------- Interface cookie --------------------------- +# +# Port of Nimony's `processForChecksum` (dist/nimony/src/lib/nifindexes.nim): +# ONE checksum per module over the importer-visible surface, stored in a tiny +# `.iface.nif` sidecar written OnlyIfChanged. deps.nim points the +# dependents' `nim_m` build edges at the sidecar instead of the bulky semmed +# NIF, so nifmake's mtime pruning stops the m-step cascade at the first +# module whose interface did not change. +# +# Hashed (importer-visible surface): +# - import/include/export entries, `(replay ...)` macro-cache actions and the +# rep* hook/converter/enumtostr registrations (all eagerly consumed by every +# importer's sem via processTopLevel/loadTransitiveHooks). +# - every EXPORTED `(sd ...)`: full content for consts/types/vars/lets; for +# EVERY routine kind (plain procs, templates, macros, iterators, generics, +# `inline` procs alike) only the SIGNATURE — the body is skipped. A routine +# body is invisible to a dependent's SEM unless the dependent expands / +# instantiates / VM-runs it, and each of those records a NeedsImpl (strong) +# edge gating the dependent on this module's IMPL cookie instead (see +# `cookieSd`). This keeps the iface cookie body-insensitive, so a body edit +# re-sems only the modules that actually consumed that body — not every +# importer (the old model folded inline-semantics bodies into the iface +# cookie, re-semming all importers on any such body edit). +# - nothing else: private defs and top-level init code are invisible to +# importers' sem (their effects on dependents' CODEGEN — and the codegen +# effect of inline iterator/proc body edits — are covered by the nifc +# backend's transitive NIF-mtime invalidation, which is unchanged). +# +# Token-content hashing only — line infos never enter the hash. Names DEFINED +# inside a hashed (sd) (params, locals, the embedded `(td `tK.item.mod)` defs) +# are replaced by per-sd ordinals and module-local `tK.item references are +# replaced by their structural td hash: both carry process-local mint counters +# that shift file-wide when an unrelated body creates a new type (measured: +# a single new instantiation renumbered every later signature), while +# dependents never reference them by name (verified over a full compiler +# cache: cross-module refs hit only top-level routine names). +# +# The cookie finally mixes in the DIRECT dependencies' sidecar contents +# ("hash chaining"): an interface change then propagates transitively +# level-by-level even when an intermediate module's own surface is unchanged +# (its sem still consumed the dep's surface, e.g. via the transitive hook +# replay). Chaining also guarantees a fired rule refreshes its sidecar mtime, +# which nifmake's max-output `needsRebuild` needs to not re-fire forever. +# +# The IMPL cookie (`.impl.nif`) complements it: a line-info-free hash +# of the module's ENTIRE content with the iface cookie mixed in. Dependents +# that consumed this module's bodies at compile time (recorded in the +# `.edges.nif` sidecar; see `ModuleGraph.icImplDeps`) are gated on it instead. + +type + CookieCtx = object + selfSuffix: string + tdRanges: Table[SymId, int] # td sym -> start of its first (td ...) tree + memo: Table[SymId, string] # td sym -> structural digest + expanding: HashSet[SymId] # cycle guard for recursive td expansion + depSuffixes: seq[string] # module suffixes of the direct imports + +proc nextTree(buf: TokenBuf; i: int): int = + ## Index just past the atom or balanced subtree starting at `i`. + result = i+1 + if buf[i].kind != ParLe: return + var nested = 0 + var j = i + while j < buf.len: + case buf[j].kind + of ParLe: inc nested + of ParRi: + dec nested + if nested == 0: return j+1 + else: discard + inc j + result = buf.len + +proc updateAtom(s: var Sha1State; t: PackedToken) = + # mirrors nimony's nifchecksums.update: token content only, no line infos + case t.kind + of ParLe: + s.update "(" + s.update pool.tags[t.tagId] + of ParRi: s.update ")" + of Ident: + s.update " " + s.update pool.strings[t.litId] + of StringLit: + s.update " \"" + s.update pool.strings[t.litId] + of IntLit: + s.update " " + s.update $pool.integers[t.intId] + of UIntLit: + s.update " " + s.update $pool.uintegers[t.uintId] + of FloatLit: + # hash the bit pattern, not a formatted float (no formatting variance) + s.update " f" + s.update $cast[uint64](pool.floats[t.floatId]) + of CharLit: + s.update " c" + s.update $t.uoperand + of DotToken: s.update "." + of UnknownToken: s.update "?" + of EofToken: s.update "!" + of Symbol, SymbolDef: discard "handled by hashRegion" + +proc isModuleLocalName(c: CookieCtx; name: string): bool = + let sn = parseSymName(name) + result = sn.module.len == 0 or sn.module == c.selfSuffix + +proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; + start, theEnd: int; skipFrom = -1; skipTo = -1; + keepFirstDefLiteral = false) + +proc expandTd(c: var CookieCtx; buf: TokenBuf; name: SymId): string = + ## Structural digest of a module-local type def: hashes the `(td ...)` tree + ## instead of the volatile `tK.item counter name. Memoized; cycles fall back + ## to the literal name (sound — at worst a spurious cookie change). + if c.memo.hasKey(name): return c.memo[name] + if not c.tdRanges.hasKey(name) or c.expanding.contains(name): + return pool.syms[name] + c.expanding.incl name + let start = c.tdRanges[name] + var sub = newSha1State() + hashRegion(sub, c, buf, start, nextTree(buf, start)) + result = "&" & $SecureHash(sub.finalize()) + c.expanding.excl name + c.memo[name] = result + +proc hashRegion(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; + start, theEnd: int; skipFrom = -1; skipTo = -1; + keepFirstDefLiteral = false) = + # pass 1: assign ordinals to every symbol DEFINED in the hashed region + # (params, locals, embedded type defs). The region's own top-level name + # (first SymbolDef) stays literal when requested — it is what importers + # reference. + var ords = initTable[SymId, int]() + var first = keepFirstDefLiteral + var i = start + while i < theEnd: + if i == skipFrom: + i = skipTo + continue + if buf[i].kind == SymbolDef: + let sym = buf[i].symId + if first: + first = false + elif isModuleLocalName(c, pool.syms[sym]) and not ords.hasKey(sym): + ords[sym] = ords.len + inc i + # pass 2: hash + first = keepFirstDefLiteral + i = start + while i < theEnd: + if i == skipFrom: + i = skipTo + continue + let t = buf[i] + if t.kind in {Symbol, SymbolDef}: + let sym = t.symId + let name = pool.syms[sym] + s.update(if t.kind == SymbolDef: " :" else: " ") + if t.kind == SymbolDef and first: + first = false + s.update name + elif ords.hasKey(sym): + s.update "%" + s.update $ords[sym] + elif name.startsWith("`t") and isModuleLocalName(c, name): + s.update expandTd(c, buf, sym) + else: + s.update name + else: + updateAtom s, t + inc i + +proc cookieSd(s: var Sha1State; c: var CookieCtx; buf: TokenBuf; start: int): int = + ## Contributes one `(sd ...)` subtree to the cookie; returns the index past it. + result = nextTree(buf, start) + if buf[start+1].kind != SymbolDef: return + let marker = buf[start+2] + if not (marker.kind == Ident and pool.strings[marker.litId] == "x"): + return # not importable -> invisible to dependents' sem (nimony parity) + # field layout, see writeSymDef: kind magic flags options offset position + # annex type owner ast loc constraint instantiatedFrom + var fields: array[13, int] = default(array[13, int]) + var i = start + 3 + for f in 0 ..< 13: + fields[f] = i + i = nextTree(buf, i) + var kind = skUnknown + {.cast(uncheckedAssign).}: + kind = parse(TSymKind, pool.tags[buf[fields[0]].tagId]) + var skipFrom = -1 + var skipTo = -1 + if kind in routineKinds: + # Routines contribute their SIGNATURE only to the iface cookie. A routine + # body is invisible to a dependent's SEM unless the dependent expands, + # instantiates, or VM-runs it — and each of those records a NeedsImpl + # (strong) edge that gates the dependent on this module's IMPL cookie + # instead (templates -> semTemplateExpr, generics -> generateInstance, + # macros/compile-time procs -> the VM's genProc, getImpl -> opcGetImpl). + # Inline iterators and `inline`-callconv procs are inlined at codegen; the + # nifc backend's transitive NIF-mtime invalidation re-codegens their users. + # So no routine body needs to live in the iface cookie. + let ast = fields[9] + if buf[ast].kind == ParLe: + # skip son `bodyPos` (6) of the routine ast tree; NOT the last element — + # sem appends the result sym at `resultPos` (7) after the body. + let astEnd = nextTree(buf, ast) + var p = ast + 1 # the flags atom + var ok = true + for _ in 0 ..< 2 + bodyPos: # flags, type, sons 0..5 + p = nextTree(buf, p) + if p >= astEnd - 1: + ok = false + break + if ok: + skipFrom = p + skipTo = nextTree(buf, p) + # non-routine kinds (consts carry their value, types their structure incl. + # default field values): hash everything. + hashRegion(s, c, buf, start, result, skipFrom, skipTo, keepFirstDefLiteral = true) + +proc scanStmtsForCookie(s: var Sha1State; c: var CookieCtx; buf: TokenBuf) = + ## Walks the whole written module, hashing only the importer-visible pieces; + ## unknown structure is descended into (var/let/type section wrappers, + ## top-level code) but contributes nothing itself — nimony-style. + let exportTag = pool.tags.getOrIncl(toNifTag(nkExportStmt)) + let exportExceptTag = pool.tags.getOrIncl(toNifTag(nkExportExceptStmt)) + var i = 0 + while i < buf.len: + let t = buf[i] + if t.kind == ParLe: + let tid = t.tagId + if tid == sdefTag: + i = cookieSd(s, c, buf, i) + elif tid == implTag: + i = nextTree(buf, i) + elif tid == replayTag or tid == repConverterTag or tid == repDestroyTag or + tid == repWasMovedTag or tid == repCopyTag or tid == repSinkTag or + tid == repDupTag or tid == repTraceTag or tid == repDeepCopyTag or + tid == repEnumToStrTag or tid == repMethodTag or + tid == exportTag or tid == exportExceptTag or tid == includeTag: + let e = nextTree(buf, i) + hashRegion(s, c, buf, i, e) + i = e + elif tid == importTag: + let e = nextTree(buf, i) + hashRegion(s, c, buf, i, e) + for j in i ..< e: + if buf[j].kind == StringLit: + let suffix = pool.strings[buf[j].litId] + if suffix notin c.depSuffixes: c.depSuffixes.add suffix + i = e + else: + inc i # descend without hashing + else: + inc i + +proc icGroupSuffixes(config: ConfigRef): HashSet[string] = + ## Module suffixes of the --icGroup cycle members compiled by this very + ## process (their sidecars are being produced concurrently, so neither + ## chaining nor edge recording may depend on them). + result = initHashSet[string]() + for p in config.icGroup: + result.incl cachedModuleSuffix(config, fileInfoIdx(config, AbsoluteFile p)) + +proc writeCookieFile(config: ConfigRef; selfSuffix, tag, hex, ext: string) = + var dest = createTokenBuf(4) + dest.addParLe pool.tags.getOrIncl(tag), NoLineInfo + dest.addStrLit hex + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ext).string + writeFile(dest, path, OnlyIfChanged) + +proc writeIfaceCookie(config: ConfigRef; thisModule: int32; buf: TokenBuf): string = + let selfSuffix = modname(thisModule, config) + var c = CookieCtx(selfSuffix: selfSuffix) + # pre-pass: first (td ...) occurrence per type name, wherever it is embedded + var i = 0 + while i < buf.len: + if buf[i].kind == ParLe and buf[i].tagId == tdefTag and i+1 < buf.len and + buf[i+1].kind == SymbolDef: + let nm = buf[i+1].symId + if not c.tdRanges.hasKey(nm): c.tdRanges[nm] = i + inc i + var s = newSha1State() + scanStmtsForCookie(s, c, buf) + # chain the direct deps' cookies; co-members of an --icGroup cycle are + # excluded (their sidecars are being produced by this very rule — chaining + # them would make the hash depend on within-group write order). + let groupSuffixes = icGroupSuffixes(config) + for dep in c.depSuffixes: + if dep == selfSuffix or dep in groupSuffixes: continue + let depIface = toGeneratedFile(config, AbsoluteFile(dep), ".iface.nif").string + s.update "|" + s.update dep + s.update ":" + s.update(try: readFile(depIface) except IOError, OSError: "") + result = $SecureHash(s.finalize()) + writeCookieFile(config, selfSuffix, "iface", result, ".iface.nif") + +proc writeImplCookie(config: ConfigRef; thisModule: int32; buf: TokenBuf; + ifaceHex: string) = + ## The implementation cookie: a line-info-free hash of the module's ENTIRE + ## serialized content (private defs and routine bodies included), with the + ## module's own iface cookie mixed in so impl sensitivity is a strict + ## superset of iface sensitivity (incl. the chained dep ifaces — a NeedsImpl + ## edge REPLACES the iface edge, it must not lose its triggers). Dependents + ## that consumed this module's bodies at compile time are gated on this file + ## instead of the iface cookie. Comment-only edits move neither cookie. + ## No id normalization here: a counter shift implies some real content + ## change elsewhere in the module, which flips the hash anyway — and any + ## body change is exactly what NeedsImpl dependents must see. + let selfSuffix = modname(thisModule, config) + var s = newSha1State() + for i in 0 ..< buf.len: + let t = buf[i] + if t.kind in {Symbol, SymbolDef}: + s.update(if t.kind == SymbolDef: " :" else: " ") + s.update pool.syms[t.symId] + else: + updateAtom s, t + s.update "|iface:" + s.update ifaceHex + writeCookieFile(config, selfSuffix, "impl", $SecureHash(s.finalize()), ".impl.nif") + +proc writeEdgesFile(config: ConfigRef; thisModule: int32; implDeps: seq[int]) = + ## Records which modules' bodies this compilation consumed at compile time + ## (`ModuleGraph.icImplDeps`): the NeedsImpl edge set. deps.nim reads this + ## sidecar when regenerating the build file and gates this module on those + ## dependencies' IMPL cookies instead of their iface cookies. + let selfSuffix = modname(thisModule, config) + let groupSuffixes = icGroupSuffixes(config) + var suffixes: seq[string] = @[] + for id in implDeps: + if id == thisModule.int: continue + let suffix = cachedModuleSuffix(config, FileIndex id) + if suffix.len == 0 or suffix == selfSuffix or suffix in groupSuffixes: + continue + if suffix notin suffixes: suffixes.add suffix + sort suffixes + var dest = createTokenBuf(4 + 2*suffixes.len) + dest.addParLe pool.tags.getOrIncl("edges"), NoLineInfo + for suffix in suffixes: + dest.addStrLit suffix + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".edges.nif").string + # Deliberately ALWAYS written (unlike every other output of the nim_m rule): + # nothing gates on this file's mtime — deps.nim only reads its content — so + # it doubles as the rule's freshness stamp. nifmake's `needsRebuild` takes + # the freshest output as proof of "ran since the inputs changed"; without an + # always-written output a rule whose re-run produces only content-identical + # (mtime-preserved) files would re-fire on every warm build (e.g. after an + # edit was reverted). Nimony's analog is its always-written `.s.nif`. + writeFile(dest, path) + +proc writeSemDeps*(config: ConfigRef; thisModule: int32; importPaths: seq[string]) = + ## The module's REAL direct imports as `nim m` sem resolved them — static + ## plus any a macro generated — recorded as full source paths. `nim ic` reads + ## this `.s.deps.nif` to re-derive the build graph: imports the static scanner + ## missed become new nodes (replacing the old build-failure discovery loop), + ## and `when false` imports the scanner over-included are pruned. Always + ## written so it is current after every successful sem (like `.edges`). + let selfSuffix = modname(thisModule, config) + var paths = importPaths + sort paths + var dest = createTokenBuf(4 + 2*paths.len) + dest.addParLe pool.tags.getOrIncl("semdeps"), NoLineInfo + for p in paths: + dest.addStrLit p + dest.addParRi + let path = toGeneratedFile(config, AbsoluteFile(selfSuffix), ".s.deps.nif").string + writeFile(dest, path) + proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; opsLog: seq[LogEntry]; - replayActions: seq[PNode] = @[]) = + replayActions: seq[PNode] = @[]; + implDeps: seq[int] = @[]; + reexportedModules: seq[(string, string)] = @[]) = var w = Writer(infos: LineInfoWriter(config: config), currentModule: thisModule) var content = createTokenBuf(300) @@ -729,6 +1223,17 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; var bottom = createTokenBuf(300) w.writeToplevelNode content, bottom, n + # Re-exported MODULES (`import x; export x`): semExport puts only x's + # member syms into the nkExportStmt; the module sym itself reaches the + # exporter's interface via `reexportSym` and acts as a QUALIFIER there + # (`asmm.x86.nd`). Serialize (name, suffix) pairs so the loader can + # rebuild that part of the interface. + for (mname, msuffix) in reexportedModules: + w.deps.addParLe reexpModTag, NoLineInfo + w.deps.addStrLit mname + w.deps.addStrLit msuffix + w.deps.addParRi + # the implTag is used to tell the loader that the # bottom of the file is the implementation of the module: content.addParLe implTag, NoLineInfo @@ -758,7 +1263,15 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; dest.addParRi() - writeFile(dest, d) + # OnlyIfChanged keeps the mtime of content-identical rewrites: nifmake's + # mtime-based `needsRebuild` then prunes the rebuild cascade level by + # level, and the nifc backend can trust "semmed NIF older than the cnif + # artifact" as an honest per-module unchanged stamp. + writeFile(dest, d, OnlyIfChanged) + if not isDefined(config, "icNoIfaceGate"): + let ifaceHex = writeIfaceCookie(config, thisModule, dest) + writeImplCookie(config, thisModule, dest, ifaceHex) + writeEdgesFile(config, thisModule, implDeps) # --------------------------- Loader (lazy!) ----------------------------------------------- @@ -819,6 +1332,8 @@ type symCounter: int32 index: Table[string, NifIndexEntry] # Simple embedded index for offsets suffix: string + contentStart: int # stream offset of the module body, so a full-AST load can + # rewind after lazy symbol loads moved the cursor DecodeContext* = object infos: LineInfoWriter @@ -827,11 +1342,32 @@ type syms: Table[string, (PSym, NifIndexEntry)] mods: Table[FileIndex, NifModule] cache: IdentCache + mainModuleSuffix: string + ## Mangled module name of the module being compiled fresh (cmdM). Symbols + ## belonging to it that are re-exported by a dependency must NOT be loaded + ## as stubs, otherwise they collide with the freshly compiled originals. proc createDecodeContext*(config: ConfigRef; cache: IdentCache): DecodeContext = ## Supposed to be a global variable result = DecodeContext(infos: LineInfoWriter(config: config), cache: cache) +proc setMainModule*(c: var DecodeContext; fileIdx: FileIndex) = + ## Records the module that is being compiled fresh so that re-exports of its + ## own symbols by dependencies are not turned into duplicate stubs. + c.mainModuleSuffix = modname(fileIdx.int, c.infos.config) + +proc getMainModuleSuffix*(c: DecodeContext): string {.inline.} = + c.mainModuleSuffix + +proc loadedState(c: DecodeContext): ItemState {.inline.} = + ## State to give a freshly loaded symbol or type. During the C code generation + ## phase (`nim nifc`) the backend (lambda lifting, the transformer, etc.) + ## legitimately mutates the loaded entities and never writes them back to a NIF, + ## so they must be mutable (`Complete`). During semantic checking (`nim m`) a + ## loaded entity belongs to an already-compiled dependency and must stay + ## `Sealed` so accidental mutations are caught. + if c.infos.config.cmd == cmdNifC: Complete else: Sealed + proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifIndexEntry; buf: var TokenBuf): Cursor = let s = addr c.mods[module].stream @@ -895,7 +1431,10 @@ proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): "whose NIF file hasn't been written yet." var stream = nifstreams.open(modFile) let index = readEmbeddedIndex(stream) - c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix) + # `readEmbeddedIndex` leaves the cursor at the start of the module body. + let contentStart = offset(stream.r) + c.mods[result] = NifModule(stream: stream, index: index, suffix: suffix, + contentStart: contentStart) proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifIndexEntry = let ii = addr c.mods[module].index @@ -920,14 +1459,17 @@ proc createTypeStub(c: var DecodeContext; t: SymId): PType = k = k * 10 + name[i].ord - ord('0') inc i if i < name.len and name[i] == '.': inc i - var itemId = 0'i32 + var itemVal = 0'i32 while i < name.len and name[i] in {'0'..'9'}: - itemId = itemId * 10'i32 + int32(name[i].ord - ord('0')) + itemVal = itemVal * 10'i32 + int32(name[i].ord - ord('0')) inc i if i < name.len and name[i] == '.': inc i let suffix = name.substr(i) - let id = ItemId(module: moduleId(c, suffix).int32, item: itemId) - let offs = c.getOffset(id.module.FileIndex, name) + let id = itemId(moduleId(c, suffix).int32, itemVal) + let ii = addr c.mods[id.module.FileIndex].index + let offs = ii[].getOrDefault(name) + if offs.offset == 0: + raiseAssert "symbol has no offset: " & name result = PType(itemId: id, uniqueId: id, kind: TTypeKind(k), state: Partial) c.types[name] = (result, offs) @@ -955,7 +1497,7 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s let module = moduleId(c, thisModule) let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: module.int32, item: val[]) + let id = itemId(module.int32, val[]) let sym = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Complete) localSyms[symName] = sym @@ -963,7 +1505,7 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s # We're currently at the `(sd` position, need to skip to SymbolDef inc n # skip past `sd` tag to get to SymbolDef loadSymFromCursor(c, sym, n, thisModule, localSyms) - sym.state = Sealed # mark as fully loaded + sym.state = c.loadedState # mark as fully loaded # Continue processing - loadSymFromCursor already advanced n past the closing `)` continue inc depth @@ -988,7 +1530,7 @@ proc loadTypeStub(c: var DecodeContext; n: var Cursor; localSyms: var Table[stri let s = n.firstSon.symId result = createTypeStub(c, s) if result.state == Partial: - result.state = Sealed # Mark as loaded to prevent loadType from re-loading with empty localSyms + result.state = c.loadedState # Mark as loaded to prevent loadType from re-loading with empty localSyms loadTypeFromCursor(c, n, result, localSyms) else: skip n # Type already loaded, skip over the td block @@ -1014,10 +1556,11 @@ proc loadSymStub(c: var DecodeContext; t: SymId; thisModule: string; let module = moduleId(c, sn.module) let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: module.int32, item: val[]) + let id = itemId(module.int32, val[]) let offs = c.getOffset(module, symAsStr) - result = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) + let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) + result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; @@ -1034,7 +1577,8 @@ proc loadSymStub(c: var DecodeContext; n: var Cursor; thisModule: string; skip n result = loadSymStub(c, s, thisModule, localSyms) else: - raiseAssert "sym expected but got " & $n.kind + raiseAssert "sym expected but got " & $n.kind & ( + if n.kind == Ident: " '" & pool.strings[n.litId] & "'" else: "") proc isStub*(t: PType): bool {.inline.} = t.state == Partial proc isStub*(s: PSym): bool {.inline.} = s.state == Partial @@ -1097,7 +1641,14 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms loadField t.sizeImpl loadField t.alignImpl loadField t.paddingAtEndImpl - loadField t.itemId.item # nonUniqueId + t.itemId = itemId(t.itemId.module, loadAtom(int32, n)) # nonUniqueId + if n.kind == StringLit: + # itemId.module differs from uniqueId.module (an `exactReplica` of a + # foreign type): restore the canonical module half + t.itemId = itemId(int32(moduleId(c, pool.strings[n.litId])), t.itemId.item) + inc n + elif n.kind == DotToken: + inc n t.typeInstImpl = loadTypeStub(c, n, localSyms) t.nImpl = loadNode(c, n, typesModule, localSyms) @@ -1112,7 +1663,7 @@ proc loadTypeFromCursor(c: var DecodeContext; n: var Cursor; t: PType; localSyms proc loadType*(c: var DecodeContext; t: PType) = if t.state != Partial: return - t.state = Sealed + t.state = c.loadedState var buf = createTokenBuf(30) let typeName = typeToNifSym(t, c.infos.config) var n = cursorFromIndexEntry(c, t.itemId.module.FileIndex, c.types[typeName][1], buf) @@ -1159,6 +1710,11 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: s.kindImpl = parse(TSymKind, pool.tags[n.tagId]) inc n + if s.kindImpl == skPackage and s.name.s.endsWith(PkgMarker): + # Fallback: stubs are normally created with the clean name already + # (see stubKindAndName); strip the NIF-only marker if one slipped through. + s.name = c.cache.getIdent(s.name.s[0 ..< s.name.s.len - PkgMarker.len]) + case s.kindImpl of skLet, skVar, skField, skForVar: s.guardImpl = loadSymStub(c, n, thisModule, localSyms) @@ -1199,7 +1755,7 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: proc loadSym*(c: var DecodeContext; s: PSym) = if s.state != Partial: return - s.state = Sealed + s.state = c.loadedState var buf = createTokenBuf(30) let symsModule = s.itemId.module.FileIndex let nifname = globalName(s, c.infos.config) @@ -1285,14 +1841,14 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; let module = moduleId(c, thisModule) let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: module.int32, item: val[]) + let id = itemId(module.int32, val[]) sym = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Complete) localSyms[symName] = sym # register for later references # Now fully load the symbol from the sdef inc n # skip `sd` tag loadSymFromCursor(c, sym, n, thisModule, localSyms) - sym.state = Sealed # mark as fully loaded + sym.state = c.loadedState # mark as fully loaded result = newSymNode(sym, info) else: sym = c.loadSymStub(name.symId, thisModule, localSyms) @@ -1391,8 +1947,9 @@ proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; let val = addr c.mods[symModule].symCounter inc val[] - let id = ItemId(module: symModule.int32, item: val[]) - result = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) + let id = itemId(symModule.int32, val[]) + let (stubKind, stubName) = stubKindAndName(c.cache, sn.name) + result = PSym(itemId: id, kindImpl: stubKind, name: stubName, disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, entry) proc extractBasename(nifName: string): string = @@ -1428,6 +1985,29 @@ proc populateInterfaceTablesFromIndex(c: var DecodeContext; module: FileIndex; # Move index table back c.mods[module].index = move indexTab +proc moduleSymbolStubs*(c: var DecodeContext; module: FileIndex): seq[PSym] = + ## Stubs for every non-type symbol serialized in `module`'s NIF index. The + ## per-module backend uses this to emit the routines a module OWNS: procs are + ## serialized as `(sd ...)` symbol-defs and loaded lazily, never as + ## `nkProcDef` statements in the top-level stmt list, so `genTopLevelStmt` + ## alone never reaches them — without this, a routine called only from other + ## modules would be emitted by nobody once the demanding module merely + ## prototypes it. + ## + ## Returns lazy stubs: the index table is moved out while iterating (loading a + ## symbol can register new modules and invalidate the iterator), so the caller + ## forces full load (`.kind`, `.ast`) and filters AFTER this returns, with the + ## index back in place. + result = @[] + if not c.mods.hasKey(module): return + var indexTab = move c.mods[module].index + let thisModule = c.mods[module].suffix + for nifName, entry in indexTab: + if nifName.startsWith("`t"): continue # types are not routines + let sym = loadSymFromIndexEntry(c, module, nifName, entry, thisModule) + if sym != nil: result.add sym + c.mods[module].index = move indexTab + proc toNifFilename*(conf: ConfigRef; f: FileIndex): string = let suffix = moduleSuffix(conf, f) result = toGeneratedFile(conf, AbsoluteFile(suffix), ".nif").string @@ -1456,7 +2036,7 @@ proc resolveSym(c: var DecodeContext; symAsStr: string; alsoConsiderPrivate: boo # Create a stub symbol let val = addr c.mods[module].symCounter inc val[] - let id = ItemId(module: int32(module), item: val[]) + let id = itemId(int32(module), val[]) result = PSym(itemId: id, kindImpl: skProc, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Partial) c.syms[symAsStr] = (result, offs) @@ -1469,10 +2049,29 @@ proc resolveHookSym*(c: var DecodeContext; symId: nifstreams.SymId): PSym = proc tryResolveCompilerProc*(c: var DecodeContext; name: string; moduleFileIdx: FileIndex): PSym = ## Tries to resolve a compiler proc from a module by checking the NIF index. - ## Returns nil if the symbol doesn't exist. + ## Returns nil if the symbol doesn't exist. The NIF disamb is mint order, so + ## `name.0.` can be any of the overloads sharing the name — for `newSeq` it + ## is the generic magic, not the RTL proc (a refc build then demands codegen + ## of the generic and dies on `seq[T]`): enumerate the index entries with + ## this basename and pick the one that carries `sfCompilerProc`. + result = nil let suffix = moduleSuffix(c.infos.config, moduleFileIdx) - let symName = name & ".0." & suffix - result = resolveSym(c, symName, true) + let module = moduleId(c, suffix) + let prefix = name & "." + var candidates: seq[int] = @[] + for key in c.mods[module].index.keys: + if key.len > prefix.len and key.startsWith(prefix): + let sn = parseSymName(key) + if sn.name == name: + candidates.add sn.count + # the loads below can grow `c.mods` (symbols reference other modules), so + # resolve only after the index iteration is done + for count in candidates: + let sym = resolveSym(c, name & "." & $count & "." & suffix, true) + if sym != nil: + loadSym(c, sym) + if sfCompilerProc in sym.flagsImpl: + return sym proc loadLogOp(c: var DecodeContext; logOps: var seq[LogEntry]; s: var Stream; kind: LogEntryKind; op: TTypeAttachedOp; module: int): PackedToken = result = next(s) @@ -1527,6 +2126,8 @@ type deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of logOps*: seq[LogEntry] module*: PSym # set by modulegraphs.nim! + reexportedModules*: seq[(string, string)] # (name, suffix) of re-exported MODULE syms; + # materialized by modulegraphs.nim proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = tok = next(s) # skip `(import` @@ -1544,6 +2145,25 @@ proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix] else: raiseAssert "expected ParRi but got " & $tok.kind +proc addReexportedEnumFields(c: var DecodeContext; sym: PSym; interf: var TStrTable) = + ## When a non-pure enum type is (re-)exported, its fields must also become + ## visible (unqualified) to importers. In a from-source build this happens via + ## `rawImportSymbol`'s enum handling when the type is imported; the lazy IC + ## importer never runs that, so we materialise the fields into the interface + ## here, when the export list is processed. + loadSym(c, sym) + if sym.kindImpl != skType or sfPure in sym.flagsImpl: return + let et = sym.typImpl + if et == nil: return + loadType(c, et) + if et.kind notin {tyEnum, tyBool}: return + let fields = et.nImpl + if fields == nil: return + for i in 0 ..< fields.len: + let f = fields[i] + if f != nil and f.kind == nkSym and f.sym != nil: + strTableAdd(interf, f.sym) + proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; interf: var TStrTable; suffix: string; module: int): PrecompiledModule = result = PrecompiledModule(topLevel: newNode(nkStmtList)) @@ -1593,6 +2213,7 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; #elif t.tagId == repClassTag: # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) elif t.tagId == exportTag: + var lastGood = "" t = next(s) # skip (export if t.kind == DotToken: t = next(s) # skip dot @@ -1601,19 +2222,53 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag]; while true: if t.kind == Symbol: let symAsStr = pool.syms[t.symId] - let sym = resolveSym(c, symAsStr, false) - if sym != nil: - strTableAdd(interf, sym) + lastGood = symAsStr + # Skip symbols that are re-exported by this dependency but actually + # belong to the module we are compiling fresh: loading them as stubs + # would shadow/collide with the freshly compiled originals. + if c.mainModuleSuffix.len == 0 or + parseSymName(symAsStr).module != c.mainModuleSuffix: + # Resolving an exported symbol of this very module (`export` of a + # symbol that lives in a `when` branch of the same file) lazily + # loads it from the stream we are currently iterating, moving the + # cursor into the symbol's `(sd ...)` definition. Save/restore the + # position so the export-list parse continues where it left off. + let saved = offset(s.r) + let sym = resolveSym(c, symAsStr, false) + if sym != nil: + strTableAdd(interf, sym) + addReexportedEnumFields(c, sym, interf) + s.r.jumpTo(saved) t = next(s) elif t.kind == ParRi: break else: - raiseAssert "expected Symbol or ParRi but got " & $t.kind + raiseAssert "expected Symbol or ParRi but got " & $t.kind & + " (" & (if t.kind == ParLe: pool.tags[t.tagId] else: "") & + ") in export list of module " & suffix & ", last symbol: " & lastGood t = next(s) elif t.tagId == includeTag: t = skipTree(s) elif t.tagId == importTag: loadImport(c, s, result.deps, t) + elif t.tagId == reexpModTag: + # a re-exported MODULE: (reexpmod "name" "suffix"); the module sym + # is a qualifier in this module's interface — materialized by the + # caller (modulegraphs), which can register interface tables + t = next(s) + var mname = "" + var msuffix = "" + if t.kind == StringLit: + mname = pool.strings[t.litId] + t = next(s) + if t.kind == StringLit: + msuffix = pool.strings[t.litId] + t = next(s) + if t.kind != ParRi: + raiseAssert "expected ParRi in reexpmod entry of module " & suffix + t = next(s) + if mname.len > 0 and msuffix.len > 0: + result.reexportedModules.add (mname, msuffix) elif t.tagId == implTag: cont = false elif LoadFullAst in flags: @@ -1636,8 +2291,11 @@ proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHi let module = moduleId(c, string(suffix), flags) # Load the module AST (or just replay actions if loadFullAst is false) - # processTopLevel also collects export instructions + # processTopLevel also collects export instructions. + # Lazy symbol loading may have moved the stream cursor since the module was + # opened, so rewind to the start of the module body before reading it. let s = addr c.mods[module].stream + s[].r.jumpTo(c.mods[module].contentStart) var t = next(s[]) if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): t = next(s[]) # skip (stmts @@ -1662,3 +2320,4 @@ when isMainModule: echo obj.name, " ", obj.module, " ", obj.count let objb = parseSymName("abcdef.0121") echo objb.name, " ", objb.module, " ", objb.count + diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 6a7b4d7788..5857531c8d 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -20,6 +20,9 @@ export int128 import nodekinds export nodekinds +import itemids +export itemids + type TCallingConvention* = enum ccNimCall = "nimcall" # nimcall, also the default @@ -571,23 +574,6 @@ const generatedMagics* = {mNone, mIsolate, mFinished, mOpenArrayToSeq} ## magics that are generated as normal procs in the backend -type - ItemId* = object - module*: int32 - item*: int32 - -proc `$`*(x: ItemId): string = - "(module: " & $x.module & ", item: " & $x.item & ")" - -proc `==`*(a, b: ItemId): bool {.inline.} = - a.item == b.item and a.module == b.module - -proc hash*(x: ItemId): Hash = - var h: Hash = hash(x.module) - h = h !& hash(x.item) - result = !$h - - type PNode* = ref TNode TNodeSeq* = seq[PNode] diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index b2521069d4..feb5babeac 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -394,7 +394,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode; result: var Builder; n # variable. Thus, we create a temporary pointer variable instead. let needsIndirect = mapType(p.config, n[0].typ, mapTypeChooser(n[0]) == skParam) != ctArray if needsIndirect: - n.typ = n.typ.exactReplica + n.typ = n.typ.exactReplica(p.module.idgen) n.typ.incl tfVarIsPtr a = initLocExprSingleUse(p, n) a = withTmpIfNeeded(p, a, needsTmp) @@ -909,6 +909,16 @@ proc isInactiveDestructorCall(p: BProc, e: PNode): bool = proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) = if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri): return + when defined(icDbgHash): + if ri[0].typ == nil: + echo "NILCALLEE kind=", ri[0].kind, + " sym=", (if ri[0].kind == nkSym: ri[0].sym.name.s else: "-"), + " symKind=", (if ri[0].kind == nkSym: $ri[0].sym.kind else: "-"), + " flags=", (if ri[0].kind == nkSym: $ri[0].sym.flags else: "-"), + " lazy=", nfLazyType in ri[0].flags, + " inProc=", (if p.prc != nil: p.prc.name.s else: "NIL"), + " module=", p.module.module.name.s + raiseAssert "nil callee type, see NILCALLEE above" if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure: genClosureCall(p, le, ri, d) elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags: diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index c6e6057223..2c1bf17590 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3491,7 +3491,23 @@ proc genConstDefinition(q: BModule; p: BProc; sym: PSym) = data.addDeclWithVisibility(Private): data.addVarWithInitializer(Local, actualConstName, typ = td): genBracedInit(q.initProc, sym.astdef, isConst = true, sym.typ, data) - q.s[cfsData].add(extract(data)) + if q.config.cmd == cmdNifC: + # Each `cg` process that demands this const emits its definition + # (emit-everywhere). Always declare it first (the data analogue of a proc + # prototype) so a TU whose copy the merge stage drops still has a valid + # declaration; wrap the definition as a droppable `'d'` unit the merge + # stage assigns to a single owner. + let cname = stripCnifMarks(actualConstName) + var decl = newBuilder("") + decl.addDeclWithVisibility(Extern): + decl.addVar(kind = Local, name = actualConstName, typ = td) + q.s[cfsData].add(extract(decl)) + q.s[cfsData].add(cnifDefDirective(cname, "d", icNifName(q, sym))) + q.s[cfsData].add(extract(data)) + q.s[cfsData].add(cnifEndDefs()) + q.icDataDefs.add (cname, icNifName(q, sym)) + else: + q.s[cfsData].add(extract(data)) if q.hcrOn: # generate the global pointer with the real name q.s[cfsVars].addVar(kind = Global, name = sym.loc.snippet, @@ -3555,6 +3571,17 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of skProc, skConverter, skIterator, skFunc: #if sym.kind == skIterator: # echo renderTree(sym.getBody, {renderIds}) + if p.config.cmd == cmdNifC and + (isGenericRoutineStrict(sym) or sfCompileTime in sym.flags or + (sym.kind == skIterator and sym.typ.callConv == ccInline)): + # Under IC a module's top-level routine definitions are serialized as bare + # symbol references that reappear in the loaded statement list. Uninstantiated + # generic routines (incl. those with type-class params like `tuple`) and + # `.compileTime` routines have no run-time code, so skip them here. + # Inline iterators likewise have no standalone code — they are always inlined + # at their for-loop call sites by the transformer (only closure iterators get + # a standalone C function), so a bare serialized def reference is a no-op. + return if sfCompileTime in sym.flags: localError(p.config, n.info, "request to generate code for .compileTime proc: " & sym.name.s) @@ -3629,6 +3656,11 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = # echo renderTree(p.prc.ast, {renderIds}) internalError(p.config, n.info, "expr: param not init " & sym.name.s & "_" & $sym.id) putLocIntoDest(p, d, sym.loc) + of skTemplate, skMacro: + # Under IC a module's top-level template/macro definitions are serialized as + # bare symbol references (only their interface matters), so they reappear in + # the loaded statement list. They are compile-time only and produce no code. + discard else: internalError(p.config, n.info, "expr(" & $sym.kind & "); unknown symbol") of nkNilLit: if not isEmptyType(n.typ): diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index bc9c06fa1d..a30c1b1bf2 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -1986,4 +1986,9 @@ proc genStmts(p: BProc, t: PNode) = if isPush: pushInfoContext(p.config, t.info) expr(p, t, a) if isPush: popInfoContext(p.config) - internalAssert p.config, a.k in {locNone, locTemp, locLocalVar, locExpr} + # A bare `nkSym` statement is how IC serializes a definition that lives inside a + # top-level block (e.g. a nested `proc`/`var`): codegen emits the definition and + # leaves the symbol's own location in `a` (e.g. `locProc`), which is discarded + # here, so the value-sanity check below does not apply to it. + internalAssert p.config, t.kind == nkSym or + a.k in {locNone, locTemp, locLocalVar, locExpr} diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 8e6b44c81d..eb82c854c7 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -72,6 +72,37 @@ proc mangleProc(m: BModule; s: PSym; makeUnique: bool): string = else: m.g.mangledPrcs.incl(result) +proc sharedInstanceCName(m: BModule; s: PSym): string = + ## The module-free canonical C name for a content-keyed generic instance, + ## or "" when the symbol must keep its module-suffixed name. With a shared + ## name, every TU that instantiated the same generic with the same type + ## arguments calls one extern definition (first claimant's TU embeds it, + ## see `genProcLvl3`) instead of compiling its own static copy. + ## + ## The name is program-unique only if the 30-bit content hash does not + ## collide for same-named instances of *different* instantiations across + ## modules — the per-module probe in `setInstanceDisamb` cannot see that. + ## Claimants therefore must present the same signature; on mismatch the + ## later one keeps its module-suffixed name (no merge, still correct). + ## Residual risk: same name and signature, different generic args, AND a + ## 30-bit collision — vanishingly unlikely; a full-typeKey verification + ## channel can close it later. + result = "" + if m.config.cmd == cmdNifC and s.kind in routineKinds and + (s.disamb and InstanceDisambBit) != 0'i32 and + s.typ != nil and s.typ.callConv != ccInline and not m.hcrOn and + {sfImportc, sfExportc, sfCodegenDecl} * s.flags == {}: + # The content-derived `disamb` is unique per process (collision-probed in + # `setInstanceDisamb`), so the mint-site-independent `_i` name is + # safe to use directly; identical instances across modules collide on it + # exactly and the merge stage keeps one. + result = s.name.s.mangle & "_i" & $s.disamb + +proc isSharedInstanceCName(m: BModule; s: PSym): bool = + m.config.cmd == cmdNifC and s.kind in routineKinds and + (s.disamb and InstanceDisambBit) != 0'i32 and + stripCnifMarks(s.loc.snippet) == s.name.s.mangle & "_i" & $s.disamb + proc fillBackendName(m: BModule; s: PSym) = if s.loc.snippet == "": var result: Rope @@ -79,13 +110,22 @@ proc fillBackendName(m: BModule; s: PSym) = m.g.config.symbolFiles == disabledSf: result = mangleProc(m, s, false).rope else: - result = s.name.s.mangle.rope - result.add mangleProcNameExt(m.g.graph, s) + let shared = sharedInstanceCName(m, s) + if shared.len > 0: + result = shared.rope + else: + result = s.name.s.mangle.rope + result.add mangleProcNameExt(m.g.graph, s) if m.hcrOn: result.add '_' result.add(idOrSig(s, m.module.name.s.mangle, m.sigConflicts, m.config)) backendEnsureMutable s - s.locImpl.snippet = result + if m.config.cmd == cmdNifC: + # mark the name so the cnif artifact writer can turn every occurrence + # into a Symbol token; stripped from the actual C output in genModule + s.locImpl.snippet = markCName(result) + else: + s.locImpl.snippet = result proc fillParamName(m: BModule; s: PSym) = if s.loc.snippet == "": @@ -373,6 +413,12 @@ proc getSimpleTypeDesc(m: BModule; typ: PType): Rope = m.typeCache[sig] = result proc pushType(m: BModule; typ: PType) = + when defined(icDbgRefc): + if typ.kind == tySequence and + typ.elementType.skipTypes({tyGenericInst, tyAlias, tySink}).kind == tyGenericParam: + echo "[icRefc] pushType seq-of-genericparam t=", typeToString(typ), + " itemId=", typ.itemId.module, ".", typ.itemId.item, " mod=", m.module.name.s + echo getStackTrace() for i in 0..high(m.typeStack): # pointer equality is good enough here: if m.typeStack[i] == typ: return @@ -618,6 +664,18 @@ proc genProcParams(m: BModule; t: PType, rettype: var Rope, params: var Builder, for i in 1..