From 548b1c6ef8d3698bcf27d0535d3c418e4323f4cb Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 19 Dec 2025 01:54:03 +0800 Subject: [PATCH 01/31] fixes #25369 (#25370) fixes #25369 --- compiler/injectdestructors.nim | 2 +- tests/global/mglobal3.nim | 2 ++ tests/global/tglobal3.nim | 4 ++++ 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 tests/global/mglobal3.nim diff --git a/compiler/injectdestructors.nim b/compiler/injectdestructors.nim index f36d11c990..e6ddf79a8a 100644 --- a/compiler/injectdestructors.nim +++ b/compiler/injectdestructors.nim @@ -969,7 +969,7 @@ proc p(n: PNode; c: var Con; s: var Scope; mode: ProcessMode; tmpFlags = {sfSing isInProc if isGlobalPragma: - c.graph.procGlobals.add n + c.graph.procGlobals.add newTree(nkFastAsgn, v, ri) else: let value = moveOrCopy(v, ri, c, s, if v.kind == nkSym: {IsDecl} else: {}) result.add value diff --git a/tests/global/mglobal3.nim b/tests/global/mglobal3.nim new file mode 100644 index 0000000000..289c2d8e47 --- /dev/null +++ b/tests/global/mglobal3.nim @@ -0,0 +1,2 @@ +proc v*() = + let u {.global.} = default(ref int) \ No newline at end of file diff --git a/tests/global/tglobal3.nim b/tests/global/tglobal3.nim index b7f3f55391..1f9536e160 100644 --- a/tests/global/tglobal3.nim +++ b/tests/global/tglobal3.nim @@ -62,3 +62,7 @@ proc m2() = assert v == "123" m2() + +import mglobal3 +block: + v() \ No newline at end of file From 1324183c38fb11dfcfdb5bd5ac656dc89df07d13 Mon Sep 17 00:00:00 2001 From: elijahr Date: Sat, 20 Dec 2025 01:56:10 -0600 Subject: [PATCH 02/31] fix #17630: Implement cycle detection for recursive concepts (#25353) fixes #17630 ## Recursive Concept Cycle Detection - Track (conceptId, typeId) pairs during matching to detect cycles - Changed marker from IntSet to HashSet[ConceptTypePair] - Removed unused depthCount field - Added recursive concepts documentation to manual - Added tests for recursive concepts, distinct chains, and co-dependent concepts ## Fix Flaky `tasyncclosestall` Test The macOS ARM64 CI jobs were failing due to a flaky async socket test (unrelated to concepts). The test only accepted `EBADF` as a valid error code when closing a socket with pending writes. However, depending on timing, the kernel may report `ECONNRESET` or `EPIPE` instead: - **EBADF**: Socket was closed locally before kernel detected remote state - **ECONNRESET**: Remote peer sent RST packet (detected first) - **EPIPE**: Socket is no longer connected (broken pipe) All three are valid disconnection errors. The fix accepts any of them, making the test reliable across platforms. --------- Co-authored-by: Andreas Rumpf --- compiler/concepts.nim | 44 ++++++--- doc/manual.md | 38 +++++++ tests/concepts/t17630.nim | 15 +++ tests/concepts/trecursive_concepts.nim | 132 +++++++++++++++++++++++++ 4 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 tests/concepts/t17630.nim create mode 100644 tests/concepts/trecursive_concepts.nim diff --git a/compiler/concepts.nim b/compiler/concepts.nim index 4329e4b4bf..040089a669 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -13,7 +13,7 @@ import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, layeredtable -import std/intsets +import std/[intsets, sets] when defined(nimPreviewSlimSystem): import std/assertions @@ -73,18 +73,20 @@ type MatchFlags* = enum mfDontBind # Do not bind generic parameters mfCheckGeneric # formal <- formal comparison as opposed to formal <- operand - + + ConceptTypePair = tuple[conceptId, typeId: ItemId] + ## Pair of (concept type id, implementation type id) used for cycle detection + MatchCon = object ## Context we pass around during concept matching. bindings: LayeredIdTable - marker: IntSet ## Some protection against wild runaway recursions. + marker: HashSet[ConceptTypePair] ## Tracks (concept, type) pairs being checked to detect cycles. potentialImplementation: PType ## the concrete type that might match the concept we try to match. magic: TMagic ## mArrGet and mArrPut is wrong in system.nim and ## cannot be fixed that easily. ## Thus we special case it here. concpt: PType ## current concept being evaluated - depthCount = 0 flags: set[MatchFlags] - + MatchKind = enum mkNoMatch, mkSubset, mkSame @@ -188,32 +190,48 @@ iterator traverseTyOr(t: PType): PType {. closure .}= proc matchConceptToImpl(c: PContext, f, potentialImpl: PType; m: var MatchCon): bool = assert not(potentialImpl.reduceToBase.kind == tyConcept) let concpt = f.reduceToBase - if m.depthCount > 0: - # concepts that are more then 2 levels deep are treated like - # tyAnything to stop dependencies from getting out of control + + # Handle self-referential concepts: when a concept references itself in its body + # (e.g., `A = concept; proc test(x: Self, y: A)`), the inner type A has n=nil. + # We detect this by checking if the concept has the same symbol name as the + # one we're currently matching and has no body (n=nil). + if concpt.n.isNil: + if concpt.sym != nil and m.concpt.sym != nil and + concpt.sym == m.concpt.sym: + # Self-reference: check if potentialImpl matches what we're already checking + return potentialImpl.id == m.potentialImplementation.id + # Concept without body that's not a self-reference - cannot match + return false + + # Cycle detection: track (concept, type) pairs to prevent infinite recursion. + # Returns true on cycle (coinductive semantics) to support co-dependent concepts. + let pair: ConceptTypePair = (concpt.itemId, potentialImpl.itemId) + if pair in m.marker: return true + m.marker.incl pair + var efPot = potentialImpl if potentialImpl.isSelf: if m.concpt.n == concpt.n: + m.marker.excl pair return true efPot = m.potentialImplementation - + var oldBindings = m.bindings m.bindings = newTypeMapLayer(m.bindings) let oldPotentialImplementation = m.potentialImplementation m.potentialImplementation = efPot let oldConcept = m.concpt m.concpt = concpt - + var invocation: PType = nil if f.kind in {tyGenericInvocation, tyGenericInst}: invocation = f - inc m.depthCount result = processConcept(c, concpt, invocation, oldBindings, m) - dec m.depthCount m.potentialImplementation = oldPotentialImplementation m.concpt = oldConcept m.bindings = oldBindings + m.marker.excl pair proc cmpConceptDefs(c: PContext, fn, an: PNode, m: var MatchCon): bool= if fn.kind != an.kind: @@ -610,7 +628,7 @@ proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var LayeredIdTable ## `C[S, T]` parent type that we look for. We need this because we need to store bindings ## for 'S' and 'T' inside 'bindings' on a successful match. It is very important that ## we do not add any bindings at all on an unsuccessful match! - var m = MatchCon(bindings: bindings, potentialImplementation: arg, concpt: concpt, flags: flags) + var m = MatchCon(bindings: bindings, potentialImplementation: arg, concpt: concpt, flags: flags, marker: initHashSet[ConceptTypePair]()) if arg.isConcept: result = conceptsMatch(c, concpt.reduceToBase, arg.reduceToBase, m) >= mkSubset elif arg.acceptsAllTypes: diff --git a/doc/manual.md b/doc/manual.md index 21abe9504c..53d867c1ad 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -3025,6 +3025,44 @@ If neither of them are subsets of one another, then the disambiguation proceeds and the concept with the most definitions wins, if any. No definite winner is an ambiguity error at compile time. +Recursive concepts +------------------ + +Concepts can reference themselves in their definitions, enabling recursive type constraints. +This is useful for matching `distinct` types that should inherit traits from their base type: + +```nim +import std/typetraits + +type + PrimitiveBase = SomeNumber | bool | ptr | pointer | enum + + # Matches PrimitiveBase directly, or any distinct type whose base is Primitive + Primitive = concept x + x is PrimitiveBase or distinctBase(x) is Primitive + + # Application: a handle type that should be treated like a primitive + Handle = distinct int + SpecialHandle = distinct Handle + +assert int is Primitive +assert Handle is Primitive +assert SpecialHandle is Primitive # works through 2 levels +assert not (string is Primitive) +``` + +Concepts can also be mutually recursive (co-dependent): + +```nim +type + Serializable = concept + proc serialize(s: Self; writer: var Writer) + Writer = concept + proc write(w: var Self; data: Serializable) +``` + +The compiler uses cycle detection to handle these cases without infinite recursion. + Statements and expressions ========================== diff --git a/tests/concepts/t17630.nim b/tests/concepts/t17630.nim new file mode 100644 index 0000000000..b0cd7fbe0f --- /dev/null +++ b/tests/concepts/t17630.nim @@ -0,0 +1,15 @@ +discard """ + action: "compile" +""" + +# https://github.com/nim-lang/Nim/issues/17630 +# A concept that references itself in a proc signature +# should not cause infinite recursion / stack overflow + +type + A = concept + proc test(x: Self, y: A) + +proc test(x: int, y: int) = discard + +discard (int is A) diff --git a/tests/concepts/trecursive_concepts.nim b/tests/concepts/trecursive_concepts.nim new file mode 100644 index 0000000000..7a2c042e20 --- /dev/null +++ b/tests/concepts/trecursive_concepts.nim @@ -0,0 +1,132 @@ +discard """ +action: "run" +output: ''' +int is Primitive: true +Handle is Primitive: true +SpecialHandle is Primitive: true +FileDescriptor is Primitive: true +float is Primitive: false +string is Primitive: false +char is PrimitiveBase: true +ptr int is PrimitiveBase: true +''' +""" + +# Test recursive concepts with cycle detection +# This tests concepts that reference themselves via distinctBase + +import std/typetraits + +block: # Basic recursive concept with distinctBase + type + PrimitiveBase = SomeInteger | bool | char | ptr | pointer + + # Recursive concept: matches PrimitiveBase or any distinct type whose base is Primitive + Primitive = concept x + x is PrimitiveBase or distinctBase(x) is Primitive + + # Real-world example: handle types that wrap integers + Handle = distinct int + SpecialHandle = distinct Handle + FileDescriptor = distinct SpecialHandle + + # Direct base types + echo "int is Primitive: ", int is Primitive + + # Single-level distinct (like a simple handle type) + echo "Handle is Primitive: ", Handle is Primitive + + # Two-level distinct + echo "SpecialHandle is Primitive: ", SpecialHandle is Primitive + + # Three-level distinct + echo "FileDescriptor is Primitive: ", FileDescriptor is Primitive + + # Non-primitive types should NOT match + echo "float is Primitive: ", float is Primitive + echo "string is Primitive: ", string is Primitive + +block: # Ensure base type matching still works + type + PrimitiveBase = SomeInteger | bool | char | ptr | pointer + + echo "char is PrimitiveBase: ", char is PrimitiveBase + echo "ptr int is PrimitiveBase: ", (ptr int) is PrimitiveBase + +block: # Test that cycle detection doesn't break normal concept matching + type + Addable = concept x, y + x + y is typeof(x) + + doAssert int is Addable + doAssert float is Addable + +block: # Test non-matching recursive case + type + IntegerBase = SomeInteger + + IntegerLike = concept x + x is IntegerBase or distinctBase(x) is IntegerLike + + Percentage = distinct float # float base, not integer + + doAssert int is IntegerLike + doAssert not(float is IntegerLike) + doAssert not(Percentage is IntegerLike) # float base doesn't match + +block: # Test deep distinct chains (5+ levels) - e.g., layered ID types + type + IdBase = SomeInteger + + IdLike = concept x + x is IdBase or distinctBase(x) is IdLike + + EntityId = distinct int + UserId = distinct EntityId + AdminId = distinct UserId + SuperAdminId = distinct AdminId + RootId = distinct SuperAdminId + + doAssert int is IdLike + doAssert EntityId is IdLike + doAssert UserId is IdLike + doAssert AdminId is IdLike + doAssert SuperAdminId is IdLike + doAssert RootId is IdLike + doAssert not(float is IdLike) + +block: # Test 3-way mutual recursion (co-dependent concepts) + # This tests that cycle detection properly handles A -> B -> C -> A cycles + type + Serializable = concept + proc serialize(x: Self): Bytes + + Bytes = concept + proc compress(x: Self): Compressed + + Compressed = concept + proc decompress(x: Self): Serializable + + Data = object + value: int + + proc serialize(x: Data): Data = x + proc compress(x: Data): Data = x + proc decompress(x: Data): Data = x + + # Data should satisfy all three mutually recursive concepts + doAssert Data is Serializable + doAssert Data is Bytes + doAssert Data is Compressed + +block: # Test concept with method returning same type + type + Cloneable = concept + proc clone(x: Self): Self + + Document = object + content: string + + proc clone(x: Document): Document = x + + doAssert Document is Cloneable From b901a80710212da44daa55c704f97c6e3306eaa9 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Sat, 20 Dec 2025 11:27:46 +0100 Subject: [PATCH 03/31] IC: progress (#25368) Co-authored-by: Jacek Sieka Co-authored-by: Ryan McConnell --- compiler/ccgtypes.nim | 10 ++++++++-- compiler/msgs.nim | 6 ++---- compiler/nifbackend.nim | 2 ++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index b09000d005..399b07d1a5 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1862,6 +1862,12 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn if t.kind == tyObject and t.baseClass != nil and optEnableDeepCopy in m.config.globalOptions: discard genTypeInfoV1(m, t, info) +proc myModuleOpenForCodegen(m: BModule; idx: FileIndex): bool {.inline.} = + if moduleOpenForCodegen(m.g.graph, idx): + result = idx.int < m.g.modules.len and m.g.modules[idx.int] != nil + else: + result = false + proc genTypeInfoV2(m: BModule; t: PType; info: TLineInfo): Rope = let origType = t # distinct types can have their own destructors @@ -1890,7 +1896,7 @@ proc genTypeInfoV2(m: BModule; t: PType; info: TLineInfo): Rope = m.typeInfoMarkerV2[sig] = result let owner = t.skipTypes(typedescPtrs).itemId.module - if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner): + if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner): # make sure the type info is created in the owner module discard genTypeInfoV2(m.g.modules[owner], origType, info) # reference the type info as extern here @@ -1975,7 +1981,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = return prefixTI(result) var owner = t.skipTypes(typedescPtrs).itemId.module - if owner != m.module.position and moduleOpenForCodegen(m.g.graph, FileIndex owner): + if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner): # make sure the type info is created in the owner module discard genTypeInfoV1(m.g.modules[owner], origType, info) # reference the type info as extern here diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 5c52c10d01..aff8a6a53d 100644 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -30,7 +30,6 @@ proc toLowerAscii(a: var string) {.inline.} = proc flushDot*(conf: ConfigRef) = ## safe to call multiple times - # xxx one edge case not yet handled is when `printf` is called at CT with `compiletimeFFI`. let stdOrr = if optStdout in conf.globalOptions: stdout else: stderr let stdOrrKind = toStdOrrKind(stdOrr) if stdOrrKind in conf.lastMsgWasDot: @@ -52,7 +51,7 @@ proc makeCString*(s: string): Rope = result = newStringOfCap(int(s.len.toFloat * 1.1) + 1) result.add("\"") for i in 0.. Date: Sun, 21 Dec 2025 07:35:40 +0100 Subject: [PATCH 04/31] [Docs] Remove horizontal scrolling on mobile (#25377) * Also use more of the available width --- doc/nimdoc.css | 5 ++++- nimdoc/testproject/expected/nimdoc.out.css | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/doc/nimdoc.css b/doc/nimdoc.css index 3fc453dc0b..1ca55a2bd0 100644 --- a/doc/nimdoc.css +++ b/doc/nimdoc.css @@ -181,6 +181,7 @@ body { .nine.columns { width: 75.0%; + margin-left: 0; padding-left: 1.5em; } .twelve.columns { @@ -192,7 +193,9 @@ body { display: none; } .nine.columns { - width: 98.0%; + width: 100%; + margin-left: 0; + padding-left: 0; } body { font-size: 1em; diff --git a/nimdoc/testproject/expected/nimdoc.out.css b/nimdoc/testproject/expected/nimdoc.out.css index 3fc453dc0b..1ca55a2bd0 100644 --- a/nimdoc/testproject/expected/nimdoc.out.css +++ b/nimdoc/testproject/expected/nimdoc.out.css @@ -181,6 +181,7 @@ body { .nine.columns { width: 75.0%; + margin-left: 0; padding-left: 1.5em; } .twelve.columns { @@ -192,7 +193,9 @@ body { display: none; } .nine.columns { - width: 98.0%; + width: 100%; + margin-left: 0; + padding-left: 0; } body { font-size: 1em; From b819472e744463aa8fa4959d6610f371240699d7 Mon Sep 17 00:00:00 2001 From: elijahr Date: Sun, 21 Dec 2025 00:37:26 -0600 Subject: [PATCH 05/31] Fix `sizeof(T)` in `typedesc` templates called from generic type `when` clauses (#25374) The `hasValuelessStatics` function in `semtypinst.nim` only checked for `tyStatic`, missing `tyTypeDesc(tyGenericParam)`. This caused `sizeof(T)` inside a typedesc template called from a generic type's `when` clause to error with "'sizeof' requires '.importc' types to be '.completeStruct'". The fix adds a check for `tyTypeDesc` wrapping `tyGenericParam`, recognizing it as an unresolved generic parameter that needs resolution before evaluation. Also documents the `completeStruct` pragma in the manual. --- changelog.md | 8 +++++ compiler/semtypinst.nim | 15 ++++++++-- doc/manual.md | 29 ++++++++++++++++++ tests/generic/tgeneric_typedesc_sizeof.nim | 34 ++++++++++++++++++++++ 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 tests/generic/tgeneric_typedesc_sizeof.nim diff --git a/changelog.md b/changelog.md index 4b320399c2..217c8c9653 100644 --- a/changelog.md +++ b/changelog.md @@ -103,7 +103,15 @@ errors. ## Compiler changes +- Fixed a bug where `sizeof(T)` inside a `typedesc` template called from a generic type's + `when` clause would error with "'sizeof' requires '.importc' types to be '.completeStruct'". + The issue was that `hasValuelessStatics` in `semtypinst.nim` didn't recognize + `tyTypeDesc(tyGenericParam)` as an unresolved generic parameter. ## Tool changes - Added `--stdinfile` flag to name of the file used when running program from stdin (defaults to `stdinfile.nim`) + +## Documentation changes + +- Added documentation for the `completeStruct` pragma in the manual. diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 031683d04a..1d3f51480b 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -249,13 +249,24 @@ proc hasValuelessStatics(n: PNode): bool = a proc doThing(_: MyThing) ]# + result = false if n.safeLen == 0 and n.kind != nkEmpty: # Some empty nodes can get in here - n.typ == nil or n.typ.kind == tyStatic + if n.typ == nil: + result = true + elif n.typ.kind == tyStatic: + result = true + elif n.typ.kind == tyTypeDesc: + # Check if the base type is an unresolved generic parameter. + # This handles cases where a template containing sizeof(T) is called + # inside a generic object's when clause - the T needs to be resolved + # before we can evaluate the condition. + let base = n.typ.skipTypes({tyTypeDesc}) + if base.kind == tyGenericParam: + result = true else: for x in n: if hasValuelessStatics(x): return true - false proc replaceTypeVarsN(cl: var TReplTypeVars, n: PNode; start=0; expectedType: PType = nil): PNode = if n == nil: return diff --git a/doc/manual.md b/doc/manual.md index 53d867c1ad..f52e0ba38c 100644 --- a/doc/manual.md +++ b/doc/manual.md @@ -7981,6 +7981,35 @@ underlying C `struct`:c: in a `sizeof` expression: ``` +CompleteStruct pragma +--------------------- +The `completeStruct` pragma is a contract indicating that an `importc` type +declaration contains all fields of the corresponding C type, allowing +`sizeof`, `alignof`, and `offsetof` to be computed at compile-time. + +By default, `importc` types are assumed to be incomplete (their size is +unknown at compile-time). Use `completeStruct` when you need compile-time +size information and can guarantee the Nim definition matches the C layout: + + ```Nim + type + InotifyEvent {.importc: "struct inotify_event", header: "", + completeStruct.} = object + wd: cint + mask: uint32 + cookie: uint32 + len: uint32 + # All fields must match the C struct exactly + ``` + +If the Nim fields don't match the C struct, a static assertion will fail +during C code generation. + +Without `completeStruct`, attempting to use `sizeof` on an `importc` type +at compile-time will error with "'sizeof' requires '.importc' types to be +'.completeStruct'". + + Compile pragma -------------- The `compile` pragma can be used to compile and link a C/C++ source file diff --git a/tests/generic/tgeneric_typedesc_sizeof.nim b/tests/generic/tgeneric_typedesc_sizeof.nim new file mode 100644 index 0000000000..b8284495ee --- /dev/null +++ b/tests/generic/tgeneric_typedesc_sizeof.nim @@ -0,0 +1,34 @@ +discard """ + output: ''' +42 +''' +""" + +# Regression test for semtypinst.nim hasValuelessStatics bug. +# +# Bug: hasValuelessStatics only checked for tyStatic, missing tyTypeDesc(tyGenericParam) +# Fix: Added check for tyTypeDesc wrapping tyGenericParam in compiler/semtypinst.nim +# +# The bug triggers when: +# 1. A generic type has a when clause calling a typedesc template with sizeof(T) +# 2. A generic proc on that type is called, triggering instantiation +# 3. The T in sizeof(T) becomes tyTypeDesc(tyGenericParam), which wasn't recognized as unresolved +# +# Error without fix: 'sizeof' requires '.importc' types to be '.completeStruct' + +template isSmall(T: typedesc): bool = + sizeof(T) <= 8 + +type Foo[T] = object + when isSmall(T): + a: T + else: + b: ptr T + +proc bar[T](x: var Foo[T]) = + discard + +var x: Foo[int] +x.a = 42 +x.bar() +echo x.a From 2dbdf08fc7007865400e406359948ea09741a455 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Sun, 21 Dec 2025 19:13:25 +0100 Subject: [PATCH 06/31] Fixes #25319 (#25380) This was a regression introduced in https://github.com/nim-lang/Nim/pull/25070. @janAkali, @Z9RO, can you verify please? --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index ff6fcb3a66..ecd3b3e8e0 100644 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -573,7 +573,7 @@ proc generateHeaders(requestUrl: Uri, httpMethod: HttpMethod, headers: HttpHeade result = $httpMethod result.add ' ' - if proxy.isNil or (requestUrl.scheme == "https" and proxy.url.scheme == "socks5h"): + if proxy.isNil or requestUrl.scheme == "https": # /path?query if not requestUrl.path.startsWith("/"): result.add '/' result.add(requestUrl.path) From 5e53a70e62d40d4484e40ec72b4c5d6d9fd3db5d Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Thu, 25 Dec 2025 07:04:18 +0800 Subject: [PATCH 07/31] fixes #25254; fixes #10395; Invalid pred in when swallowed (#25385) fixes #25254 fixes #10395 --- compiler/vm.nim | 6 ++++++ compiler/vmdef.nim | 2 +- compiler/vmgen.nim | 5 +++++ tests/errmsgs/tvmranges.nim | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/errmsgs/tvmranges.nim diff --git a/compiler/vm.nim b/compiler/vm.nim index 08ac142f37..251017c208 100644 --- a/compiler/vm.nim +++ b/compiler/vm.nim @@ -1723,6 +1723,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg = let max = (1.BiggestInt shl (rb-1))-1 if regs[ra].intVal < min or regs[ra].intVal > max: stackTrace(c, tos, pc, "unhandled exception: value out of range") + of opcNarrowR: + decodeBC(rkInt) + let min = regs[rb].intVal + let max = regs[rc].intVal + if regs[ra].intVal < min or regs[ra].intVal > max: + stackTrace(c, tos, pc, "unhandled exception: value out of range") of opcNarrowU: decodeB(rkInt) regs[ra].intVal = regs[ra].intVal and ((1'i64 shl rb)-1) diff --git a/compiler/vmdef.nim b/compiler/vmdef.nim index e8336aaba4..a3ac120f99 100644 --- a/compiler/vmdef.nim +++ b/compiler/vmdef.nim @@ -105,7 +105,7 @@ type opcIsNil, opcOf, opcIs, opcParseFloat, opcConv, opcCast, opcQuit, opcInvalidField, - opcNarrowS, opcNarrowU, + opcNarrowS, opcNarrowU, opcNarrowR opcSignExtend, opcAddStrCh, diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 8c5460b330..11b7b27fe7 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -798,6 +798,11 @@ proc genNarrow(c: PCtx; n: PNode; dest: TDest) = c.gABC(n, opcNarrowU, dest, TRegister(size*8)) elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and size < 8): c.gABC(n, opcNarrowS, dest, TRegister(size*8)) + elif t.kind in {tyEnum, tyRange}: + let intType = getSysType(c.graph, n.info, tyInt) + let first = c.genx(newIntTypeNode(firstOrd(c.config, t), intType)) + let last = c.genx(newIntTypeNode(lastOrd(c.config, t), intType)) + c.gABC(n, opcNarrowR, dest, first, last) proc genNarrowU(c: PCtx; n: PNode; dest: TDest) = let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) diff --git a/tests/errmsgs/tvmranges.nim b/tests/errmsgs/tvmranges.nim new file mode 100644 index 0000000000..236da34148 --- /dev/null +++ b/tests/errmsgs/tvmranges.nim @@ -0,0 +1,17 @@ +discard """ + action: reject + nimout: ''' +stack trace: (most recent call last) +tvmranges.nim(14, 10) +tvmranges.nim(14, 10) Error: unhandled exception: value out of range +''' +""" + +type X = enum + a + b + +when pred(a) == b: + echo "a" +else: + echo "b" \ No newline at end of file From a41bbf6901532d7bb1bac8b74e1e0ba4290a252b Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Fri, 26 Dec 2025 04:02:54 +0800 Subject: [PATCH 08/31] fixes #25387; `embedsrc` breaks with Line Continuation (#25388) fixes #25387 https://stackoverflow.com/questions/30286253/how-to-escape-backslash-in-comment - adding a whitespace or `\t` after `\` breaks the `goto` block - `\* *\` doesn't support nesting, causing problems for using it in the Nim comments --- compiler/cgen.nim | 5 ++++- tests/ccgbugs/t25387.nim | 8 ++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/ccgbugs/t25387.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index c271cbda31..48bf1ad6e3 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -337,7 +337,10 @@ proc genLineDir(p: BProc, t: PNode) = let line = t.info.safeLineNm if optEmbedOrigSrc in p.config.globalOptions: - p.s(cpsStmts).add("//" & sourceLine(p.config, t.info) & "\L") + var code = sourceLine(p.config, t.info) + if code.endsWith('\\'): + code.add "#" + p.s(cpsStmts).add("// " & code & "\L") let lastFileIndex = p.lastLineInfo.fileIndex let freshLine = freshLineInfo(p, t.info) if freshLine: diff --git a/tests/ccgbugs/t25387.nim b/tests/ccgbugs/t25387.nim new file mode 100644 index 0000000000..d694e3351d --- /dev/null +++ b/tests/ccgbugs/t25387.nim @@ -0,0 +1,8 @@ +discard """ + matrix: "--embedsrc=on" +""" + +proc trim() = + let s = 10 + let x = s + 5 # user entered literal \ +trim() \ No newline at end of file From a061f026a886a72cddf310501d80f737ccbebbe0 Mon Sep 17 00:00:00 2001 From: bptato <60043228+bptato@users.noreply.github.com> Date: Thu, 25 Dec 2025 21:04:04 +0100 Subject: [PATCH 09/31] Fix std/hashes completely ignoring endianness (#25386) This is a problem on big-endian CPUs because you end up with nimvm computing something different than Nim proper, so e.g. a const table won't work. I also took the liberty to replace a redundant implementation of load4 in murmurHash. (Thanks to barracuda156 for helping debug this.) --- lib/pure/hashes.nim | 71 +++++++++++++------------------ tests/pragmas/thintprocessing.nim | 2 +- 2 files changed, 31 insertions(+), 42 deletions(-) diff --git a/lib/pure/hashes.nim b/lib/pure/hashes.nim index c0171237d0..f53a88db8c 100644 --- a/lib/pure/hashes.nim +++ b/lib/pure/hashes.nim @@ -304,6 +304,35 @@ else: proc rotl32(x: uint32, r: int): uint32 {.inline.} = (x shl r) or (x shr (32 - r)) +proc load4e(s: openArray[byte], o=0): uint32 {.inline.} = + uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or + uint32(s[o + 1]) shl 8 or uint32(s[o + 0]) + +proc load8e(s: openArray[byte], o=0): uint64 {.inline.} = + uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or + uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or + uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or + uint64(s[o + 1]) shl 8 or uint64(s[o + 0]) + +when declared(copyMem): + from std/endians import littleEndian64, littleEndian32 + +proc load4(s: openArray[byte], o=0): uint32 {.inline.} = + when nimvm: result = load4e(s, o) + else: + when declared copyMem: + result = uint32(0) + littleEndian32(addr result, addr s[o]) + else: result = load4e(s, o) + +proc load8(s: openArray[byte], o=0): uint64 {.inline.} = + when nimvm: result = load8e(s, o) + else: + when declared copyMem: + result = uint64(0) + littleEndian64(addr result, addr s[o]) + else: result = load8e(s, o) + proc murmurHash(x: openArray[byte]): Hash = # https://github.com/PeterScott/murmur3/blob/master/murmur3.c const @@ -320,24 +349,10 @@ proc murmurHash(x: openArray[byte]): Hash = h1: uint32 = uint32(0) i = 0 - - template impl = - var j = stepSize - while j > 0: - dec j - k1 = (k1 shl 8) or (ord(x[i+j])).uint32 - # body while i < n * stepSize: - var k1: uint32 = uint32(0) + var k1 = load4(x, i) - when nimvm: - impl() - else: - when declared(copyMem): - copyMem(addr k1, addr x[i], 4) - else: - impl() inc i, stepSize k1 = imul(k1, c1) @@ -384,32 +399,6 @@ const k0 = 0xc3a5c85c97cb3127u64 # Primes on (2^63, 2^64) for various uses const k1 = 0xb492b66fbe98f273u64 const k2 = 0x9ae16a3b2f90404fu64 -proc load4e(s: openArray[byte], o=0): uint32 {.inline.} = - uint32(s[o + 3]) shl 24 or uint32(s[o + 2]) shl 16 or - uint32(s[o + 1]) shl 8 or uint32(s[o + 0]) - -proc load8e(s: openArray[byte], o=0): uint64 {.inline.} = - uint64(s[o + 7]) shl 56 or uint64(s[o + 6]) shl 48 or - uint64(s[o + 5]) shl 40 or uint64(s[o + 4]) shl 32 or - uint64(s[o + 3]) shl 24 or uint64(s[o + 2]) shl 16 or - uint64(s[o + 1]) shl 8 or uint64(s[o + 0]) - -proc load4(s: openArray[byte], o=0): uint32 {.inline.} = - when nimvm: result = load4e(s, o) - else: - when declared copyMem: - result = uint32(0) - copyMem result.addr, s[o].addr, result.sizeof - else: result = load4e(s, o) - -proc load8(s: openArray[byte], o=0): uint64 {.inline.} = - when nimvm: result = load8e(s, o) - else: - when declared copyMem: - result = uint64(0) - copyMem result.addr, s[o].addr, result.sizeof - else: result = load8e(s, o) - proc lenU(s: openArray[byte]): uint64 {.inline.} = s.len.uint64 proc shiftMix(v: uint64): uint64 {.inline.} = v xor (v shr 47) diff --git a/tests/pragmas/thintprocessing.nim b/tests/pragmas/thintprocessing.nim index 943d921669..93b8fa4a61 100644 --- a/tests/pragmas/thintprocessing.nim +++ b/tests/pragmas/thintprocessing.nim @@ -3,7 +3,7 @@ discard """ matrix: "--hint:processing" nimout: ''' compile start -... +.... warn_module.nim(6, 6) Hint: 'test' is declared but not used [XDeclaredButNotUsed] compile end ''' From c48347136f868e05345a189637da7a85042381bb Mon Sep 17 00:00:00 2001 From: Tomohiro Date: Sat, 27 Dec 2025 05:59:38 +0900 Subject: [PATCH 10/31] Refactoring #25302; don't store procedure's parameter types to `PType.sonsImpl` (#25351) --- compiler/ast.nim | 58 ++++++++++++++++++++++------ compiler/ic/ic.nim | 10 ++++- compiler/seminst.nim | 21 +++++----- compiler/semtypinst.nim | 7 ++-- compiler/sigmatch.nim | 2 - compiler/sinkparameter_inference.nim | 2 +- 6 files changed, 69 insertions(+), 31 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 556df74080..bc28cff845 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -549,18 +549,28 @@ proc addAllowNil*(father, son: PNode) {.inline.} = father.sons.add(son) proc add*(father, son: PType) = + assert father.kind != tyProc or father.sonsImpl.len == 0 assert son != nil father.sonsImpl.add son proc addAllowNil*(father, son: PType) {.inline.} = + assert father.kind != tyProc or father.sonsImpl.len == 0 father.sonsImpl.add son template `[]`*(n: PType, i: int): PType = if n.state == Partial: loadType(n) - n.sonsImpl[i] + if n.kind == tyProc and i > 0: + assert n.nImpl[i] != nil and n.nImpl[i].sym != nil + n.nImpl[i].sym.typ + else: + n.sonsImpl[i] template `[]=`*(n: PType, i: int; x: PType) = if n.state == Partial: loadType(n) - n.sonsImpl[i] = x + if n.kind == tyProc and i > 0: + assert n.nImpl[i] != nil and n.nImpl[i].sym != nil + n.nImpl[i].sym.typ = x + else: + n.sonsImpl[i] = x template `[]`*(n: PType, i: BackwardsIndex): PType = if n.state == Partial: loadType(n) @@ -806,7 +816,10 @@ proc replaceSon*(n: PNode; i: int; newson: PNode) {.inline.} = proc last*(n: PType): PType {.inline.} = if n.state == Partial: loadType(n) - n.sonsImpl[^1] + if n.kind == tyProc and n.nImpl.len > 1: + n.nImpl[^1].sym.typ + else: + n.sonsImpl[^1] proc elementType*(n: PType): PType {.inline.} = if n.state == Partial: loadType(n) @@ -842,7 +855,10 @@ proc setIndexType*(n, idx: PType) {.inline.} = proc firstParamType*(n: PType): PType {.inline.} = if n.state == Partial: loadType(n) - n.sonsImpl[1] + if n.kind == tyProc: + n.nImpl[1].sym.typ + else: + n.sonsImpl[1] proc firstGenericParam*(n: PType): PType {.inline.} = if n.state == Partial: loadType(n) @@ -914,10 +930,13 @@ proc `$`*(s: PSym): string = result = "" proc len*(n: PType): int {.inline.} = - result = n.sonsImpl.len + if n.kind == tyProc: + result = if n.nImpl == nil: 0 else: n.nImpl.len + else: + result = n.sonsImpl.len proc sameTupleLengths*(a, b: PType): bool {.inline.} = - result = a.sonsImpl.len == b.sonsImpl.len + result = a.len == b.len iterator tupleTypePairs*(a, b: PType): (int, PType, PType) = for i in 0 ..< a.len: @@ -1012,15 +1031,20 @@ proc newType*(kind: TTypeKind; idgen: IdGenerator; owner: PSym; son: sink PType alignImpl: defaultAlignment, itemId: id, uniqueId: id, sonsImpl: @[]) if son != nil: + assert kind != tyProc result.sonsImpl.add son when false: if result.itemId.module == 55 and result.itemId.item == 2: echo "KNID ", kind writeStackTrace() -proc setSons*(dest: PType; sons: sink seq[PType]) {.inline.} = dest.sonsImpl = sons -proc setSon*(dest: PType; son: sink PType) {.inline.} = dest.sonsImpl = @[son] +proc setSons*(dest: PType; sons: sink seq[PType]) {.inline.} = + assert dest.kind != tyProc or sons.len <= 1 + dest.sonsImpl = sons +proc setSon*(dest: PType; son: sink PType) {.inline.} = + dest.sonsImpl = @[son] proc setSonsLen*(dest: PType; len: int) {.inline.} = + assert dest.kind != tyProc or len <= 1 setLen(dest.sonsImpl, len) proc mergeLoc(a: var TLoc, b: TLoc) = @@ -1034,6 +1058,7 @@ proc newSons*(father: PNode, length: int) = setLen(father.sons, length) proc newSons*(father: PType, length: int) = + assert father.kind != tyProc or length <= 1 setLen(father.sonsImpl, length) proc truncateInferredTypeCandidates*(t: PType) {.inline.} = @@ -1058,8 +1083,16 @@ proc assignType*(dest, src: PType) = mergeLoc(dest.sym.locImpl, src.sym.loc) else: dest.symImpl = src.sym - newSons(dest, src.len) - for i in 0.. 0: + setLen(dest.sonsImpl, 1) + dest.sonsImpl[0] = src.sonsImpl[0] + else: + newSons(dest, src.len) + for i in 0.. 0: + # if kind == tyProc, parameter types are stored in t.n + # and you can access them with `kits` iterator. + # return type is stored in t.sons[0]. + p.types.add t[0].storeType(c, m) + else: + for kid in kids t: + p.types.add kid.storeType(c, m) c.addMissing t.sym p.sym = t.sym.safeItemId(c, m) c.addMissing t.owner diff --git a/compiler/seminst.nim b/compiler/seminst.nim index b34c7ef58e..a34467636a 100644 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -244,7 +244,8 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable, var result = instCopyType(cl, prc.typ) let originalParams = result.n result.n = originalParams.shallowCopy - for i, resulti in paramTypes(result): + for i in 1 ..< originalParams.len: + let resulti = originalParams[i].sym.typ # twrong_field_caching requires these 'resetIdTable' calls: if i > FirstParamAt: resetIdTable(cl.symMap) @@ -258,23 +259,23 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable, let needsTypeDescSkipping = resulti.kind == tyTypeDesc and tfUnresolved in resulti.flags if resulti.kind == tyFromExpr: resulti.incl tfNonConstExpr - result[i] = replaceTypeVarsT(cl, resulti) + var paramType = replaceTypeVarsT(cl, resulti) if needsStaticSkipping: - result[i] = result[i].skipTypes({tyStatic}) + paramType = paramType.skipTypes({tyStatic}) if needsTypeDescSkipping: - result[i] = result[i].skipTypes({tyTypeDesc}) - typeToFit = result[i] + paramType = paramType.skipTypes({tyTypeDesc}) + typeToFit = paramType # ...otherwise, we use the instantiated type in `fitNode` if (typeToFit.kind != tyTypeDesc or typeToFit.base.kind != tyNone) and (typeToFit.kind != tyStatic): - typeToFit = result[i] + typeToFit = paramType internalAssert c.config, originalParams[i].kind == nkSym let oldParam = originalParams[i].sym let param = copySym(oldParam, c.idgen) setOwner(param, prc) - param.typ = result[i] + param.typ = paramType # The default value is instantiated and fitted against the final # concrete param type. We avoid calling `replaceTypeVarsN` on the @@ -305,12 +306,12 @@ proc instantiateProcType(c: PContext, pt: LayeredIdTable, param.ast.typ = def.typ else: param.ast = fitNodePostMatch(c, typeToFit, converted) - param.typ = result[i] + param.typ = paramType result.n[i] = newSymNode(param) - if isRecursiveStructuralType(result[i]): + if isRecursiveStructuralType(paramType): localError(c.config, originalParams[i].sym.info, "illegal recursion in type '" & typeToString(result[i]) & "'") - propagateToOwner(result, result[i]) + propagateToOwner(result, paramType) addDecl(c, param) resetIdTable(cl.symMap) diff --git a/compiler/semtypinst.nim b/compiler/semtypinst.nim index 1d3f51480b..ed9200f7f0 100644 --- a/compiler/semtypinst.nim +++ b/compiler/semtypinst.nim @@ -553,14 +553,12 @@ proc eraseVoidParams*(t: PType) = for i in FirstParamAt.. Date: Mon, 29 Dec 2025 02:45:07 +1100 Subject: [PATCH 11/31] Fix `tupleLen` not skipping aliases (#25392) This code was failing to compile with `Error: unhandled exception: semmagic.nim(247, 5) operand.kind == tyTuple tyAlias [AssertionDefect]` ```nim import std/typetraits type Bar[T] = T Foo = Bar[tuple[a: int]] echo Foo.tupleLen ``` Fix was just making `tupleLen` skip alias types also --- compiler/semmagic.nim | 2 +- tests/metatype/ttypetraits.nim | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compiler/semmagic.nim b/compiler/semmagic.nim index 9de290f4ce..8a91d820f0 100644 --- a/compiler/semmagic.nim +++ b/compiler/semmagic.nim @@ -243,7 +243,7 @@ proc evalTypeTrait(c: PContext; traitCall: PNode, operand: PType, context: PSym) let cond = operand.kind == tyTuple and operand.n != nil result = newIntNodeT(toInt128(ord(cond)), traitCall, c.idgen, c.graph) of "tupleLen": - var operand = operand.skipTypes({tyGenericInst}) + var operand = operand.skipTypes({tyGenericInst, tyAlias}) assert operand.kind == tyTuple, $operand.kind result = newIntNodeT(toInt128(operand.len), traitCall, c.idgen, c.graph) of "distinctBase": diff --git a/tests/metatype/ttypetraits.nim b/tests/metatype/ttypetraits.nim index 74ace75c3a..0107f6b049 100644 --- a/tests/metatype/ttypetraits.nim +++ b/tests/metatype/ttypetraits.nim @@ -194,6 +194,11 @@ block: # tupleLen MyGenericTuple2Alias2 = MyGenericTuple2Alias[float] static: doAssert MyGenericTuple2Alias2.tupleLen == 3 + type + MyGenericTuple3[T] = T + MyGenericTuple3Alias = MyGenericTuple3[(string, int)] + static: doAssert MyGenericTuple3Alias.tupleLen == 2 + static: doAssert (int, float).tupleLen == 2 static: doAssert (1, ).tupleLen == 1 static: doAssert ().tupleLen == 0 From 02893e2f4c2bca4cb107ce7673c615362a91e33f Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 29 Dec 2025 00:20:33 +0100 Subject: [PATCH 12/31] IC: code generation progress (#25379) --- compiler/ast2nif.nim | 184 +++++++++++++++++++-------- compiler/ccgstmts.nim | 24 ++-- compiler/ccgtypes.nim | 6 +- compiler/cgen.nim | 45 ++++--- compiler/cgendata.nim | 2 +- compiler/ic/cbackend.nim | 2 +- compiler/ic/enum2nif.nim | 260 +++++++++++++++++++------------------- compiler/modulegraphs.nim | 34 ++--- compiler/nifbackend.nim | 118 +++++++++-------- compiler/options.nim | 1 + compiler/pipelines.nim | 7 +- compiler/renderer.nim | 3 + compiler/scriptconfig.nim | 2 + tools/enumgen.nim | 80 ++++++++++-- 14 files changed, 465 insertions(+), 303 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index c419a934d8..01830b65fe 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -164,7 +164,7 @@ type const # Symbol kinds that are always local to a proc and should never have module suffix - skLocalSymKinds = {skParam, skGenericParam, skForVar, skResult, skTemp} + skLocalSymKinds = {skParam, skForVar, skResult, skTemp} proc isLocalSym(sym: PSym): bool {.inline.} = sym.kindImpl in skLocalSymKinds or @@ -362,10 +362,7 @@ proc writeSymDef(w: var Writer; dest: var TokenBuf; sym: PSym) = writeSym(w, dest, sym.ownerFieldImpl) # Store the AST for routine symbols and constants # Constants need their AST for astdef() to return the constant's value - if sym.kindImpl in routineKinds + {skConst}: - writeNode(w, dest, sym.astImpl, forAst = true) - else: - dest.addDotToken + writeNode(w, dest, sym.astImpl, forAst = true) writeLoc w, dest, sym.locImpl writeNode(w, dest, sym.constraintImpl) writeSym(w, dest, sym.instantiatedFromImpl) @@ -438,14 +435,20 @@ proc addLocalSym(w: var Writer; n: PNode) = w.locals.incl(n.sym.itemId) proc addLocalSyms(w: var Writer; n: PNode) = - if n.kind in {nkIdentDefs, nkVarTuple}: + case n.kind + of nkIdentDefs, nkVarTuple: # nkIdentDefs: [ident1, ident2, ..., type, default] # All children except the last two are identifiers for i in 0 ..< max(0, n.len - 2): addLocalSyms(w, n[i]) - elif n.kind == nkSym: + of nkPostfix: + addLocalSyms(w, n[1]) + of nkPragmaExpr: + addLocalSyms(w, n[0]) + of nkSym: addLocalSym(w, n) - + else: + discard proc trInclude(w: var Writer; n: PNode) = w.deps.addParLe pool.tags.getOrIncl(toNifTag(n.kind)), trLineInfo(w, n.info) @@ -456,6 +459,9 @@ proc trInclude(w: var Writer; n: PNode) = w.deps.addStrLit child.strVal # raw string literal, no wrapper needed w.deps.addParRi +proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = + cachedModuleSuffix(conf, f) + proc trImport(w: var Writer; n: PNode) = for child in n: if child.kind == nkSym: @@ -464,7 +470,7 @@ proc trImport(w: var Writer; n: PNode) = w.deps.addDotToken # type let s = child.sym assert s.kindImpl == skModule - let fp = toFullPath(w.infos.config, s.positionImpl.FileIndex) + let fp = moduleSuffix(w.infos.config, s.positionImpl.FileIndex) w.deps.addStrLit fp # raw string literal, no wrapper needed w.deps.addParRi @@ -512,14 +518,14 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = of nkNilLit: w.withNode dest, n: discard - of nkLetSection, nkVarSection, nkConstSection, nkGenericParams: + of nkLetSection, nkVarSection, nkConstSection: # Track local variables declared in let/var sections w.withNode dest, n: for child in n: addLocalSyms w, child # Process the child node writeNode(w, dest, child, forAst) - of nkForStmt, nkTypeDef: + of nkForStmt: # Track for loop variable (first child is the loop variable) w.withNode dest, n: if n.len > 0: @@ -535,7 +541,7 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = addLocalSyms(w, n[i]) writeNode(w, dest, n[i], forAst) dec w.inProc - of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef: + of nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef, nkTemplateDef: # For top-level named routines (not forAst), just write the symbol. # The full AST will be stored in the symbol's sdef. if not forAst and n[namePos].kind == nkSym: @@ -602,16 +608,53 @@ proc writeNode(w: var Writer; dest: var TokenBuf; n: PNode; forAst = false) = # Write the export statement as a regular node w.withNode dest, n: for i in 0 ..< n.len: - writeNode(w, dest, n[i], forAst) + if n[i].kind == nkSym and n[i].sym.kindImpl == skModule: + discard "do not write module syms here" + else: + writeNode(w, dest, n[i], forAst) else: w.withNode dest, n: for i in 0 ..< n.len: writeNode(w, dest, n[i], forAst) -proc writeToplevelNode(w: var Writer; dest: var TokenBuf; n: PNode) = +proc writeGlobal(w: var Writer; dest: var TokenBuf; n: PNode) = + case n.kind + of nkVarTuple: + writeNode(w, dest, n) + of nkIdentDefs, nkConstDef: + # nkIdentDefs: [ident1, ident2, ..., type, default] + # All children except the last two are identifiers + for i in 0 ..< max(0, n.len - 2): + writeGlobal(w, dest, n[i]) + of nkPostfix: + writeGlobal(w, dest, n[1]) + of nkPragmaExpr: + writeGlobal(w, dest, n[0]) + of nkSym: + writeSym(w, dest, n.sym) + else: + discard + +proc writeGlobals(w: var Writer; dest: var TokenBuf; n: PNode) = + w.withNode dest, n: + for child in n: + writeGlobal(w, dest, child) + +proc writeToplevelNode(w: var Writer; dest, bottom: var TokenBuf; n: PNode) = case n.kind of nkStmtList, nkStmtListExpr: - for son in n: writeToplevelNode(w, dest, son) + for son in n: writeToplevelNode(w, dest, bottom, son) + of nkEmpty: + discard "ignore" + of nkTypeSection, nkCommentStmt, nkMixinStmt, nkBindStmt, nkUsingStmt, + nkPragma, + nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef, nkMacroDef, nkTemplateDef: + # We write purely declarative nodes at the bottom of the file + writeNode(w, bottom, n) + of nkConstSection: + writeGlobals(w, bottom, n) + of nkLetSection, nkVarSection: + writeGlobals(w, dest, n) else: writeNode w, dest, n @@ -652,6 +695,7 @@ let repMethodTag = registerTag("repmethod") #let repClassTag = registerTag("repclass") let includeTag = registerTag("include") let importTag = registerTag("import") +let implTag = registerTag("implementation") proc writeOp(w: var Writer; content: var TokenBuf; op: LogEntry) = case op.kind @@ -706,8 +750,14 @@ proc writeNifModule*(config: ConfigRef; thisModule: int32; n: PNode; if op.module == thisModule.int: writeOp(w, content, op) - w.writeToplevelNode content, n + var bottom = createTokenBuf(300) + w.writeToplevelNode content, bottom, n + # the implTag is used to tell the loader that the + # bottom of the file is the implementation of the module: + content.addParLe implTag, NoLineInfo + content.addParRi() + content.add bottom content.addParRi() let m = modname(w.currentModule, w.infos.config) @@ -817,10 +867,14 @@ proc cursorFromIndexEntry(c: var DecodeContext; module: FileIndex; entry: NifInd nifcursors.parse(s[], buf, entry.info) result = cursorAt(buf, 0) -proc moduleId(c: var DecodeContext; suffix: string): FileIndex = +type + LoadFlag* = enum + LoadFullAst, AlwaysLoadInterface + +proc moduleId(c: var DecodeContext; suffix: string; flags: set[LoadFlag] = {}): FileIndex = var isKnownFile = false result = c.infos.config.registerNifSuffix(suffix, isKnownFile) - if not isKnownFile: + if not isKnownFile or AlwaysLoadInterface in flags: let modFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".nif")).string let idxFile = (getNimcacheDir(c.infos.config) / RelativeFile(suffix & ".s.idx.nif")).string if not fileExists(modFile): @@ -1099,6 +1153,8 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: inc n var isKnownFile = false s.positionImpl = int c.infos.config.registerNifSuffix(thisModule, isKnownFile) + # do to the precompiled mechanism things end up as main modules which are not! + excl s.flagsImpl, sfMainModule else: loadField s.positionImpl @@ -1110,12 +1166,7 @@ proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: s.ownerFieldImpl = loadSymStub(c, n, thisModule, localSyms) # Load the AST for routine symbols and constants # Constants need their AST for astdef() to return the constant's value - if s.kindImpl in routineKinds + {skConst}: - s.astImpl = loadNode(c, n, thisModule, localSyms) - elif n.kind == DotToken: - inc n - else: - raiseAssert "expected '.' for non-routine symbol AST but got " & $n.kind + s.astImpl = loadNode(c, n, thisModule, localSyms) loadLoc c, n, s.locImpl s.constraintImpl = loadNode(c, n, thisModule, localSyms) s.instantiatedFromImpl = loadSymStub(c, n, thisModule, localSyms) @@ -1303,9 +1354,6 @@ proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; else: raiseAssert "expected string literal but got " & $n.kind -proc moduleSuffix(conf: ConfigRef; f: FileIndex): string = - cachedModuleSuffix(conf, f) - proc loadSymFromIndexEntry(c: var DecodeContext; module: FileIndex; nifName: string; entry: NifIndexEntry; thisModule: string): PSym = ## Loads a symbol from the NIF index entry using the entry directly. @@ -1494,8 +1542,32 @@ proc nextSubtree(r: var Stream; dest: var TokenBuf; tok: var PackedToken) = dec nested if nested == 0: break -proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suffix: string; logOps: var seq[LogEntry]; module: int): PNode = - result = newNode(nkStmtList) +type + ModuleSuffix* = distinct string + PrecompiledModule* = object + topLevel*: PNode # top level statements of the main module + deps*: seq[ModuleSuffix] # other modules we need to process the top level statements of + logOps*: seq[LogEntry] + module*: PSym # set by modulegraphs.nim! + +proc loadImport(c: var DecodeContext; s: var Stream; deps: var seq[ModuleSuffix]; tok: var PackedToken) = + tok = next(s) # skip `(import` + if tok.kind == DotToken: + tok = next(s) # skip dot + if tok.kind == DotToken: + tok = next(s) # skip dot + if tok.kind == StringLit: + deps.add ModuleSuffix(pool.strings[tok.litId]) + tok = next(s) + else: + raiseAssert "expected StringLit but got " & $tok.kind + if tok.kind == ParRi: + tok = next(s) # skip ) + else: + raiseAssert "expected ParRi but got " & $tok.kind + +proc processTopLevel(c: var DecodeContext; s: var Stream; flags: set[LoadFlag] = {}; suffix: string; module: int): PrecompiledModule = + result = PrecompiledModule(topLevel: newNode(nkStmtList)) var localSyms = initTable[string, PSym]() var t = next(s) # skip dot @@ -1512,60 +1584,62 @@ proc processTopLevel(c: var DecodeContext; s: var Stream; loadFullAst: bool; suf var cursor = cursorAt(buf, 0) let replayNode = loadNode(c, cursor, suffix, localSyms) if replayNode != nil: - result.sons.add replayNode + result.topLevel.sons.add replayNode t = next(s) if t.kind == ParRi: t = next(s) else: raiseAssert "expected ParRi but got " & $t.kind elif t.tagId == repConverterTag: - t = loadLogOp(c, logOps, s, ConverterEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, ConverterEntry, attachedTrace, module) elif t.tagId == repDestroyTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedDestructor, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedDestructor, module) elif t.tagId == repWasMovedTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedWasMoved, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedWasMoved, module) elif t.tagId == repCopyTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedAsgn, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedAsgn, module) elif t.tagId == repSinkTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedSink, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedSink, module) elif t.tagId == repDupTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedDup, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedDup, module) elif t.tagId == repTraceTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedTrace, module) elif t.tagId == repDeepCopyTag: - t = loadLogOp(c, logOps, s, HookEntry, attachedDeepCopy, module) + t = loadLogOp(c, result.logOps, s, HookEntry, attachedDeepCopy, module) elif t.tagId == repEnumToStrTag: - t = loadLogOp(c, logOps, s, EnumToStrEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, EnumToStrEntry, attachedTrace, module) elif t.tagId == repMethodTag: - t = loadLogOp(c, logOps, s, MethodEntry, attachedTrace, module) + t = loadLogOp(c, result.logOps, s, MethodEntry, attachedTrace, module) #elif t.tagId == repClassTag: # t = loadLogOp(c, logOps, s, ClassEntry, attachedTrace, module) - elif t.tagId == includeTag or t.tagId == importTag: + elif t.tagId == includeTag: t = skipTree(s) - elif loadFullAst: + elif t.tagId == importTag: + loadImport(c, s, result.deps, t) + elif t.tagId == implTag: + cont = false + elif LoadFullAst in flags: # Parse the full statement var buf = createTokenBuf(50) nextSubtree(s, buf, t) + t = next(s) # skip ParRi var cursor = cursorAt(buf, 0) let stmtNode = loadNode(c, cursor, suffix, localSyms) if stmtNode != nil: - result.sons.add stmtNode + result.topLevel.sons.add stmtNode else: cont = false else: cont = false -proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; - logOps: var seq[LogEntry]; - loadFullAst: bool = false): PNode = - let suffix = moduleSuffix(c.infos.config, f) - - # Ensure module index is loaded - moduleId returns the FileIndex for this suffix - let module = moduleId(c, suffix) +proc loadNifModule*(c: var DecodeContext; suffix: ModuleSuffix; interf, interfHidden: var TStrTable; + flags: set[LoadFlag] = {}): PrecompiledModule = + # Ensure module index is loaded - moduleId returns the FileIndex for this suffix + let module = moduleId(c, string(suffix), flags) # Populate interface tables from the NIF index structure # Symbols are created as stubs (Partial state) and will be loaded lazily via loadSym - populateInterfaceTablesFromIndex(c, module, interf, interfHidden, suffix) + populateInterfaceTablesFromIndex(c, module, interf, interfHidden, string(suffix)) # Load the module AST (or just replay actions if loadFullAst is false) let s = addr c.mods[module].stream @@ -1575,10 +1649,14 @@ proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: va if t.kind == ParLe and pool.tags[t.tagId] == toNifTag(nkStmtList): t = next(s[]) # skip (stmts t = next(s[]) # skip flags - result = processTopLevel(c, s[], loadFullAst, suffix, logOps, f.int) + result = processTopLevel(c, s[], flags, string(suffix), module.int) else: - result = newNode(nkStmtList) + result = PrecompiledModule(topLevel: newNode(nkStmtList)) +proc loadNifModule*(c: var DecodeContext; f: FileIndex; interf, interfHidden: var TStrTable; + flags: set[LoadFlag] = {}): PrecompiledModule = + let suffix = ModuleSuffix(moduleSuffix(c.infos.config, f)) + result = loadNifModule(c, suffix, interf, interfHidden, flags) when isMainModule: import std / syncio diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index fa7440aa8e..7deaa18157 100644 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -126,7 +126,7 @@ proc genVarTuple(p: BProc, n: PNode) = let vn = n[i] let v = vn.sym if sfCompileTime in v.flags: continue - ensureMutable v + backendEnsureMutable v if sfGlobal in v.flags: assignGlobalVar(p, vn, "") genObjectInit(p, cpsInit, v.typ, v.locImpl, constructObj) @@ -136,7 +136,7 @@ proc genVarTuple(p: BProc, n: PNode) = initLocalVar(p, v, immediateAsgn=isAssignedImmediately(p.config, n[^1])) var field = initLoc(locExpr, vn, tup.storage) let rtup = rdLoc(tup) - let fieldName = + let fieldName = if t.kind == tyTuple: "Field" & $i else: @@ -490,14 +490,17 @@ proc genClosureVar(p: BProc, a: PNode) = constructLoc(p, v) proc genVarStmt(p: BProc, n: PNode) = - for it in n.sons: - if it.kind == nkCommentStmt: continue - if it.kind == nkIdentDefs: + for it in n: + case it.kind + of nkCommentStmt: discard + of nkIdentDefs: # can be a lifted var nowadays ... if it[0].kind == nkSym: genSingleVar(p, it) else: genClosureVar(p, it) + of nkSym: + genSingleVar(p, it.sym, newSymNode(it.sym), it.sym.astdef) else: genVarTuple(p, it) @@ -740,9 +743,10 @@ proc genBlock(p: BProc, n: PNode, d: var TLoc) = # named block? assert(n[0].kind == nkSym) var sym = n[0].sym - ensureMutable sym + backendEnsureMutable sym sym.locImpl.k = locOther - sym.position = p.breakIdx+1 + sym.positionImpl = p.breakIdx+1 + # ^ IC: review this expr(p, n[1], d) endSimpleBlock(p, scope) @@ -1255,7 +1259,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = initElifBranch(p.s(cpsStmts), ifStmt, orExpr) if exvar != nil: fillLocalName(p, exvar.sym) - ensureMutable exvar.sym + backendEnsureMutable exvar.sym fillLoc(exvar.sym.locImpl, locTemp, exvar, OnStack) linefmt(p, cpsStmts, "$1 $2 = T$3_;$n", [getTypeDesc(p.module, exvar.sym.typ), rdLoc(exvar.sym.loc), rope(etmp+1)]) @@ -1304,7 +1308,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) = if isImportedException(typeNode.typ, p.config): let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:` fillLocalName(p, exvar.sym) - ensureMutable exvar.sym + backendEnsureMutable exvar.sym fillLoc(exvar.sym.locImpl, locTemp, exvar, OnStack) startBlockWith(p): lineCg(p, cpsStmts, "catch ($1& $2) {$n", [getTypeDesc(p.module, typeNode.typ), rdLoc(exvar.sym.loc)]) @@ -1396,7 +1400,7 @@ proc genTryCppOld(p: BProc, t: PNode, d: var TLoc) = if t[i][j].isInfixAs(): let exvar = t[i][j][2] # ex1 in `except ExceptType as ex1:` fillLocalName(p, exvar.sym) - ensureMutable exvar.sym + backendEnsureMutable exvar.sym fillLoc(exvar.sym.locImpl, locTemp, exvar, OnUnknown) startBlockWith(p): lineCg(p, cpsStmts, "catch ($1& $2) {$n", [getTypeDesc(p.module, t[i][j][1].typ), rdLoc(exvar.sym.loc)]) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index 399b07d1a5..b8de2a6de5 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -1864,7 +1864,7 @@ proc genTypeInfoV2Impl(m: BModule; t, origType: PType, name: Rope; info: TLineIn proc myModuleOpenForCodegen(m: BModule; idx: FileIndex): bool {.inline.} = if moduleOpenForCodegen(m.g.graph, idx): - result = idx.int < m.g.modules.len and m.g.modules[idx.int] != nil + result = idx.int < m.g.mods.len and m.g.mods[idx.int] != nil else: result = false @@ -1898,7 +1898,7 @@ proc genTypeInfoV2(m: BModule; t: PType; info: TLineInfo): Rope = let owner = t.skipTypes(typedescPtrs).itemId.module if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner): # make sure the type info is created in the owner module - discard genTypeInfoV2(m.g.modules[owner], origType, info) + discard genTypeInfoV2(m.g.mods[owner], origType, info) # reference the type info as extern here cgsym(m, "TNimTypeV2") declareNimType(m, "TNimTypeV2", result, owner) @@ -1983,7 +1983,7 @@ proc genTypeInfoV1(m: BModule; t: PType; info: TLineInfo): Rope = var owner = t.skipTypes(typedescPtrs).itemId.module if owner != m.module.position and myModuleOpenForCodegen(m, FileIndex owner): # make sure the type info is created in the owner module - discard genTypeInfoV1(m.g.modules[owner], origType, info) + discard genTypeInfoV1(m.g.mods[owner], origType, info) # reference the type info as extern here cgsym(m, "TNimType") cgsym(m, "TNimNode") diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 48bf1ad6e3..b380b136d2 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -67,19 +67,19 @@ proc findPendingModule(m: BModule, s: PSym): BModule = # TODO fixme if m.config.symbolFiles == v2Sf or optCompress in m.config.globalOptions: let ms = s.itemId.module #getModule(s) - result = m.g.modules[ms] + result = m.g.mods[ms] elif m.config.cmd in {cmdNifC, cmdM}: var ms = getModule(s) registerModule m.g.graph, ms - if ms.position >= m.g.modules.len: + if ms.position >= m.g.mods.len: result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms)) else: - result = m.g.modules[ms.position] + result = m.g.mods[ms.position] if result == nil: result = newModule(m.g, ms, m.config, idGeneratorFromModule(ms)) else: var ms = getModule(s) - result = m.g.modules[ms.position] + result = m.g.mods[ms.position] proc initLoc(k: TLocKind, lode: PNode, s: TStorageLoc, flags: TLocFlags = {}): TLoc = result = TLoc(k: k, storage: s, lode: lode, @@ -133,10 +133,10 @@ proc getModuleDllPath(m: BModule): Rope = result = makeCString(dir.string & "/" & filename) proc getModuleDllPath(m: BModule, module: int): Rope = - result = getModuleDllPath(m.g.modules[module]) + result = getModuleDllPath(m.g.mods[module]) proc getModuleDllPath(m: BModule, s: PSym): Rope = - result = getModuleDllPath(m.g.modules[s.itemId.module]) + result = getModuleDllPath(m.g.mods[s.itemId.module]) import std/macros @@ -1720,9 +1720,12 @@ proc genMainProcs(m: BModule) = proc genMainProcsWithResult(m: BModule) = genMainProcs(m) - var res = "nim_program_result" - if m.hcrOn: res = cDeref(res) - m.s[cfsProcs].addReturn(res) + if m.config.cmd != cmdNifC: + var res = "nim_program_result" + if m.hcrOn: res = cDeref(res) + m.s[cfsProcs].addReturn(res) + else: + m.s[cfsProcs].addReturn(cIntValue(0)) proc genNimMainInner(m: BModule) = m.s[cfsProcs].addDeclWithVisibility(Private): @@ -1960,7 +1963,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = if m.hcrOn: var hcrModuleMeta = newBuilder("") - let systemModulePath = getModuleDllPath(m, g.modules[g.graph.config.m.systemFileIdx.int].module) + let systemModulePath = getModuleDllPath(m, g.mods[g.graph.config.m.systemFileIdx.int].module) let mainModulePath = getModuleDllPath(m, m.module) hcrModuleMeta.addDeclWithVisibility(Private): hcrModuleMeta.addArrayVarWithInitializer(kind = Local, @@ -1977,7 +1980,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) = g.graph.importDeps.withValue(FileIndex(m.module.position), deps): for curr in deps[]: hcrModuleMeta.addField(modules, ""): - hcrModuleMeta.add(getModuleDllPath(m, g.modules[curr.int].module)) + hcrModuleMeta.add(getModuleDllPath(m, g.mods[curr.int].module)) hcrModuleMeta.addField(modules, ""): hcrModuleMeta.add("\"\"") hcrModuleMeta.addDeclWithVisibility(ExportLib): @@ -2170,6 +2173,8 @@ proc genInitCode(m: BModule) = else: prcBody.add(extract(m.thing.s(section))) + #echo "PRE INIT PROC ", m.module.name.s, " ", m.s[cfsVars].buf.len + if m.preInitProc.s(cpsInit).buf.len > 0 or m.preInitProc.s(cpsStmts).buf.len > 0: # Give this small function its own scope prcBody.addScope(): @@ -2386,10 +2391,10 @@ proc newModule(g: BModuleList; module: PSym; conf: ConfigRef; idgen: IdGenerator # we should create only one cgen module for each module sym result = rawNewModule(g, module, conf) result.idgen = idgen - if module.position >= g.modules.len: - setLen(g.modules, module.position + 1) + if module.position >= g.mods.len: + setLen(g.mods, module.position + 1) #growCache g.modules, module.position - g.modules[module.position] = result + g.mods[module.position] = result template injectG() {.dirty.} = if graph.backend == nil: @@ -2523,13 +2528,7 @@ proc shouldRecompile(m: BModule; code: Rope, cfile: Cfile): bool = rawMessage(m.config, errCannotOpenFile, cfile.cname.string) result = true -# We need 2 different logics here: pending modules (including -# 'nim__dat') may require file merging for the combination of dead code -# elimination and incremental compilation! Non pending modules need no -# such logic and in fact the logic hurts for the main module at least; -# it would generate multiple 'main' procs, for instance. - -proc writeModule(m: BModule, pending: bool) = +proc writeModule(m: BModule) = let cfile = getCFile(m) if moduleHasChanged(m.g.graph, m.module): genInitCode(m) @@ -2658,7 +2657,7 @@ proc genForwardedProcs(g: BModuleList) = while g.forwardedProcs.len > 0: let prc = g.forwardedProcs.pop() - m = g.modules[prc.itemId.module] + m = g.mods[prc.itemId.module] if sfForward in prc.flags: internalError(m.config, prc.info, "still forwarded: " & prc.name.s) @@ -2674,6 +2673,6 @@ proc cgenWriteModules*(backend: RootRef, config: ConfigRef) = genForwardedProcs(g) for m in cgenModules(g): - m.writeModule(pending=true) + m.writeModule() writeMapping(config, g.mapping) if g.generatedHeader != nil: writeHeader(g.generatedHeader) diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim index 479babb0b9..5b5668024a 100644 --- a/compiler/cgendata.nim +++ b/compiler/cgendata.nim @@ -117,7 +117,7 @@ type BModuleList* = ref object of RootObj mainModProcs*, mainModInit*, otherModsInit*, mainDatInit*: Builder mapping*: Rope # the generated mapping file (if requested) - modules*: seq[BModule] # list of all compiled modules + mods*: seq[BModule] # list of all compiled modules modulesClosed*: seq[BModule] # list of the same compiled modules, but in the order they were closed forwardedProcs*: seq[PSym] # procs that did not yet have a body generatedHeader*: BModule diff --git a/compiler/ic/cbackend.nim b/compiler/ic/cbackend.nim index 1cf5301bc0..0ea7d66e59 100644 --- a/compiler/ic/cbackend.nim +++ b/compiler/ic/cbackend.nim @@ -40,7 +40,7 @@ proc setupBackendModule(g: ModuleGraph; m: var LoadedModule) = var bmod = cgen.newModule(BModuleList(g.backend), m.module, g.config, idgenFromLoadedModule(m)) proc generateCodeForModule(g: ModuleGraph; m: var LoadedModule; alive: var AliveSyms) = - var bmod = BModuleList(g.backend).modules[m.module.position] + var bmod = BModuleList(g.backend).mods[m.module.position] assert bmod != nil bmod.flags.incl useAliveDataFromDce bmod.alive = move alive[m.module.position] diff --git a/compiler/ic/enum2nif.nim b/compiler/ic/enum2nif.nim index bb0ed83ad1..4b7860fc96 100644 --- a/compiler/ic/enum2nif.nim +++ b/compiler/ic/enum2nif.nim @@ -404,140 +404,140 @@ proc parse*(t: typedesc[TSymKind]; s: string): TSymKind = proc toNifTag*(s: TTypeKind): string = case s - of tyNone: "none" - of tyBool: "bool" - of tyChar: "char" - of tyEmpty: "empty" - of tyAlias: "alias" - of tyNil: "nil" - of tyUntyped: "untyped" - of tyTyped: "typed" - of tyTypeDesc: "typedesc" - of tyGenericInvocation: "ginvoke" - of tyGenericBody: "gbody" - of tyGenericInst: "ginst" - of tyGenericParam: "gparam" - of tyDistinct: "distinct" - of tyEnum: "enum" - of tyOrdinal: "ordinal" - of tyArray: "array" - of tyObject: "object" - of tyTuple: "tuple" - of tySet: "set" - of tyRange: "range" - of tyPtr: "ptr" - of tyRef: "ref" - of tyVar: "mut" - of tySequence: "seq" - of tyProc: "proctype" - of tyPointer: "pointer" - of tyOpenArray: "openarray" - of tyString: "string" - of tyCstring: "cstring" - of tyForward: "forward" - of tyInt: "int" - of tyInt8: "int8" - of tyInt16: "int16" - of tyInt32: "int32" - of tyInt64: "int64" - of tyFloat: "float" - of tyFloat32: "float32" - of tyFloat64: "float64" - of tyFloat128: "float128" - of tyUInt: "uint" - of tyUInt8: "uint8" - of tyUInt16: "uint16" - of tyUInt32: "uint32" - of tyUInt64: "uint64" - of tyOwned: "owned" - of tySink: "sink" - of tyLent: "lent" - of tyVarargs: "varargs" - of tyUncheckedArray: "uarray" - of tyError: "error" - of tyBuiltInTypeClass: "bconcept" - of tyUserTypeClass: "uconcept" - of tyUserTypeClassInst: "uconceptinst" - of tyCompositeTypeClass: "cconcept" - of tyInferred: "inferred" - of tyAnd: "and" - of tyOr: "or" - of tyNot: "not" - of tyAnything: "anything" - of tyStatic: "static" - of tyFromExpr: "fromx" - of tyConcept: "concept" - of tyVoid: "void" - of tyIterable: "iterable" + of tyNone: "n0" + of tyBool: "b0" + of tyChar: "c0" + of tyEmpty: "e0" + of tyAlias: "a0" + of tyNil: "n1" + of tyUntyped: "U0" + of tyTyped: "t0" + of tyTypeDesc: "t1" + of tyGenericInvocation: "g0" + of tyGenericBody: "g1" + of tyGenericInst: "g2" + of tyGenericParam: "g4" + of tyDistinct: "d0" + of tyEnum: "e1" + of tyOrdinal: "o0" + of tyArray: "a1" + of tyObject: "o1" + of tyTuple: "t2" + of tySet: "s0" + of tyRange: "r0" + of tyPtr: "p0" + of tyRef: "r1" + of tyVar: "v0" + of tySequence: "s1" + of tyProc: "p1" + of tyPointer: "p2" + of tyOpenArray: "o3" + of tyString: "s2" + of tyCstring: "c1" + of tyForward: "F0" + of tyInt: "i0" + of tyInt8: "i1" + of tyInt16: "i2" + of tyInt32: "i3" + of tyInt64: "i4" + of tyFloat: "f0" + of tyFloat32: "f1" + of tyFloat64: "f2" + of tyFloat128: "f3" + of tyUInt: "u0" + of tyUInt8: "u1" + of tyUInt16: "u2" + of tyUInt32: "u3" + of tyUInt64: "u4" + of tyOwned: "o2" + of tySink: "s3" + of tyLent: "L0" + of tyVarargs: "v1" + of tyUncheckedArray: "U1" + of tyError: "e2" + of tyBuiltInTypeClass: "b1" + of tyUserTypeClass: "U2" + of tyUserTypeClassInst: "U3" + of tyCompositeTypeClass: "c2" + of tyInferred: "I0" + of tyAnd: "a2" + of tyOr: "o4" + of tyNot: "n2" + of tyAnything: "a3" + of tyStatic: "s4" + of tyFromExpr: "F1" + of tyConcept: "c3" + of tyVoid: "v2" + of tyIterable: "I1" proc parse*(t: typedesc[TTypeKind]; s: string): TTypeKind = case s - of "none": tyNone - of "bool": tyBool - of "char": tyChar - of "empty": tyEmpty - of "alias": tyAlias - of "nil": tyNil - of "untyped": tyUntyped - of "typed": tyTyped - of "typedesc": tyTypeDesc - of "ginvoke": tyGenericInvocation - of "gbody": tyGenericBody - of "ginst": tyGenericInst - of "gparam": tyGenericParam - of "distinct": tyDistinct - of "enum": tyEnum - of "ordinal": tyOrdinal - of "array": tyArray - of "object": tyObject - of "tuple": tyTuple - of "set": tySet - of "range": tyRange - of "ptr": tyPtr - of "ref": tyRef - of "mut": tyVar - of "seq": tySequence - of "proctype": tyProc - of "pointer": tyPointer - of "openarray": tyOpenArray - of "string": tyString - of "cstring": tyCstring - of "forward": tyForward - of "int": tyInt - of "int8": tyInt8 - of "int16": tyInt16 - of "int32": tyInt32 - of "int64": tyInt64 - of "float": tyFloat - of "float32": tyFloat32 - of "float64": tyFloat64 - of "float128": tyFloat128 - of "uint": tyUInt - of "uint8": tyUInt8 - of "uint16": tyUInt16 - of "uint32": tyUInt32 - of "uint64": tyUInt64 - of "owned": tyOwned - of "sink": tySink - of "lent": tyLent - of "varargs": tyVarargs - of "uarray": tyUncheckedArray - of "error": tyError - of "bconcept": tyBuiltInTypeClass - of "uconcept": tyUserTypeClass - of "uconceptinst": tyUserTypeClassInst - of "cconcept": tyCompositeTypeClass - of "inferred": tyInferred - of "and": tyAnd - of "or": tyOr - of "not": tyNot - of "anything": tyAnything - of "static": tyStatic - of "fromx": tyFromExpr - of "concept": tyConcept - of "void": tyVoid - of "iterable": tyIterable + of "n0": tyNone + of "b0": tyBool + of "c0": tyChar + of "e0": tyEmpty + of "a0": tyAlias + of "n1": tyNil + of "U0": tyUntyped + of "t0": tyTyped + of "t1": tyTypeDesc + of "g0": tyGenericInvocation + of "g1": tyGenericBody + of "g2": tyGenericInst + of "g4": tyGenericParam + of "d0": tyDistinct + of "e1": tyEnum + of "o0": tyOrdinal + of "a1": tyArray + of "o1": tyObject + of "t2": tyTuple + of "s0": tySet + of "r0": tyRange + of "p0": tyPtr + of "r1": tyRef + of "v0": tyVar + of "s1": tySequence + of "p1": tyProc + of "p2": tyPointer + of "o3": tyOpenArray + of "s2": tyString + of "c1": tyCstring + of "F0": tyForward + of "i0": tyInt + of "i1": tyInt8 + of "i2": tyInt16 + of "i3": tyInt32 + of "i4": tyInt64 + of "f0": tyFloat + of "f1": tyFloat32 + of "f2": tyFloat64 + of "f3": tyFloat128 + of "u0": tyUInt + of "u1": tyUInt8 + of "u2": tyUInt16 + of "u3": tyUInt32 + of "u4": tyUInt64 + of "o2": tyOwned + of "s3": tySink + of "L0": tyLent + of "v1": tyVarargs + of "U1": tyUncheckedArray + of "e2": tyError + of "b1": tyBuiltInTypeClass + of "U2": tyUserTypeClass + of "U3": tyUserTypeClassInst + of "c2": tyCompositeTypeClass + of "I0": tyInferred + of "a2": tyAnd + of "o4": tyOr + of "n2": tyNot + of "a3": tyAnything + of "s4": tyStatic + of "F1": tyFromExpr + of "c3": tyConcept + of "v2": tyVoid + of "I1": tyIterable else: tyNone diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index d338194ea5..372b096782 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -471,7 +471,7 @@ proc copyTypeProps*(g: ModuleGraph; module: int; dest, src: PType) = proc loadCompilerProc*(g: ModuleGraph; name: string): PSym = result = nil - if g.config.symbolFiles == disabledSf: + if g.config.symbolFiles == disabledSf and optWithinConfigSystem notin g.config.globalOptions: # For NIF-based compilation, search in loaded NIF modules when not defined(nimKochBootstrap): # Only try to resolve from NIF if we're actually using NIF files (cmdNifC) @@ -599,9 +599,10 @@ proc registerModule*(g: ModuleGraph; m: PSym) = if m.position >= g.packed.len: setLen(g.packed.pm, m.position + 1) - g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], - uniqueName: rope(uniqueModuleName(g.config, m))) - initStrTables(g, m) + if g.ifaces[m.position].module == nil: + g.ifaces[m.position] = Iface(module: m, converters: @[], patterns: @[], + uniqueName: rope(uniqueModuleName(g.config, m))) + initStrTables(g, m) proc registerModuleById*(g: ModuleGraph; m: FileIndex) = registerModule(g, g.packed[int m].module) @@ -814,31 +815,33 @@ proc moduleFromRodFile*(g: ModuleGraph; fileIdx: FileIndex; when not defined(nimKochBootstrap): proc moduleFromNifFile*(g: ModuleGraph; fileIdx: FileIndex; - cachedModules: var seq[FileIndex]; - loadFullAst: bool = false): PSym = + flags: set[LoadFlag] = {}): PrecompiledModule = ## Returns 'nil' if the module needs to be recompiled. ## Loads module from NIF file when optCompress is enabled. ## When loadFullAst is true, loads the complete module AST for code generation. if not fileExists(toNifFilename(g.config, fileIdx)): - return nil + return PrecompiledModule(module: nil) # Create module symbol let filename = AbsoluteFile toFullPath(g.config, fileIdx) - result = PSym( + + let m = PSym( kindImpl: skModule, itemId: ItemId(module: int32(fileIdx), item: 0'i32), name: getIdent(g.cache, splitFile(filename).name), infoImpl: newLineInfo(fileIdx, 1, 1), positionImpl: int(fileIdx)) - setOwner(result, getPackage(g.config, g.cache, fileIdx)) - + setOwner(m, getPackage(g.config, g.cache, fileIdx)) # Register module in graph - registerModule(g, result) - var opsLog: seq[LogEntry] = @[] - result.astImpl = loadNifModule(ast.program, fileIdx, g.ifaces[fileIdx.int].interf, - g.ifaces[fileIdx.int].interfHidden, opsLog, loadFullAst) + registerModule(g, m) + + result = loadNifModule(ast.program, fileIdx, + g.ifaces[fileIdx.int].interf, + g.ifaces[fileIdx.int].interfHidden, flags) + result.module = m + # Register hooks from NIF index with the module graph - for x in opsLog: + for x in result.logOps: case x.kind of HookEntry: g.loadedOps[x.op][x.key] = x.sym @@ -852,7 +855,6 @@ when not defined(nimKochBootstrap): raiseAssert "GenericInstEntry should not be in the NIF index" # Register methods per type from NIF index discard "todo" - cachedModules.add fileIdx proc configComplete*(g: ModuleGraph) = rememberStartupConfig(g.startupPackedConfig, g.config) diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index b061591bf8..39da0d762e 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -25,30 +25,36 @@ when defined(nimPreviewSlimSystem): import ast, options, lineinfos, modulegraphs, cgendata, cgen, pathutils, extccomp, msgs, modulepaths, idents, types, ast2nif -proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[PSym] = +proc loadModuleDependencies(g: ModuleGraph; mainFileIdx: FileIndex): seq[PrecompiledModule] = ## Traverse the module dependency graph using a stack. ## Returns all modules that need code generation, in dependency order. - var visited = initIntSet() - var stack: seq[FileIndex] = @[mainFileIdx] + let mainModule = moduleFromNifFile(g, mainFileIdx, {LoadFullAst}) + + var stack: seq[ModuleSuffix] = @[] result = @[] - var cachedModules: seq[FileIndex] = @[] + + if mainModule.module != nil: + incl mainModule.module.flagsImpl, sfMainModule + for dep in mainModule.deps: + stack.add dep + + var visited = initHashSet[string]() while stack.len > 0: - let fileIdx = stack.pop() + let suffix = stack.pop() - if not visited.containsOrIncl(int(fileIdx)): - # Only load full AST for main module; others are loaded lazily by codegen - let isMainModule = fileIdx == mainFileIdx - let module = moduleFromNifFile(g, fileIdx, cachedModules, loadFullAst=isMainModule) - if module != nil: - result.add module - if isMainModule: - incl module.flagsImpl, sfMainModule - # Add dependencies to stack (they come from cachedModules) - for dep in cachedModules: - if not visited.contains(int(dep)): + if not visited.containsOrIncl(suffix.string): + let nifFile = toGeneratedFile(g.config, AbsoluteFile(suffix.string), ".nif") + let fileIdx = msgs.fileInfoIdx(g.config, nifFile) + let precomp = moduleFromNifFile(g, fileIdx, {LoadFullAst}) + if precomp.module != nil: + result.add precomp + for dep in precomp.deps: + if not visited.contains(dep.string): stack.add dep - cachedModules.setLen(0) + + if mainModule.module != nil: + result.add mainModule proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule = ## Set up a BModule for code generation from a NIF module. @@ -56,38 +62,44 @@ proc setupNifBackendModule(g: ModuleGraph; module: PSym): BModule = g.backend = cgendata.newModuleList(g) result = cgen.newModule(BModuleList(g.backend), module, g.config, idGeneratorFromModule(module)) -proc generateCodeForModule(g: ModuleGraph; module: PSym) = - ## Generate C code for a single module. - let moduleId = module.position - var bmod = BModuleList(g.backend).modules[moduleId] - if bmod == nil: - bmod = setupNifBackendModule(g, module) - - # Generate code for the module's top-level statements - if module.ast != nil: - cgen.genTopLevelStmt(bmod, module.ast) - +proc finishModule(g: ModuleGraph; bmod: BModule) = # Finalize the module (this adds it to modulesClosed) # Create an empty stmt list as the init body - genInitCode in writeModule will set it up properly - let initStmt = newNodeI(nkStmtList, module.info) + let initStmt = newNode(nkStmtList) finalCodegenActions(g, bmod, initStmt) # Generate dispatcher methods for disp in getDispatchers(g): genProcLvl3(bmod, disp) +proc generateCodeForModule(g: ModuleGraph; precomp: PrecompiledModule) = + ## Generate C code for a single module. + let moduleId = precomp.module.position + var bmod = BModuleList(g.backend).mods[moduleId] + if bmod == nil: + bmod = setupNifBackendModule(g, precomp.module) + + # Generate code for the module's top-level statements + if precomp.topLevel != nil: + cgen.genTopLevelStmt(bmod, precomp.topLevel) + proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = ## Main entry point for NIF-based C code generation. ## Traverses the module dependency graph and generates C code. # Reset backend state resetForBackend(g) - let mainModule = g.getModule(mainFileIdx) + + var isKnownFile = false + let systemFileIdx = registerNifSuffix(g.config, "sysma2dyk", isKnownFile) + g.config.m.systemFileIdx = systemFileIdx + #msgs.fileInfoIdx(g.config, + # g.config.libpath / RelativeFile"system.nim") # Load system module first - it's always needed and contains essential hooks - var cachedModules: seq[FileIndex] = @[] - if g.config.m.systemFileIdx != InvalidFileIdx: - g.systemModule = moduleFromNifFile(g, g.config.m.systemFileIdx, cachedModules) + var precompSys = PrecompiledModule(module: nil) + precompSys = moduleFromNifFile(g, systemFileIdx, {LoadFullAst, AlwaysLoadInterface}) + g.systemModule = precompSys.module # Load all modules in dependency order using stack traversal # This must happen BEFORE any code generation so that hooks are loaded into loadedOps @@ -98,29 +110,35 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = return # Set up backend modules for all modules that need code generation - for module in modules: - discard setupNifBackendModule(g, module) + for m in modules: + discard setupNifBackendModule(g, m.module) # Also ensure system module is set up and generated first if it exists - if g.systemModule != nil and g.systemModule != mainModule: - let systemBmod = BModuleList(g.backend).modules[g.systemModule.position] - if systemBmod == nil: - discard setupNifBackendModule(g, g.systemModule) - generateCodeForModule(g, g.systemModule) + if precompSys.module != nil: + discard setupNifBackendModule(g, precompSys.module) + generateCodeForModule(g, precompSys) - # Generate code for all modules except main (main goes last) - # This ensures all modules are added to modulesClosed - for module in modules: - if module != mainModule and module != g.systemModule: - generateCodeForModule(g, module) + # Track which modules have been processed to avoid duplicates + var processed = initIntSet() + if precompSys.module != nil: + processed.incl precompSys.module.position - # Generate main module last (so all init procs are registered) - if mainModule != nil: - generateCodeForModule(g, mainModule) + # Generate code for all modules (skip system since it's already processed) + for m in modules: + if not processed.containsOrIncl(m.module.position): + generateCodeForModule(g, m) + + # during code generation of `main.nim` we can trigger the code generation + # of symbols in different modules so we need to finish these modules + # here later, after the above loop! + for m in BModuleList(g.backend).mods: + if m != nil: + assert m.module != nil + #if sfMainModule notin m.module.flags: + finishModule g, m # Write C files - if g.backend != nil: - cgenWriteModules(g.backend, g.config) + cgenWriteModules(g.backend, g.config) # Run C compiler if g.config.cmd != cmdTcc: diff --git a/compiler/options.nim b/compiler/options.nim index 479148d07c..6dcec635b3 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -112,6 +112,7 @@ type # please make sure we have under 32 options optJsBigInt64 # use bigints for 64-bit integers in JS optItaniumMangle # mangling follows the Itanium spec optCompress # turn on AST compression by converting it to NIF + optWithinConfigSystem # we still compile within the configuration system TGlobalOptions* = set[TGlobalOption] diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 7834a013c2..989f9c2d9a 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -286,8 +286,8 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF sfMainModule notin flags and not graph.withinSystem and not graph.config.isDefined("nimscript"): - result = moduleFromNifFile(graph, fileIdx, cachedModules) - if result == nil: + let precomp = moduleFromNifFile(graph, fileIdx) + if precomp.module == nil: let nifPath = toNifFilename(graph.config, fileIdx) localError(graph.config, unknownLineInfo, "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) & @@ -385,7 +385,8 @@ proc compilePipelineProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx graph.config.libpath / RelativeFile"system.nim") var cachedModules: seq[FileIndex] = @[] when not defined(nimKochBootstrap): - graph.systemModule = moduleFromNifFile(graph, graph.config.m.systemFileIdx, cachedModules) + let precomp = moduleFromNifFile(graph, graph.config.m.systemFileIdx) + graph.systemModule = precomp.module if graph.systemModule == nil: let nifPath = toNifFilename(graph.config, graph.config.m.systemFileIdx) localError(graph.config, unknownLineInfo, diff --git a/compiler/renderer.nim b/compiler/renderer.nim index a2e7626b42..e8cdfad6d2 100644 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1836,6 +1836,9 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext, fromStmtList = false) = putWithSpace(g, tkSymbol, "error") #gcomma(g, n, c) gsub(g, n[0], c) + of nkReplayAction: + put(g, tkSymbol, "replayaction") + #gsons(g, n, c, 0) else: #nkNone, nkExplicitTypeListCall: internalError(g.config, n.info, "renderer.gsub(" & $n.kind & ')') diff --git a/compiler/scriptconfig.nim b/compiler/scriptconfig.nim index e2df695268..10d3f73bc0 100644 --- a/compiler/scriptconfig.nim +++ b/compiler/scriptconfig.nim @@ -213,6 +213,7 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; unregisterArcOrc(conf) conf.globalOptions.excl optOwnedRefs conf.selectedGC = gcUnselected + conf.globalOptions.incl optWithinConfigSystem var m = graph.makeModule(scriptName) incl(m, sfMainModule) @@ -251,4 +252,5 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile; #initDefines() undefSymbol(conf.symbols, "nimscript") undefSymbol(conf.symbols, "nimconfig") + conf.globalOptions.excl optWithinConfigSystem conf.symbolFiles = oldSymbolFiles diff --git a/tools/enumgen.nim b/tools/enumgen.nim index 655cd030c2..d1a6473475 100644 --- a/tools/enumgen.nim +++ b/tools/enumgen.nim @@ -28,10 +28,6 @@ const ("nkError", "err"), ("nkType", "onlytype"), ("nkTypeSection", "type"), - ("tySequence", "seq"), - ("tyVar", "mut"), - ("tyProc", "proctype"), - ("tyUncheckedArray", "uarray"), ("nkExprEqExpr", "vv"), ("nkExprColonExpr", "kv"), ("nkDerefExpr", "deref"), @@ -55,17 +51,75 @@ const ("mVar", "varm"), ("mInSet", "contains"), ("mNil", "nilm"), - ("tyBuiltInTypeClass", "bconcept"), - ("tyUserTypeClass", "uconcept"), - ("tyUserTypeClassInst", "uconceptinst"), - ("tyCompositeTypeClass", "cconcept"), - ("tyGenericInvocation", "ginvoke"), - ("tyGenericBody", "gbody"), - ("tyGenericInst", "ginst"), - ("tyGenericParam", "gparam"), ("nkStmtList", "stmts"), ("nkDotExpr", "dot"), - ("nkBracketExpr", "at") + ("nkBracketExpr", "at"), + + ("tyNone", "n0"), # we always use a digit for type kinds so there can be no overlap with node kinds + ("tyBool", "b0"), + ("tyChar", "c0"), + ("tyEmpty", "e0"), + ("tyAlias", "a0"), + ("tyNil", "n1"), + ("tyUntyped", "U0"), + ("tyTyped", "t0"), + ("tyTypeDesc", "t1"), + ("tyGenericInvocation", "g0"), + ("tyGenericBody", "g1"), + ("tyGenericInst", "g2"), + ("tyGenericParam", "g4"), + ("tyDistinct", "d0"), + ("tyEnum", "e1"), + ("tyOrdinal", "o0"), + ("tyArray", "a1"), + ("tyObject", "o1"), + ("tyTuple", "t2"), + ("tySet", "s0"), + ("tyRange", "r0"), + ("tyPtr", "p0"), + ("tyRef", "r1"), + ("tyVar", "v0"), + ("tySequence", "s1"), + ("tyProc", "p1"), + ("tyPointer", "p2"), + ("tyOpenArray", "o3"), + ("tyString", "s2"), + ("tyCstring", "c1"), + ("tyForward", "F0"), + ("tyInt", "i0"), + ("tyInt8", "i1"), + ("tyInt16", "i2"), + ("tyInt32", "i3"), + ("tyInt64", "i4"), + ("tyFloat", "f0"), + ("tyFloat32", "f1"), + ("tyFloat64", "f2"), + ("tyFloat128", "f3"), + ("tyUInt", "u0"), + ("tyUInt8", "u1"), + ("tyUInt16", "u2"), + ("tyUInt32", "u3"), + ("tyUInt64", "u4"), + ("tyOwned", "o2"), + ("tySink", "s3"), + ("tyLent", "L0"), + ("tyVarargs", "v1"), + ("tyUncheckedArray", "U1"), + ("tyError", "e2"), + ("tyBuiltInTypeClass", "b1"), + ("tyUserTypeClass", "U2"), + ("tyUserTypeClassInst", "U3"), + ("tyCompositeTypeClass", "c2"), + ("tyInferred", "I0"), + ("tyAnd", "a2"), + ("tyOr", "o4"), + ("tyNot", "n2"), + ("tyAnything", "a3"), + ("tyStatic", "s4"), + ("tyFromExpr", "F1"), + ("tyConcept", "c3"), + ("tyVoid", "v2"), + ("tyIterable", "I1") ] SuffixesToReplace = [ ("Section", ""), ("Branch", ""), ("Stmt", ""), ("I", ""), From 22d4644d36209b13456d0b31034056ecf67d24e2 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 29 Dec 2025 10:23:46 +0100 Subject: [PATCH 13/31] refactoring (#25394) --- compiler/ccgtypes.nim | 99 ++++++++++++++++++++++-------------------- compiler/sighashes.nim | 4 ++ 2 files changed, 55 insertions(+), 48 deletions(-) diff --git a/compiler/ccgtypes.nim b/compiler/ccgtypes.nim index b8de2a6de5..6a74f4a298 100644 --- a/compiler/ccgtypes.nim +++ b/compiler/ccgtypes.nim @@ -844,6 +844,54 @@ proc getOpenArrayDesc(m: BModule; t: PType, check: var IntSet; kind: TypeDescKin m.s[cfsTypes].addField(name = "Field0", typ = ptrType(elemType)) m.s[cfsTypes].addField(name = "Field1", typ = NimInt) +proc importedCppObject(m: BModule; t, tt: PType; check: var IntSet; kind: TypeDescKind; sig: SigHash; result: var Rope) = + let cppNameAsRope = getTypeName(m, t, sig) + let cppName = $cppNameAsRope + var i = 0 + var chunkStart = 0 + + template addResultType(ty: untyped) = + if ty == nil or ty.kind == tyVoid: + result.add(CVoid) + elif ty.kind == tyStatic: + internalAssert m.config, ty.n != nil + result.add ty.n.renderTree + else: + result.add getTypeDescAux(m, ty, check, kind) + + while i < cppName.len: + if cppName[i] == '\'': + var chunkEnd = i-1 + var idx, stars: int = 0 + if scanCppGenericSlot(cppName, i, idx, stars): + result.add cppName.substr(chunkStart, chunkEnd) + chunkStart = i + + let typeInSlot = resolveStarsInCppType(tt, idx + 1, stars) + addResultType(typeInSlot) + else: + inc i + + if chunkStart != 0: + result.add cppName.substr(chunkStart) + else: + result = cppNameAsRope & "<" + for needsComma, a in tt.genericInstParams: + if needsComma: result.add(" COMMA ") + addResultType(a) + result.add("> ") + # always call for sideeffects: + assert t.kind != tyTuple + discard getRecordDesc(m, t, result, check) + # The resulting type will include commas and these won't play well + # with the C macros for defining procs such as N_NIMCALL. We must + # create a typedef for the type and use it in the proc signature: + let typedefName = "TY" & $sig + m.s[cfsTypes].addTypedef(name = typedefName): + m.s[cfsTypes].add(result) + m.typeCache[sig] = typedefName + result = typedefName + proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDescKind): Rope = # returns only the type's name var t = origTyp.skipTypes(irrelevantForBackend-{tyOwned}) @@ -859,7 +907,7 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes # tyDistinct matters if it is an importc type result = getTypePre(m, origTyp.skipTypes(irrelevantForBackend-{tyOwned, tyDistinct}), sig) - defer: # defer is the simplest in this case + defer: if isImportedType(t) and not m.typeABICache.containsOrIncl(sig): addAbiCheck(m, t, result) @@ -993,7 +1041,7 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes m.s[cfsTypes].addArrayTypedef(name = result, len = 1): m.s[cfsTypes].add(et) of tyArray: - var n: BiggestInt = toInt64(lengthOrd(m.config, t)) + var n = toInt64(lengthOrd(m.config, t)) if n <= 0: n = 1 # make an array of at least one element result = getTypeName(m, origTyp, sig) m.typeCache[sig] = result @@ -1004,52 +1052,7 @@ proc getTypeDescAux(m: BModule; origTyp: PType, check: var IntSet; kind: TypeDes of tyObject, tyTuple: let tt = origTyp.skipTypes({tyDistinct}) if isImportedCppType(t) and tt.kind == tyGenericInst: - let cppNameAsRope = getTypeName(m, t, sig) - let cppName = $cppNameAsRope - var i = 0 - var chunkStart = 0 - - template addResultType(ty: untyped) = - if ty == nil or ty.kind == tyVoid: - result.add(CVoid) - elif ty.kind == tyStatic: - internalAssert m.config, ty.n != nil - result.add ty.n.renderTree - else: - result.add getTypeDescAux(m, ty, check, kind) - - while i < cppName.len: - if cppName[i] == '\'': - var chunkEnd = i-1 - var idx, stars: int = 0 - if scanCppGenericSlot(cppName, i, idx, stars): - result.add cppName.substr(chunkStart, chunkEnd) - chunkStart = i - - let typeInSlot = resolveStarsInCppType(tt, idx + 1, stars) - addResultType(typeInSlot) - else: - inc i - - if chunkStart != 0: - result.add cppName.substr(chunkStart) - else: - result = cppNameAsRope & "<" - for needsComma, a in tt.genericInstParams: - if needsComma: result.add(" COMMA ") - addResultType(a) - result.add("> ") - # always call for sideeffects: - assert t.kind != tyTuple - discard getRecordDesc(m, t, result, check) - # The resulting type will include commas and these won't play well - # with the C macros for defining procs such as N_NIMCALL. We must - # create a typedef for the type and use it in the proc signature: - let typedefName = "TY" & $sig - m.s[cfsTypes].addTypedef(name = typedefName): - m.s[cfsTypes].add(result) - m.typeCache[sig] = typedefName - result = typedefName + importedCppObject(m, t, tt, check, kind, sig, result) else: result = cacheGetType(m.forwTypeCache, sig) if result == "": diff --git a/compiler/sighashes.nim b/compiler/sighashes.nim index f7d89037e3..5d6d0e9a5b 100644 --- a/compiler/sighashes.nim +++ b/compiler/sighashes.nim @@ -106,6 +106,10 @@ proc hashType(c: var MD5Context, t: PType; flags: set[ConsiderFlag]; conf: Confi c &= "\254" return + # Ensure type is fully loaded before hashing to avoid hash changing + # as properties are accessed and trigger lazy loading. + backendEnsureMutable(t) + case t.kind of tyGenericInvocation: for a in t.kids: From f1b97caf92dab122063a598b680a464b438e74bc Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 29 Dec 2025 17:25:56 +0800 Subject: [PATCH 14/31] fixes #19983; implements bitmasked bitshifting for all backends (#25390) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit replaces https://github.com/nim-lang/Nim/pull/11555 fixes https://github.com/nim-lang/Nim/issues/19983 fixes https://github.com/nim-lang/Nim/issues/13566 - [x] JS backend --------- Co-authored-by: Arne Döring --- changelog.md | 2 ++ compiler/ccgexprs.nim | 6 ++-- compiler/jsgen.nim | 29 +++++++++-------- compiler/semfold.nim | 36 +++++++++++---------- compiler/vmgen.nim | 45 ++++++++++++++++++++++---- lib/system/arithmetics.nim | 15 ++++++--- tests/int/tarithm.nim | 65 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 154 insertions(+), 44 deletions(-) diff --git a/changelog.md b/changelog.md index 217c8c9653..08aafba6b8 100644 --- a/changelog.md +++ b/changelog.md @@ -31,6 +31,8 @@ errors. - The second parameter of `succ`, `pred`, `inc`, and `dec` in `system` now accepts `SomeInteger` (previously `Ordinal`). +- Bitshift operators (`shl`, `shr`, `ashr`) now apply bitmasking to the right operand in the C/C++/VM/JS backends. + ## Standard library additions and changes [//]: # "Additions:" diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 5e37709af9..2ef134f497 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -749,16 +749,16 @@ proc binaryArith(p: BProc, e: PNode, d: var TLoc, op: TMagic) = let t = getType() let at = cUintType(k) let bt = cUintType(s) - res = cCast(t, cOp(Shr, at, cCast(at, ra), cCast(bt, rb))) + res = cCast(t, cOp(Shr, at, cCast(at, ra), cOp(BitAnd, at, cCast(bt, rb), cIntLiteral(k - 1)))) of mShlI: let t = getType() let at = cUintType(s) - res = cCast(t, cOp(Shl, at, cCast(at, ra), cCast(at, rb))) + res = cCast(t, cOp(Shl, at, cCast(at, ra), cOp(BitAnd, at, cCast(at, rb), cIntLiteral(k - 1)))) of mAshrI: let t = getType() let at = cIntType(s) let bt = cUintType(s) - res = cCast(t, cOp(Shr, at, cCast(at, ra), cCast(bt, rb))) + res = cCast(t, cOp(Shr, at, cCast(at, ra), cOp(BitAnd, at, cCast(bt, rb), cIntLiteral(k - 1)))) of mBitandI: let t = getType() res = cCast(t, cOp(BitAnd, t, ra, rb)) diff --git a/compiler/jsgen.nim b/compiler/jsgen.nim index c52b26e69b..99582b0fd6 100644 --- a/compiler/jsgen.nim +++ b/compiler/jsgen.nim @@ -729,44 +729,47 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) = of mShrI: let typ = n[1].typ.skipTypes(abstractVarRange) if typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions: - applyFormat("BigInt.asIntN(64, BigInt.asUintN(64, $1) >> BigInt($2))") + applyFormat("BigInt.asIntN(64, BigInt.asUintN(64, $1) >> (BigInt($2) & 63n))") elif typ.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions: - applyFormat("($1 >> BigInt($2))") + applyFormat("($1 >> (BigInt($2) & 63n))") else: + let bitmask = typ.size * 8 - 1 if typ.kind in {tyInt..tyInt32}: let trimmerU = unsignedTrimmer(typ.size) let trimmerS = signedTrimmer(typ.size) - r.res = "((($1 $2) >>> $3) $4)" % [xLoc, trimmerU, yLoc, trimmerS] + r.res = "((($1 $2) >>> ($3 & $5)) $4)" % [xLoc, trimmerU, yLoc, trimmerS, $bitmask] else: - applyFormat("($1 >>> $2)") + r.res = "($1 >>> ($2 & $3))" % [xLoc, yLoc, $bitmask] of mShlI: let typ = n[1].typ.skipTypes(abstractVarRange) if typ.size == 8: if typ.kind == tyInt64 and optJsBigInt64 in p.config.globalOptions: - applyFormat("BigInt.asIntN(64, $1 << BigInt($2))") + applyFormat("BigInt.asIntN(64, $1 << (BigInt($2) & 63n))") elif typ.kind == tyUInt64 and optJsBigInt64 in p.config.globalOptions: - applyFormat("BigInt.asUintN(64, $1 << BigInt($2))") + applyFormat("BigInt.asUintN(64, $1 << (BigInt($2) & 63n))") else: - applyFormat("($1 * Math.pow(2, $2))") + applyFormat("($1 * Math.pow(2, ($2 & 63)))") else: + let bitmask = typ.size * 8 - 1 if typ.kind in {tyUInt..tyUInt32}: let trimmer = unsignedTrimmer(typ.size) - r.res = "(($1 << $2) $3)" % [xLoc, yLoc, trimmer] + r.res = "(($1 << ($2 & $4)) $3)" % [xLoc, yLoc, trimmer, $bitmask] else: let trimmer = signedTrimmer(typ.size) - r.res = "(($1 << $2) $3)" % [xLoc, yLoc, trimmer] + r.res = "(($1 << ($2 & $4)) $3)" % [xLoc, yLoc, trimmer, $bitmask] of mAshrI: let typ = n[1].typ.skipTypes(abstractVarRange) if typ.size == 8: if optJsBigInt64 in p.config.globalOptions: - applyFormat("($1 >> BigInt($2))") + applyFormat("($1 >> (BigInt($2) & 63n))") else: - applyFormat("Math.floor($1 / Math.pow(2, $2))") + applyFormat("Math.floor($1 / Math.pow(2, ($2 & 63)))") else: + let bitmask = typ.size * 8 - 1 if typ.kind in {tyUInt..tyUInt32}: - applyFormat("($1 >>> $2)") + r.res = "($1 >>> ($2 & $3)))" % [xLoc, yLoc, $bitmask] else: - applyFormat("($1 >> $2)") + r.res = "($1 >> ($2 & $3))" % [xLoc, yLoc, $bitmask] of mBitandI: bitwiseExpr("&") of mBitorI: bitwiseExpr("|") of mBitxorI: bitwiseExpr("^") diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 020d1e46a7..501e66969a 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -179,29 +179,30 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P let argB = getInt(b) result = newIntNodeT(if argA > argB: argA else: argB, n, idgen, g) of mShlI: + let valueB = toInt64(getInt(b)) and (n.typ.size * 8 - 1) case skipTypes(n.typ, abstractRange).kind - of tyInt8: result = newIntNodeT(toInt128(toInt8(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) - of tyInt16: result = newIntNodeT(toInt128(toInt16(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) - of tyInt32: result = newIntNodeT(toInt128(toInt32(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) - of tyInt64: result = newIntNodeT(toInt128(toInt64(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) + of tyInt8: result = newIntNodeT(toInt128(toInt8(getInt(a)) shl valueB), n, idgen, g) + of tyInt16: result = newIntNodeT(toInt128(toInt16(getInt(a)) shl valueB), n, idgen, g) + of tyInt32: result = newIntNodeT(toInt128(toInt32(getInt(a)) shl valueB), n, idgen, g) + of tyInt64: result = newIntNodeT(toInt128(toInt64(getInt(a)) shl valueB), n, idgen, g) of tyInt: if g.config.target.intSize == 4: - result = newIntNodeT(toInt128(toInt32(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) + result = newIntNodeT(toInt128(toInt32(getInt(a)) shl valueB), n, idgen, g) else: - result = newIntNodeT(toInt128(toInt64(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) - of tyUInt8: result = newIntNodeT(toInt128(toUInt8(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) - of tyUInt16: result = newIntNodeT(toInt128(toUInt16(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) - of tyUInt32: result = newIntNodeT(toInt128(toUInt32(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) - of tyUInt64: result = newIntNodeT(toInt128(toUInt64(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) + result = newIntNodeT(toInt128(toInt64(getInt(a)) shl valueB), n, idgen, g) + of tyUInt8: result = newIntNodeT(toInt128(toUInt8(getInt(a)) shl valueB), n, idgen, g) + of tyUInt16: result = newIntNodeT(toInt128(toUInt16(getInt(a)) shl valueB), n, idgen, g) + of tyUInt32: result = newIntNodeT(toInt128(toUInt32(getInt(a)) shl valueB), n, idgen, g) + of tyUInt64: result = newIntNodeT(toInt128(toUInt64(getInt(a)) shl valueB), n, idgen, g) of tyUInt: if g.config.target.intSize == 4: - result = newIntNodeT(toInt128(toUInt32(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) + result = newIntNodeT(toInt128(toUInt32(getInt(a)) shl valueB), n, idgen, g) else: - result = newIntNodeT(toInt128(toUInt64(getInt(a)) shl toInt64(getInt(b))), n, idgen, g) + result = newIntNodeT(toInt128(toUInt64(getInt(a)) shl valueB), n, idgen, g) else: internalError(g.config, n.info, "constant folding for shl") of mShrI: var a = cast[uint64](getInt(a)) - let b = cast[uint64](getInt(b)) + let b = cast[uint64](getInt(b)) and cast[uint64](n.typ.size * 8 - 1) # To support the ``-d:nimOldShiftRight`` flag, we need to mask the # signed integers to cut off the extended sign bit in the internal # representation. @@ -220,12 +221,13 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P let c = cast[BiggestInt](a shr b) result = newIntNodeT(toInt128(c), n, idgen, g) of mAshrI: + let valueB = toInt64(getInt(b)) and (n.typ.size * 8 - 1) case skipTypes(n.typ, abstractRange).kind - of tyInt8: result = newIntNodeT(toInt128(ashr(toInt8(getInt(a)), toInt8(getInt(b)))), n, idgen, g) - of tyInt16: result = newIntNodeT(toInt128(ashr(toInt16(getInt(a)), toInt16(getInt(b)))), n, idgen, g) - of tyInt32: result = newIntNodeT(toInt128(ashr(toInt32(getInt(a)), toInt32(getInt(b)))), n, idgen, g) + of tyInt8: result = newIntNodeT(toInt128(ashr(toInt8(getInt(a)), valueB)), n, idgen, g) + of tyInt16: result = newIntNodeT(toInt128(ashr(toInt16(getInt(a)), valueB)), n, idgen, g) + of tyInt32: result = newIntNodeT(toInt128(ashr(toInt32(getInt(a)), valueB)), n, idgen, g) of tyInt64, tyInt: - result = newIntNodeT(toInt128(ashr(toInt64(getInt(a)), toInt64(getInt(b)))), n, idgen, g) + result = newIntNodeT(toInt128(ashr(toInt64(getInt(a)), valueB)), n, idgen, g) else: internalError(g.config, n.info, "constant folding for ashr") of mDivI: let argA = getInt(a) diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim index 11b7b27fe7..ddcc834c7e 100644 --- a/compiler/vmgen.nim +++ b/compiler/vmgen.nim @@ -1074,6 +1074,19 @@ proc whichAsgnOpc(n: PNode; requiresCopy = true): TOpcode = else: (if requiresCopy: opcAsgnComplex else: opcFastAsgnComplex) +proc sizeLog2(typeSize: BiggestInt): TRegister = + case typeSize: + of 8: + result = 3 + of 16: + result = 4 + of 32: + result = 5 + of 64: + result = 6 + else: + raiseAssert $(typeSize) + proc genMagic(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}, m: TMagic) = case m of mAnd: c.genAndOr(n, opcFJmp, dest) @@ -1159,24 +1172,42 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}, m: TMag of mDivF64: genBinaryABC(c, n, dest, opcDivFloat) of mShrI: # modified: genBinaryABC(c, n, dest, opcShrInt) - # narrowU is applied to the left operandthe idea here is to narrow the left operand + # narrowU is applied to the left operand the idea here is to narrow the left operand + let typ = skipTypes(n.typ, abstractVar-{tyTypeDesc}) + let size = getSize(c.config, typ) let tmp = c.genx(n[1]) c.genNarrowU(n, tmp) let tmp2 = c.genx(n[2]) if dest < 0: dest = c.getTemp(n.typ) + c.gABC(n, opcNarrowU, tmp2, sizeLog2(size * 8)) c.gABC(n, opcShrInt, dest, tmp, tmp2) c.freeTemp(tmp) c.freeTemp(tmp2) of mShlI: - genBinaryABC(c, n, dest, opcShlInt) + let typ = skipTypes(n.typ, abstractVar-{tyTypeDesc}) + let size = getSize(c.config, typ) + let tmp1 = c.genx(n[1]) + let tmp2 = c.genx(n[2]) + if dest < 0: dest = c.getTemp(n.typ) + c.gABC(n, opcNarrowU, tmp2, sizeLog2(size * 8)) + c.gABC(n, opcShlInt, dest, tmp1, tmp2) + c.freeTemp(tmp1) + c.freeTemp(tmp2) # genNarrowU modified - let t = skipTypes(n.typ, abstractVar-{tyTypeDesc}) - let size = getSize(c.config, t) - if t.kind in {tyUInt8..tyUInt32} or (t.kind == tyUInt and size < 8): + if typ.kind in {tyUInt8..tyUInt32} or (typ.kind == tyUInt and size < 8): c.gABC(n, opcNarrowU, dest, TRegister(size*8)) - elif t.kind in {tyInt8..tyInt32} or (t.kind == tyInt and size < 8): + elif typ.kind in {tyInt8..tyInt32} or (typ.kind == tyInt and size < 8): c.gABC(n, opcSignExtend, dest, TRegister(size*8)) - of mAshrI: genBinaryABC(c, n, dest, opcAshrInt) + of mAshrI: + let typ = skipTypes(n.typ, abstractVar-{tyTypeDesc}) + let size = getSize(c.config, typ) + let tmp1 = c.genx(n[1]) + let tmp2 = c.genx(n[2]) + if dest < 0: dest = c.getTemp(n.typ) + c.gABC(n, opcNarrowU, tmp2, sizeLog2(size * 8)) + c.gABC(n, opcAshrInt, dest, tmp1, tmp2) + c.freeTemp(tmp1) + c.freeTemp(tmp2) of mBitandI: genBinaryABC(c, n, dest, opcBitandInt) of mBitorI: genBinaryABC(c, n, dest, opcBitorInt) of mBitxorI: genBinaryABC(c, n, dest, opcBitxorInt) diff --git a/lib/system/arithmetics.nim b/lib/system/arithmetics.nim index 71e6b69d4c..5711004822 100644 --- a/lib/system/arithmetics.nim +++ b/lib/system/arithmetics.nim @@ -136,7 +136,10 @@ when defined(nimOldShiftRight): else: proc `shr`*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} = ## Computes the `shift right` operation of `x` and `y`, filling - ## vacant bit positions with the sign bit. + ## vacant bit positions with the sign bit. `y` (the number of + ## positions to shift) is reduced to modulo `sizeof(x) * 8`. + ## That is `15'i32 shr 35` is equivalent to `15'i32 shr 3` + ## bitmasked to always be in the range `0 ..< sizeof(int)`. ## ## **Note**: `Operator precedence `_ ## is different than in *C*. @@ -158,7 +161,9 @@ else: proc `shl`*(x: int, y: SomeInteger): int {.magic: "ShlI", noSideEffect.} = - ## Computes the `shift left` operation of `x` and `y`. + ## Computes the `shift left` operation of `x` and `y`. `y` (the number of + ## positions to shift) is reduced to modulo `sizeof(x) * 8`. + ## That is `15'i32 shl 35` is equivalent to `15'i32 shl 3`. ## ## **Note**: `Operator precedence `_ ## is different than in *C*. @@ -172,7 +177,9 @@ proc `shl`*(x: int64, y: SomeInteger): int64 {.magic: "ShlI", noSideEffect.} proc ashr*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} = ## Shifts right by pushing copies of the leftmost bit in from the left, - ## and let the rightmost bits fall off. + ## and let the rightmost bits fall off. `y` (the number of + ## positions to shift) is reduced to modulo `sizeof(x) * 8`. + ## That is `ashr(15'i32, 35)` is equivalent to `ashr(15'i32, 3)`. ## ## Note that `ashr` is not an operator so use the normal function ## call syntax for it. @@ -181,7 +188,7 @@ proc ashr*(x: int, y: SomeInteger): int {.magic: "AshrI", noSideEffect.} = ## * `shr func<#shr,int,SomeInteger>`_ runnableExamples: assert ashr(0b0001_0000'i8, 2) == 0b0000_0100'i8 - assert ashr(0b1000_0000'i8, 8) == 0b1111_1111'i8 + assert ashr(0b1000_0000'i8, 8) == 0b1000_0000'i8 assert ashr(0b1000_0000'i8, 1) == 0b1100_0000'i8 proc ashr*(x: int8, y: SomeInteger): int8 {.magic: "AshrI", noSideEffect.} proc ashr*(x: int16, y: SomeInteger): int16 {.magic: "AshrI", noSideEffect.} diff --git a/tests/int/tarithm.nim b/tests/int/tarithm.nim index d0943d225d..ff770e54f3 100644 --- a/tests/int/tarithm.nim +++ b/tests/int/tarithm.nim @@ -14,6 +14,7 @@ int32 0 tUnsignedOps OK ''' +targets: "c cpp js" nimout: "tUnsignedOps OK" """ @@ -185,3 +186,67 @@ block tUnsignedOps: testUnsignedOps() static: testUnsignedOps() + +block tshl: + # Signed types + block: + const t0: int8 = 1'i8 shl 8 + const t1: int16 = 1'i16 shl 16 + const t2: int32 = 1'i32 shl 32 + const t3: int64 = 1'i64 shl 64 + doAssert t0 == 1 + doAssert t1 == 1 + doAssert t2 == 1 + doAssert t3 == 1 + + # Unsigned types + block: + const t0: uint8 = 1'u8 shl 8 + const t1: uint16 = 1'u16 shl 16 + const t2: uint32 = 1'u32 shl 32 + const t3: uint64 = 1'u64 shl 64 + doAssert t0 == 1 + doAssert t1 == 1 + doAssert t2 == 1 + doAssert t3 == 1 + +block bitmaking: + + # test semfold (single expression) + doAssert (0x10'i8 shr 2) == (0x10'i8 shr 0b1010_1010) + doAssert (0x10'u8 shr 2) == (0x10'u8 shr 0b0101_1010) + doAssert (0x10'i16 shr 2) == (0x10'i16 shr 0b1011_0010) + doAssert (0x10'u16 shr 2) == (0x10'u16 shr 0b0101_0010) + doAssert (0x10'i32 shr 2) == (0x10'i32 shr 0b1010_0010) + doAssert (0x10'u32 shr 2) == (0x10'u32 shr 0b0110_0010) + doAssert (0x10'i64 shr 2) == (0x10'i32 shr 0b1100_0010) + doAssert (0x10'u64 shr 2) == (0x10'u32 shr 0b0100_0010) + + doAssert (0x10'i8 shl 2) == (0x10'i8 shl 0b1010_1010) + doAssert (0x10'u8 shl 2) == (0x10'u8 shl 0b0101_1010) + doAssert (0x10'i16 shl 2) == (0x10'i16 shl 0b1011_0010) + doAssert (0x10'u16 shl 2) == (0x10'u16 shl 0b0101_0010) + doAssert (0x10'i32 shl 2) == (0x10'i32 shl 0b1010_0010) + doAssert (0x10'u32 shl 2) == (0x10'u32 shl 0b0110_0010) + doAssert (0x10'i64 shl 2) == (0x10'i32 shl 0b1100_0010) + doAssert (0x10'u64 shl 2) == (0x10'u32 shl 0b0100_0010) + + proc testVmAndBackend[T: SomeInteger](a: T, b1, b2: int) {.sideeffect.} = + # this echo is to cause a side effect and therefore ensure this + # proc isn't evaluated at compile time when it should not. + doAssert((a shr b1) == (a shr b2)) + doAssert((a shl b1) == (a shl b2)) + + proc callTestVmAndBackend() = + testVmAndBackend(0x10'i8, 2, 0b1010_1010) + testVmAndBackend(0x10'u8, 2, 0b0101_1010) + testVmAndBackend(0x10'i16, 2, 0b1011_0010) + testVmAndBackend(0x10'u16, 2, 0b0101_0010) + testVmAndBackend(0x10'i32, 2, 0b1010_0010) + testVmAndBackend(0x10'u32, 2, 0b0110_0010) + testVmAndBackend(0x10'i64, 2, 0b1100_0010) + testVmAndBackend(0x10'u64, 2, 0b0100_0010) + + callTestVmAndBackend() # test at runtime + static: + callTestVmAndBackend() # test at compiletime From 234c73c58a3d6bdfbc9c9370cd620c2d4990ae09 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Mon, 29 Dec 2025 13:52:22 +0100 Subject: [PATCH 15/31] refactoring for IC (#25395) --- compiler/ast.nim | 7 +++++++ compiler/cgen.nim | 4 ++-- compiler/nifbackend.nim | 11 +++++++++-- compiler/semstmts.nim | 2 +- compiler/suggest.nim | 6 +++--- 5 files changed, 22 insertions(+), 8 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index bc28cff845..bafc02dba2 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1680,3 +1680,10 @@ type template initSymMapping*(): SymMapping = initIdTable[PSym]() template initTypeMapping*(): TypeMapping = initIdTable[PType]() + +proc sameModules*(a, b: PSym): bool {.inline.} = + assert a.kind == skModule and b.kind == skModule + result = a.position == b.position + +proc sameOwners*(a, b: PSym): bool = + result = a == b or (a.kind == skModule and b.kind == skModule and a.position == b.position) or a.id == b.id diff --git a/compiler/cgen.nim b/compiler/cgen.nim index b380b136d2..591979aea1 100644 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -1543,7 +1543,7 @@ proc genProcLvl2(m: BModule, prc: PSym) = # externally-to-the-current-module defined proc, also important # to do the declaredProtos check before the call to genProcPrototype if isReloadable(m, prc) and prc.id notin m.declaredProtos and - q != nil and q.module.id != m.module.id: + q != nil and not sameModules(q.module, m.module): m.s[cfsDynLibInit].add('\t') m.s[cfsDynLibInit].addAssignment(prc.loc.snippet, cCast(getProcTypeCast(m, prc), @@ -1601,7 +1601,7 @@ proc genVarPrototype(m: BModule, n: PNode) = if (lfNoDecl in sym.loc.flags) or contains(m.declaredThings, sym.id): return - if sym.owner.id != m.module.id: + if not sameOwners(sym.owner, m.module): # else we already have the symbol generated! assert(sym.loc.snippet != "") incl(m.declaredThings, sym.id) diff --git a/compiler/nifbackend.nim b/compiler/nifbackend.nim index 39da0d762e..fa293bbfe3 100644 --- a/compiler/nifbackend.nim +++ b/compiler/nifbackend.nim @@ -131,11 +131,18 @@ proc generateCode*(g: ModuleGraph; mainFileIdx: FileIndex) = # during code generation of `main.nim` we can trigger the code generation # of symbols in different modules so we need to finish these modules # here later, after the above loop! + # Important: The main module must be finished LAST so that all other modules + # have registered their init procs before genMainProc uses them. + var mainModule: BModule = nil for m in BModuleList(g.backend).mods: if m != nil: assert m.module != nil - #if sfMainModule notin m.module.flags: - finishModule g, m + if sfMainModule in m.module.flags: + mainModule = m + else: + finishModule g, m + if mainModule != nil: + finishModule g, mainModule # Write C files cgenWriteModules(g.backend, g.config) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c86af27c91..be9e409108 100644 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -2361,7 +2361,7 @@ proc semCppMember(c: PContext; s: PSym; n: PNode) = typ = typ.elementType if typ.kind != tyObject: localError(c.config, n.info, pragmaName & " must be either ptr to object or object type.") - if typ.owner.id == s.owner.id and c.module.id == s.owner.id: + if sameOwners(typ.owner, s.owner) and sameOwners(c.module, s.owner): c.graph.memberProcsPerType.mgetOrPut(typ.itemId, @[]).add s else: localError(c.config, n.info, diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 5c3265dba2..a1cd8b9237 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -356,12 +356,12 @@ proc filterSymNoOpr(s: PSym; prefix: PNode; res: var PrefixMatch): bool {.inline not isKeyword(s.name) proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} = - let fmoduleId = getModule(f).id - result = sfExported in f.flags or fmoduleId == c.module.id + let fmodule = getModule(f) + result = sfExported in f.flags or sameModules(fmodule, c.module) if not result: for module in c.friendModules: - if fmoduleId == module.id: return true + if sameModules(fmodule, module): return true if f.kind == skField: var symObj = f.owner.typ.toObjectFromRefPtrGeneric.sym assert symObj != nil From e97b0bb541ee182c4f38709e2dbbaee03be12138 Mon Sep 17 00:00:00 2001 From: bptato <60043228+bptato@users.noreply.github.com> Date: Tue, 30 Dec 2025 23:09:01 +0100 Subject: [PATCH 16/31] Do not directly cast int128 to uint64 in semfold (#25396) int128 is an array of uint32s, so while this works on little-endian CPUs, it's completely broken on big-endian. e.g. following snippet would fail: const x = 0xFFFFFFFF'u32 const y = (x shr 1) echo y # amd64: 2147483647, s390x: 0 That in turn broke float printing, resulting in miscompilation of any code that used floats. To fix this, we now call the aptly named castToUInt64 procedure which performs the same cast portably. (Thanks to barracuda156 for helping debug this.) --- compiler/semfold.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/semfold.nim b/compiler/semfold.nim index 501e66969a..1a3f40a47a 100644 --- a/compiler/semfold.nim +++ b/compiler/semfold.nim @@ -201,8 +201,8 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; idgen: IdGenerator; g: ModuleGraph): P result = newIntNodeT(toInt128(toUInt64(getInt(a)) shl valueB), n, idgen, g) else: internalError(g.config, n.info, "constant folding for shl") of mShrI: - var a = cast[uint64](getInt(a)) - let b = cast[uint64](getInt(b)) and cast[uint64](n.typ.size * 8 - 1) + var a = castToUInt64(getInt(a)) + let b = castToUInt64(getInt(b)) and cast[uint64](n.typ.size * 8 - 1) # To support the ``-d:nimOldShiftRight`` flag, we need to mask the # signed integers to cut off the extended sign bit in the internal # representation. From 61970be479c0fd4ef4c2319feb9128575c530c99 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Wed, 31 Dec 2025 13:33:57 +0100 Subject: [PATCH 17/31] reduce imports (#25398) --- compiler/ast.nim | 2 +- compiler/astdef.nim | 3 +-- compiler/closureiters.nim | 3 +-- compiler/concepts.nim | 4 ++-- compiler/deps.nim | 2 +- compiler/docgen.nim | 2 +- compiler/importer.nim | 2 +- compiler/layeredtable.nim | 1 - compiler/magicsys.nim | 2 +- compiler/modulegraphs.nim | 2 +- compiler/pipelines.nim | 3 ++- compiler/sempass2.nim | 2 +- compiler/treetab.nim | 2 +- 13 files changed, 14 insertions(+), 16 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index bafc02dba2..89f24c63ca 100644 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -10,7 +10,7 @@ # abstract syntax tree + symbol table import - lineinfos, options, ropes, idents, int128, wordrecg + lineinfos, options, idents, int128, wordrecg import std/[tables, hashes] from std/strutils import toLowerAscii diff --git a/compiler/astdef.nim b/compiler/astdef.nim index 30b2298fb2..b9a8aab3e1 100644 --- a/compiler/astdef.nim +++ b/compiler/astdef.nim @@ -8,10 +8,9 @@ # import - lineinfos, options, ropes, idents, int128, wordrecg + lineinfos, options, ropes, idents, int128 import std/[tables, hashes] -from std/strutils import toLowerAscii when defined(nimPreviewSlimSystem): import std/assertions diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim index 8fca38957d..ddf9c2704c 100644 --- a/compiler/closureiters.nim +++ b/compiler/closureiters.nim @@ -139,8 +139,7 @@ import ast, msgs, idents, - renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos, - options + renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos import std/tables diff --git a/compiler/concepts.nim b/compiler/concepts.nim index 040089a669..4f531c2cf9 100644 --- a/compiler/concepts.nim +++ b/compiler/concepts.nim @@ -11,9 +11,9 @@ ## for details. Note this is a first implementation and only the "Concept matching" ## section has been implemented. -import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, layeredtable +import ast, semdata, lookups, lineinfos, idents, msgs, renderer, types, layeredtable -import std/[intsets, sets] +import std/sets when defined(nimPreviewSlimSystem): import std/assertions diff --git a/compiler/deps.nim b/compiler/deps.nim index 255cd3e80f..aa5322ecb6 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -11,7 +11,7 @@ ## This enables incremental and parallel compilation using the `m` switch. import std / [os, tables, sets, times, osproc, strutils] -import options, msgs, pathutils, lineinfos +import options, msgs, lineinfos import "../dist/nimony/src/lib" / [nifstreams, nifcursors, bitabs, nifreader, nifbuilder] import "../dist/nimony/src/gear2" / modnames diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 5f5b42b32f..8167fc4b68 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -19,7 +19,7 @@ import wordrecg, syntaxes, renderer, lexer, packages/docutils/[rst, rstidx, rstgen, dochelpers], trees, types, - typesrenderer, astalgo, lineinfos, + typesrenderer, lineinfos, pathutils, nimpaths, renderverbatim, packages import packages/docutils/rstast except FileIndex, TLineInfo diff --git a/compiler/importer.nim b/compiler/importer.nim index 8ff3bcfdb3..2d50973756 100644 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -10,7 +10,7 @@ ## This module implements the symbol importing mechanism. import - ast, astalgo, msgs, options, idents, lookups, + ast, msgs, options, idents, lookups, semdata, modulepaths, sigmatch, lineinfos, modulegraphs, wordrecg from std/strutils import `%`, startsWith diff --git a/compiler/layeredtable.nim b/compiler/layeredtable.nim index 248ec4bcf2..81c6c63d75 100644 --- a/compiler/layeredtable.nim +++ b/compiler/layeredtable.nim @@ -1,4 +1,3 @@ -import std/[tables] import ast, astalgo type diff --git a/compiler/magicsys.nim b/compiler/magicsys.nim index c51ad690c7..a4e76f7acb 100644 --- a/compiler/magicsys.nim +++ b/compiler/magicsys.nim @@ -10,7 +10,7 @@ # Built-in types and compilerprocs are registered here. import - ast, astalgo, msgs, platform, idents, + ast, msgs, platform, idents, modulegraphs, lineinfos export createMagic diff --git a/compiler/modulegraphs.nim b/compiler/modulegraphs.nim index 372b096782..40415091fc 100644 --- a/compiler/modulegraphs.nim +++ b/compiler/modulegraphs.nim @@ -11,7 +11,7 @@ ## represents a complete Nim project. Single modules can either be kept in RAM ## or stored in a rod-file. -import std/[intsets, tables, hashes, strtabs, algorithm, os, strutils, parseutils] +import std/[intsets, tables, hashes, strtabs, os, strutils, parseutils] import ../dist/checksums/src/checksums/md5 import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages, suggestsymdb import ic / [packed_ast, ic] diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index 989f9c2d9a..fd1193bd3b 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -1,9 +1,10 @@ import sem, cgen, modulegraphs, ast, llstream, parser, msgs, lineinfos, reorder, options, semdata, cgendata, modules, pathutils, - packages, syntaxes, depends, vm, vmdef, pragmas, idents, lookups, wordrecg, + packages, syntaxes, depends, vm, pragmas, idents, lookups, wordrecg, liftdestructors, nifgen when not defined(nimKochBootstrap): + import vmdef import ast2nif import "../dist/nimony/src/lib" / [nifstreams, bitabs] diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index 73853a9d6b..88106b635f 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -11,7 +11,7 @@ import ast, astalgo, msgs, renderer, magicsys, types, idents, trees, wordrecg, options, guards, lineinfos, semfold, semdata, modulegraphs, varpartitions, typeallowed, nilcheck, errorhandling, - semstrictfuncs, suggestsymdb, pushpoppragmas, lowerings + semstrictfuncs, suggestsymdb, pushpoppragmas import std/[tables, intsets, strutils, sequtils] diff --git a/compiler/treetab.nim b/compiler/treetab.nim index b8b0f7b191..fd6db77fa6 100644 --- a/compiler/treetab.nim +++ b/compiler/treetab.nim @@ -9,7 +9,7 @@ # Implements a table from trees to trees. Does structural equivalence checking. -import ast, astalgo, types +import ast, types import std/hashes From ee55ddcffd784015ef89bfd59a2d61967ffa17e0 Mon Sep 17 00:00:00 2001 From: Pierre Thibault Date: Wed, 31 Dec 2025 07:34:18 -0500 Subject: [PATCH 18/31] Missleading sentence about array indexing (#25367) I added some precision. The first time I read this sentence, I was confused. This applies to the above example, but it cannot be generalized, since every array has its own range of valid indexes. I think this change make the documentation clearer. --------- Co-authored-by: Andreas Rumpf --- doc/tut1.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/doc/tut1.md b/doc/tut1.md index 3eaa1b1610..f116357394 100644 --- a/doc/tut1.md +++ b/doc/tut1.md @@ -1278,9 +1278,10 @@ Arrays can be constructed using `[]`: echo x[i] ``` -The notation `x[i]` is used to access the i-th element of `x`. -Array access is always bounds checked (at compile-time or at runtime). These -checks can be disabled via pragmas or invoking the compiler with the +The notation `x[i]` is used to access the i-th element of `x` in the example +above. Valid indexes can be defined by any subrange. Array access is +always bounds checked (at compile-time or at runtime). These checks can be +disabled via pragmas or invoking the compiler with the ``--bound_checks:off`` command line switch. Arrays are value types, like any other Nim type. The assignment operator From ae8a1739f8f703bff5df56a236887211e5cab2c3 Mon Sep 17 00:00:00 2001 From: Esteban C Borsani Date: Wed, 31 Dec 2025 21:31:33 -0300 Subject: [PATCH 19/31] Add `parseEnum` support for triple quoted string and raw string enum values (#25401) --- lib/std/enumutils.nim | 4 ++-- tests/stdlib/tstrutils.nim | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/lib/std/enumutils.nim b/lib/std/enumutils.nim index 9c338817d3..8bb593d74d 100644 --- a/lib/std/enumutils.nim +++ b/lib/std/enumutils.nim @@ -47,7 +47,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, of nnkEnumFieldDef: fVal = f[0].strVal case f[1].kind - of nnkStrLit: + of nnkStrLit .. nnkTripleStrLit: fStr = f[1].strVal of nnkTupleConstr: fStr = f[1][1].strVal @@ -57,7 +57,7 @@ macro genEnumCaseStmt*(typ: typedesc, argSym: typed, default: typed, fNum = f[1].intVal else: let fAst = f[0].getImpl - if fAst.kind == nnkStrLit: + if fAst.kind in {nnkStrLit .. nnkTripleStrLit}: fStr = fAst.strVal else: error("Invalid tuple syntax!", f[1]) diff --git a/tests/stdlib/tstrutils.nim b/tests/stdlib/tstrutils.nim index dfa72faf22..d57fa2d8ae 100644 --- a/tests/stdlib/tstrutils.nim +++ b/tests/stdlib/tstrutils.nim @@ -642,6 +642,30 @@ template main() = let myA = CAMPAIGN_TABLE doAssert $parseEnum[Tables](myA) == "wikientries_campaign" + block: + const tripleQuotedStr = """foobar""" + + type MyEnum = enum + a = tripleQuotedStr + b = """bazquz""" + + let myA = tripleQuotedStr + doAssert $parseEnum[MyEnum](myA) == myA + let myB = "bazquz" + doAssert $parseEnum[MyEnum](myB) == myB + + block: + const rawStr = r"foobar" + + type MyEnum = enum + a = rawStr + b = r"bazquz" + + let myA = rawStr + doAssert $parseEnum[MyEnum](myA) == myA + let myB = r"bazquz" + doAssert $parseEnum[MyEnum](myB) == myB + block: # check enum defined in block type Bar = enum From 92ad98f5d89bfbce6e8d94896e3f4d7fcb84a2f7 Mon Sep 17 00:00:00 2001 From: Jacek Sieka Date: Thu, 1 Jan 2026 01:33:35 +0100 Subject: [PATCH 20/31] pegs: get rid of spurious exception effects (#25399) Pegs raise only their own error, but the forward declaration causes an unwanted Exception effect * use strformat which does compile-time analysis of the format string to avoid exceptions * also in parsecfg --- lib/pure/parsecfg.nim | 10 ++++----- lib/pure/pegs.nim | 39 ++++++++++++++++++------------------ lib/std/private/ospaths2.nim | 2 +- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/lib/pure/parsecfg.nim b/lib/pure/parsecfg.nim index 99b1c9a41e..c5e71c0179 100644 --- a/lib/pure/parsecfg.nim +++ b/lib/pure/parsecfg.nim @@ -170,7 +170,7 @@ runnableExamples: assert dict.getSectionValue(section4, "does_that_mean_anything_special") == "False" assert dict.getSectionValue(section4, "purpose") == "formatting for readability" -import std/[strutils, lexbase, streams, tables] +import std/[strformat, strutils, lexbase, streams, tables] import std/private/decode_helpers import std/private/since @@ -220,7 +220,7 @@ type const SymChars = {'a'..'z', 'A'..'Z', '0'..'9', '_', ' ', '\x80'..'\xFF', '.', '/', '\\', '-'} -proc rawGetTok(c: var CfgParser, tok: var Token) {.gcsafe.} +proc rawGetTok(c: var CfgParser, tok: var Token) {.gcsafe, raises: [ValueError, OSError, IOError].} proc open*(c: var CfgParser, input: Stream, filename: string, lineOffset = 0) {.rtl, extern: "npc$1".} = @@ -428,14 +428,12 @@ proc rawGetTok(c: var CfgParser, tok: var Token) = proc errorStr*(c: CfgParser, msg: string): string {.rtl, extern: "npc$1".} = ## Returns a properly formatted error message containing current line and ## column information. - result = `%`("$1($2, $3) Error: $4", - [c.filename, $getLine(c), $getColumn(c), msg]) + &"{c.filename}({getLine(c)}, {getColumn(c)}) Error: {msg}" proc warningStr*(c: CfgParser, msg: string): string {.rtl, extern: "npc$1".} = ## Returns a properly formatted warning message containing current line and ## column information. - result = `%`("$1($2, $3) Warning: $4", - [c.filename, $getLine(c), $getColumn(c), msg]) + &"{c.filename}({getLine(c)}, {getColumn(c)}) Warning: {msg}" proc ignoreMsg*(c: CfgParser, e: CfgEvent): string {.rtl, extern: "npc$1".} = ## Returns a properly formatted warning message containing that diff --git a/lib/pure/pegs.nim b/lib/pure/pegs.nim index 451c7ee035..97d586a7c1 100644 --- a/lib/pure/pegs.nim +++ b/lib/pure/pegs.nim @@ -19,10 +19,12 @@ include "system/inclrtl" when defined(nimPreviewSlimSystem): import std/[syncio, assertions] +{.push gcsafe.} + const useUnicode = true ## change this to deactivate proper UTF-8 support -import std/[strutils, macros] +import std/[strformat, strutils, macros] import std/private/decode_helpers when useUnicode: @@ -562,10 +564,10 @@ template matchOrParse(mopProc: untyped) = # procs. For the former, *enter* and *leave* event handler code generators # are provided which just return *discard*. - proc mopProc(s: string, p: Peg, start: int, c: var Captures): int {.gcsafe, raises: [].} = + proc mopProc(s: string, p: Peg, start: int, c: var Captures): int {.raises: [].} = result = 0 - proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int = + proc matchBackRef(s: string, p: Peg, start: int, c: var Captures): int {.raises: [].}= # Parse handler code must run in an *of* clause of its own for each # *PegKind*, so we encapsulate the identical clause body for # *pkBackRef..pkBackRefIgnoreStyle* here. @@ -1031,7 +1033,7 @@ template eventParser*(pegAst, handlers: untyped): (proc(s: string): int) = ## Symbols declared in an *enter* handler can be made visible in the ## corresponding *leave* handler by annotating them with an *inject* pragma. proc rawParse(s: string, p: Peg, start: int, c: var Captures): int - {.gensym.} = + {.gensym, raises: [ValueError].} = # binding from *macros* bind strVal @@ -1297,7 +1299,7 @@ when not defined(nimHasEffectsOf): {.pragma: effectsOf.} func replace*(s: string, sub: Peg, cb: proc( - match: int, cnt: int, caps: openArray[string]): string): string {. + match: int, cnt: int, caps: openArray[string]): string {.gcsafe.}): string {. rtl, extern: "npegs$1cb", effectsOf: cb.} = ## Replaces `sub` in `s` by the resulting strings from the callback. ## The callback proc receives the index of the current match (starting with 0), @@ -1343,7 +1345,7 @@ func replace*(s: string, sub: Peg, cb: proc( when not defined(js): proc transformFile*(infile, outfile: string, subs: varargs[tuple[pattern: Peg, repl: string]]) {. - rtl, extern: "npegs$1".} = + rtl, extern: "npegs$1", raises: [ValueError, IOError].} = ## reads in the file `infile`, performs a parallel replacement (calls ## `parallelReplace`) and writes back to `outfile`. Raises ``IOError`` if an ## error occurs. This is supposed to be used for quick scripting. @@ -1482,9 +1484,9 @@ func getLine(L: PegLexer): int {.inline.} = result = L.lineNumber func errorStr(L: PegLexer, msg: string, line = -1, col = -1): string = - var line = if line < 0: getLine(L) else: line - var col = if col < 0: getColumn(L) else: col - result = "$1($2, $3) Error: $4" % [L.filename, $line, $col, msg] + let line = if line < 0: getLine(L) else: line + let col = if col < 0: getColumn(L) else: col + &"{L.filename}({line}, {col}) Error: {msg}" func getEscapedChar(c: var PegLexer, tok: var Token) = inc(c.bufpos) @@ -1679,7 +1681,7 @@ func getBuiltin(c: var PegLexer, tok: var Token) = tok.kind = tkEscaped getEscapedChar(c, tok) # may set tok.kind to tkInvalid -func getTok(c: var PegLexer, tok: var Token) = +func getTok(c: var PegLexer, tok: var Token) {.raises: [].} = tok.kind = tkInvalid tok.modifier = modNone setLen(tok.literal, 0) @@ -1822,11 +1824,10 @@ type identIsVerbatim: bool skip: Peg -func pegError(p: PegParser, msg: string, line = -1, col = -1) {.noreturn.} = - var e = (ref EInvalidPeg)(msg: errorStr(p, msg, line, col)) - raise e +func pegError(p: PegParser, msg: string, line = -1, col = -1) {.noreturn, raises: [EInvalidPeg].} = + raise (ref EInvalidPeg)(msg: errorStr(p, msg, line, col)) -func getTok(p: var PegParser) = +func getTok(p: var PegParser) {.raises: [EInvalidPeg].}= getTok(p, p.tok) if p.tok.kind == tkInvalid: pegError(p, "'" & p.tok.literal & "' is invalid token") @@ -1834,7 +1835,7 @@ func eat(p: var PegParser, kind: TokKind) = if p.tok.kind == kind: getTok(p) else: pegError(p, tokKindToStr[kind] & " expected") -func parseExpr(p: var PegParser): Peg {.gcsafe.} +func parseExpr(p: var PegParser): Peg {.raises: [EInvalidPeg].} func getNonTerminal(p: var PegParser, name: string): NonTerminal = for i in 0..high(p.nonterms): @@ -1883,7 +1884,7 @@ func token(terminal: Peg, p: PegParser): Peg = if p.skip.kind == pkEmpty: result = terminal else: result = sequence(p.skip, terminal) -func primary(p: var PegParser): Peg = +func primary(p: var PegParser): Peg {.raises: [EInvalidPeg].}= case p.tok.kind of tkAmp: getTok(p) @@ -1976,7 +1977,7 @@ func primary(p: var PegParser): Peg = getTok(p) else: break -func seqExpr(p: var PegParser): Peg = +func seqExpr(p: var PegParser): Peg {.raises: [EInvalidPeg].}= result = primary(p) while true: case p.tok.kind @@ -2042,7 +2043,7 @@ func rawParse(p: var PegParser): Peg = elif ntUsed notin nt.flags and i > 0: pegError(p, "unused rule: " & nt.name, nt.line, nt.col) -func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = +func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg {.raises: [EInvalidPeg].} = ## constructs a Peg object from `pattern`. `filename`, `line`, `col` are ## used for error messages, but they only provide start offsets. `parsePeg` ## keeps track of line and column numbers within `pattern`. @@ -2057,7 +2058,7 @@ func parsePeg*(pattern: string, filename = "pattern", line = 1, col = 0): Peg = getTok(p) result = rawParse(p) -func peg*(pattern: string): Peg = +func peg*(pattern: string): Peg {.raises: [EInvalidPeg].} = ## constructs a Peg object from the `pattern`. The short name has been ## chosen to encourage its use as a raw string modifier: ## diff --git a/lib/std/private/ospaths2.nim b/lib/std/private/ospaths2.nim index 43185f50a0..240736856f 100644 --- a/lib/std/private/ospaths2.nim +++ b/lib/std/private/ospaths2.nim @@ -41,7 +41,7 @@ proc normalizePathAux(path: var string){.inline, raises: [], noSideEffect.} import std/private/osseps export osseps -proc absolutePathInternal(path: string): string {.gcsafe.} +proc absolutePathInternal(path: string): string {.gcsafe, raises: [ValueError, OSerror].} proc normalizePathEnd*(path: var string, trailingSep = false) = ## Ensures ``path`` has exactly 0 or 1 trailing `DirSep`, depending on From 4b615aca46d1f2f0932e8a1a0319e449ff9e9468 Mon Sep 17 00:00:00 2001 From: Ryan McConnell Date: Sat, 3 Jan 2026 12:08:12 -0500 Subject: [PATCH 21/31] `memfiles.nim` resizeFile fallback logic bug (#25408) `e` is not cleared when falling back to `ftruncate` --- lib/pure/memfiles.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pure/memfiles.nim b/lib/pure/memfiles.nim index 2ba26e5c84..8e2f61868e 100644 --- a/lib/pure/memfiles.nim +++ b/lib/pure/memfiles.nim @@ -57,8 +57,12 @@ proc setFileSize(fh: FileHandle, newFileSize = -1, oldSize = -1): OSErrorCode = when declared(posix_fallocate): while (e = posix_fallocate(fh, 0, newFileSize); e == EINTR): discard - if (e == EINVAL or e == EOPNOTSUPP) and ftruncate(fh, newFileSize) == -1: - result = osLastError() # fallback arguable; Most portable BUT allows SEGV + if e == EINVAL or e == EOPNOTSUPP or e == ENOSYS: + # fallback arguable; Most portable BUT allows SEGV + if ftruncate(fh, newFileSize) == -1: + result = osLastError() + else: + discard elif e != 0: result = osLastError() else: # shrink the file From 1a651c17b3bbd6c1bd1578bf01b514b7c361da47 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 5 Jan 2026 19:36:33 +0800 Subject: [PATCH 22/31] hello 2026 (#25410) --- compiler/options.nim | 2 +- copying.txt | 2 +- readme.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 6dcec635b3..a1c373828b 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -25,7 +25,7 @@ const useEffectSystem* = true useWriteTracking* = false hasFFI* = defined(nimHasLibFFI) - copyrightYear* = "2025" + copyrightYear* = "2026" nimEnableCovariance* = defined(nimEnableCovariance) diff --git a/copying.txt b/copying.txt index 4025beacba..d56a058a8f 100644 --- a/copying.txt +++ b/copying.txt @@ -1,7 +1,7 @@ ===================================================== Nim -- a Compiler for Nim. https://nim-lang.org/ -Copyright (C) 2006-2025 Andreas Rumpf. All rights reserved. +Copyright (C) 2006-2026 Andreas Rumpf. All rights reserved. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/readme.md b/readme.md index 8aeec5c8e4..22d5294c2f 100644 --- a/readme.md +++ b/readme.md @@ -202,7 +202,7 @@ Nim. You are explicitly permitted to develop commercial applications using Nim. Please read the [copying.txt](copying.txt) file for more details. -Copyright © 2006-2025 Andreas Rumpf, all rights reserved. +Copyright © 2006-2026 Andreas Rumpf, all rights reserved. [nim-site]: https://nim-lang.org [nim-forum]: https://forum.nim-lang.org From a6c7989c7f6f0ae41e36ac60bfd10cae818088ea Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Mon, 5 Jan 2026 22:14:36 +0800 Subject: [PATCH 23/31] remove duplicated module imports (#25411) --- lib/system/alloc.nim | 1 - lib/system/channels_builtin.nim | 2 -- 2 files changed, 3 deletions(-) diff --git a/lib/system/alloc.nim b/lib/system/alloc.nim index 4109348fc2..8a29b3bf30 100644 --- a/lib/system/alloc.nim +++ b/lib/system/alloc.nim @@ -11,7 +11,6 @@ {.push profiler:off.} include osalloc -import std/private/syslocks template track(op, address, size) = when defined(memTracker): diff --git a/lib/system/channels_builtin.nim b/lib/system/channels_builtin.nim index 1cc9443778..2123707301 100644 --- a/lib/system/channels_builtin.nim +++ b/lib/system/channels_builtin.nim @@ -143,8 +143,6 @@ when not declared(ThisIsSystem): {.error: "You must not import this module explicitly".} -import std/private/syslocks - type pbytes = ptr UncheckedArray[byte] RawChannel {.pure, final.} = object ## msg queue for a thread From 780c9eeef027248984f564e4dec1ea04a1dbd70f Mon Sep 17 00:00:00 2001 From: elijahr Date: Mon, 5 Jan 2026 08:21:59 -0600 Subject: [PATCH 24/31] fixes #25405; initialization for objects with opaque importc fields (#25406) Objects containing `importc` fields without `completeStruct` fail to compile when used as const/static. The C codegen generates "aggregate initialization" which is invalid for opaque types. Fixes #25405. Nim code: ```nim type OpaqueInt {.importc: "_Atomic int", nodecl.} = object ContainsImportc = object normal: int opaque: OpaqueInt const c = default(ContainsImportc) ``` Resulting C code: ```c // Invalid C - cannot aggregate-init opaque type NIM_CONST ContainsImportc c = {((NI) 0), {}}; ^^ error: illegal initializer type ``` ## Solution Fix in `ccgexprs.nim`: 1. Skip opaque importc fields when building aggregate initializers 2. Use "designated initializers" (`siNamedStruct`) when opaque fields are present to avoid positional misalignment ```c // Valid C: // - opaque field is omitted and implicitly zero-initialized by C // - other fields are explitly named and initialized NIM_CONST ContainsImportc c = {.normal = ((NI) 0)}; ``` This correctly handles the case where the opaque fields might be in any order. A field is considered "opaque importc" if: - Has `sfImportc` flag - Does NOT have `tfCompleteStruct` flag - Either has `tfIncompleteStruct` OR is an object with no visible fields The `containsOpaqueImportcField` proc recursively checks all object fields, including nested objects and variant branches. Anonymous unions (from variant objects) are handled by passing an empty field name, which skips the `.fieldname = ` prefix since C anonymous unions have no field name. Note that initialization for structs without opaque importc fields remains the same as before this changeset. ## Test Coverage `tests/ccgbugs/timportc_field_init.nim` covers: - Simple struct with one importc field - Nested struct containing struct with importc field - Variant object (case object) with importc field in a branch - Array of structs with importc fields - Tuple containing struct with importc field - `completeStruct` importc types (still use aggregate init) - Sandwich case (opaque field between two non-opaque fields) - Fields with different C names (`{.importc: "c_name".}`, `{.exportc.}`) - `{.packed.}` structs with opaque fields - `{.union.}` types with opaque fields - Deep nesting (3+ levels) - Multiple opaque fields with renamed fields between them --- compiler/cbuilderdecls.nim | 12 +- compiler/ccgexprs.nim | 102 ++++++++++++--- tests/ccgbugs/timportc_field_init.nim | 178 ++++++++++++++++++++++++++ 3 files changed, 269 insertions(+), 23 deletions(-) create mode 100644 tests/ccgbugs/timportc_field_init.nim diff --git a/compiler/cbuilderdecls.nim b/compiler/cbuilderdecls.nim index 0b170c7183..eb6dd3d627 100644 --- a/compiler/cbuilderdecls.nim +++ b/compiler/cbuilderdecls.nim @@ -154,14 +154,14 @@ template addField(builder: var Builder, constr: var StructInitializer, name: str # no name, can just add value valueBody of siOrderedStruct: - # no name, can just add value on C - assert name.len != 0, "name has to be given for struct initializer field" + # positional init - name not used in output (empty allowed for anonymous unions) valueBody of siNamedStruct: - assert name.len != 0, "name has to be given for struct initializer field" - builder.add(".") - builder.add(name) - builder.add(" = ") + # designated init - empty name for anonymous unions (skips .name = prefix) + if name.len != 0: + builder.add(".") + builder.add(name) + builder.add(" = ") valueBody proc finishStructInitializer(builder: var Builder, constr: StructInitializer) = diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 2ef134f497..e4e51f65be 100644 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -3713,6 +3713,64 @@ proc expr(p: BProc, n: PNode, d: var TLoc) = of nkMixinStmt, nkBindStmt, nkReplayAction: discard else: internalError(p.config, n.info, "expr(" & $n.kind & "); unknown node kind") +proc isOpaqueImportcType(t: PType): bool = + # importc type without completeStruct that can't use aggregate init (e.g. C11 _Atomic) + if t.sym != nil and sfImportc in t.sym.flags: + if tfCompleteStruct notin t.flags: + if tfIncompleteStruct in t.flags: + return true + if t.kind == tyObject and (t.n == nil or t.n.len == 0): + return true + return false + +proc containsOpaqueImportcField(typ: PType): bool + +proc containsOpaqueImportcFieldAux(t: PType; n: PNode): bool = + if n == nil: return false + case n.kind + of nkRecList: + for child in n.sons: + if containsOpaqueImportcFieldAux(t, child): + return true + of nkRecCase: + if containsOpaqueImportcFieldAux(t, n[0]): + return true + for i in 1..".} = object + +type + SimpleStruct = object + normal: int + opaque: OpaqueFile + + NestedStruct = object + inner: SimpleStruct + value: float + + VariantStruct = object + case kind: bool + of true: + opaque: OpaqueFile + of false: + normal: int + + ArrayElementStruct = object + id: int + atom: OpaqueFile + +const simple = default(SimpleStruct) +const nested = default(NestedStruct) +const variant = default(VariantStruct) +const arr = default(array[3, ArrayElementStruct]) + +static: + doAssert simple.normal == 0 + doAssert nested.value == 0.0 + doAssert arr[0].id == 0 + +# completeStruct types use normal aggregate init +type CompleteImportc {.importc: "int", completeStruct, nodecl.} = object + value: cint + +type StructWithComplete = object + c: CompleteImportc + x: int + +const withComplete = default(StructWithComplete) + +type TupleWithOpaque = tuple[x: int, s: SimpleStruct, y: float] +const tupleVal = default(TupleWithOpaque) + +# Sandwich: opaque between non-opaque fields requires designated init +type SandwichStruct = object + first: int + opaque: OpaqueFile + last: float + +const sandwich = default(SandwichStruct) + +static: + doAssert withComplete.x == 0 + doAssert tupleVal.x == 0 + doAssert sandwich.first == 0 + doAssert sandwich.last == 0.0 + +proc useSimple(s: ptr SimpleStruct) {.exportc, noinline.} = discard +proc useNested(s: ptr NestedStruct) {.exportc, noinline.} = discard +proc useArr(a: ptr array[3, ArrayElementStruct]) {.exportc, noinline.} = discard +proc useComplete(s: ptr StructWithComplete) {.exportc, noinline.} = discard +proc useVariant(v: ptr VariantStruct) {.exportc, noinline.} = discard +proc useTuple(t: TupleWithOpaque) {.exportc, noinline.} = discard +proc useSandwich(s: ptr SandwichStruct) {.exportc, noinline.} = discard + +useSimple(simple.addr) +useNested(nested.addr) +useArr(arr.addr) +useComplete(withComplete.addr) +useVariant(variant.addr) +useTuple(tupleVal) +useSandwich(sandwich.addr) + +# Edge cases: different C/Nim names +type OpaqueWithCName {.importc: "FILE", header: "".} = object + +type StructWithRenamedField = object + nimName {.importc: "c_name".}: int + opaque: OpaqueWithCName + +const renamedField = default(StructWithRenamedField) +proc useRenamedField(s: ptr StructWithRenamedField) {.exportc, noinline.} = discard +useRenamedField(renamedField.addr) +static: doAssert renamedField.nimName == 0 + +type NimTypeName {.importc: "int", completeStruct, nodecl.} = distinct cint + +type StructContainingRenamedType = object + inner: NimTypeName + opaque: OpaqueFile + +const withRenamedType = default(StructContainingRenamedType) +proc useRenamedType(s: ptr StructContainingRenamedType) {.exportc, noinline.} = discard +useRenamedType(withRenamedType.addr) + +type StructWithExportedField = object + nimField {.exportc: "exported_field".}: int + opaque: OpaqueFile + +const withExported = default(StructWithExportedField) +proc useExportedField(s: ptr StructWithExportedField) {.exportc, noinline.} = discard +useExportedField(withExported.addr) +static: doAssert withExported.nimField == 0 + +type ByCopyStruct {.bycopy.} = object + data: int + opaque: OpaqueFile + +const byCopyVal = default(ByCopyStruct) +proc useByCopy(s: ByCopyStruct) {.exportc, noinline.} = discard +useByCopy(byCopyVal) +static: doAssert byCopyVal.data == 0 + +type PackedStruct {.packed.} = object + a: int8 + opaque: OpaqueFile + b: int8 + +const packedVal = default(PackedStruct) +proc usePacked(s: ptr PackedStruct) {.exportc, noinline.} = discard +usePacked(packedVal.addr) +static: + doAssert packedVal.a == 0 + doAssert packedVal.b == 0 + +type UnionWithOpaque {.union.} = object + intVal: int + opaque: OpaqueFile + +const unionVal = default(UnionWithOpaque) +proc useUnion(u: ptr UnionWithOpaque) {.exportc, noinline.} = discard +useUnion(unionVal.addr) + +# Deep nesting +type DeepLevel1 = object + field1: int + opaque: OpaqueFile + +type DeepLevel2 = object + nested: DeepLevel1 + field2: float + +type DeepLevel3 = object + deep: DeepLevel2 + field3: int + opaque2: OpaqueWithCName + +const deepVal = default(DeepLevel3) +proc useDeep(d: ptr DeepLevel3) {.exportc, noinline.} = discard +useDeep(deepVal.addr) +static: + doAssert deepVal.deep.nested.field1 == 0 + doAssert deepVal.deep.field2 == 0.0 + doAssert deepVal.field3 == 0 + +# Multiple opaque fields with renamed non-opaque fields +type MultiOpaque = object + first {.importc: "first_field".}: int + opaque1: OpaqueFile + second {.importc: "second_field".}: float + opaque2: OpaqueWithCName + third: int + +const multiOpaque = default(MultiOpaque) +proc useMultiOpaque(m: ptr MultiOpaque) {.exportc, noinline.} = discard +useMultiOpaque(multiOpaque.addr) + +static: + doAssert multiOpaque.first == 0 + doAssert multiOpaque.second == 0.0 + doAssert multiOpaque.third == 0 From d3be5e5e135401e9a403d9b95e76c888d492c724 Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Tue, 6 Jan 2026 00:16:19 +0100 Subject: [PATCH 25/31] IC: need a more recent Nimony for its improved Nifler tool (#25412) --- compiler/commands.nim | 3 ++- compiler/ic/navigator.nim | 2 +- compiler/options.nim | 1 + koch.nim | 7 +++++-- testament/categories.nim | 6 +++--- tests/ic/tgenericinst.nim | 4 ++-- 6 files changed, 14 insertions(+), 9 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 622e5536fe..869fc682a7 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1000,7 +1000,8 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; # xxx maybe also ic, since not in help? if pass in {passCmd2, passPP}: case arg.normalize - of "on": conf.symbolFiles = v2Sf + of "on": conf.ic = true + of "legacy": conf.symbolFiles = v2Sf of "off": conf.symbolFiles = disabledSf of "writeonly": conf.symbolFiles = writeOnlySf of "readonly": conf.symbolFiles = readOnlySf diff --git a/compiler/ic/navigator.nim b/compiler/ic/navigator.nim index 39037b94f2..9d58aa3840 100644 --- a/compiler/ic/navigator.nim +++ b/compiler/ic/navigator.nim @@ -7,7 +7,7 @@ # distribution, for details about the copyright. # -## Supports the "nim check --ic:on --defusages:FILE,LINE,COL" +## Supports the "nim check --ic:legacy --defusages:FILE,LINE,COL" ## IDE-like features. It uses the set of .rod files to accomplish ## its task. The set must cover a complete Nim project. diff --git a/compiler/options.nim b/compiler/options.nim index a1c373828b..28e7014497 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -369,6 +369,7 @@ type numberOfProcessors*: int # number of processors lastCmdTime*: float # when caas is enabled, we measure each command symbolFiles*: SymbolFilesOption + ic*: bool # whether ic is enabled spellSuggestMax*: int # max number of spelling suggestions for typos cppDefines*: HashSet[string] # (*) diff --git a/koch.nim b/koch.nim index 58df9fadb7..0a7cd2ece4 100644 --- a/koch.nim +++ b/koch.nim @@ -16,7 +16,7 @@ const ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1 SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01" - NimonyStableCommit = "322178d9af6676363d5237382c6d6c1b4e56d3cd" # unversioned \ + NimonyStableCommit = "e2cd6eadcaa68eb8ab380cb4d3bdd7fd260677b4" # unversioned \ # Note that Nimony uses Nim as a git submodule but we don't want to install # Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive # is **required** here. @@ -188,6 +188,9 @@ proc bundleChecksums(latest: bool) = let nimonyCommit = if latest: "HEAD" else: NimonyStableCommit cloneDependency(distDir, "https://github.com/nim-lang/nimony.git", nimonyCommit, allowBundled = true) + nimCompileFold("Compile nifler", "dist/nimony/src/nifler/nifler.nim", options = "-d:release") + nimCompileFold("Compile nifmake", "dist/nimony/src/nifmake/nifmake.nim", options = "-d:release") + proc bundleNimsuggest(args: string) = bundleChecksums(false) nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim", @@ -553,7 +556,7 @@ proc icTest(args: string) = for fragment in content.split("#!EDIT!#"): let file = inp.replace(".nim", "_temp.nim") writeFile(file, fragment) - var cmd = nimExe & " cpp --ic:on -d:nimIcIntegrityChecks --listcmd " + var cmd = nimExe & " cpp --ic:legacy -d:nimIcIntegrityChecks --listcmd " if i == 0: cmd.add "-f " cmd.add quoteShell(file) diff --git a/testament/categories.nim b/testament/categories.nim index eba1e3cb27..b16ddbb91d 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -493,8 +493,8 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; tooltests = ["compiler/nim.nim"] writeOnly = " --incremental:writeonly " readOnly = " --incremental:readonly " - incrementalOn = " --incremental:on -d:nimIcIntegrityChecks " - navTestConfig = " --ic:on -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " + incrementalOn = " --incremental:legacy -d:nimIcIntegrityChecks " + navTestConfig = " --ic:legacy -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " template test(x: untyped) = testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) @@ -508,7 +508,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; template checkTest() = var test = makeRawTest(file, options, cat) - test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:on $options " & file + test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:legacy $options " & file testSpecWithNimcache(r, test, nimcache) if not isNavigatorTest: diff --git a/tests/ic/tgenericinst.nim b/tests/ic/tgenericinst.nim index 3346764f54..dea55235b1 100644 --- a/tests/ic/tgenericinst.nim +++ b/tests/ic/tgenericinst.nim @@ -1,5 +1,5 @@ discard """ - cmd: "nim cpp --incremental:on $file" + cmd: "nim cpp --incremental:legacy $file" """ {.emit:"""/*TYPESECTION*/ @@ -8,4 +8,4 @@ discard """ """.} type Foo {.importcpp.} = object -echo $Foo() #Notice the generic is instantiate in the this module if not, it wouldnt find Foo \ No newline at end of file +echo $Foo() #Notice the generic is instantiate in the this module if not, it wouldnt find Foo From 89c8f0aa494bae4a607d139bbf07d7d918572784 Mon Sep 17 00:00:00 2001 From: ringabout <43030857+ringabout@users.noreply.github.com> Date: Wed, 7 Jan 2026 16:32:25 +0800 Subject: [PATCH 26/31] closes #23394; adds a test case (#25416) closes #23394 --- tests/arc/tgenerics.nim | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/arc/tgenerics.nim diff --git a/tests/arc/tgenerics.nim b/tests/arc/tgenerics.nim new file mode 100644 index 0000000000..20495dc029 --- /dev/null +++ b/tests/arc/tgenerics.nim @@ -0,0 +1,19 @@ +discard """ + matrix: "--mm:refc" +""" +type + State = enum + Uninit + Init + Uart[T: static State] = object + baudRate: int + port: int + +proc `=destroy`(uart: var Uart[Init]) = raiseAssert "Destroyed" + +# proc `=copy`(a: var Uart[Init], b: Uart[Init]) {.error.} # Error: signature for '=copy' must be proc[T: object](x: var T; y: T) + +proc main() = + var a = Uart[Uninit]() + +main() \ No newline at end of file From 251b4a23c30be46c0aab1e565ba589ceff74f07b Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 7 Jan 2026 13:45:26 +0100 Subject: [PATCH 27/31] IC: run nifmake automatically (#25415) --- compiler/commands.nim | 2 +- compiler/deps.nim | 87 +++++++++++++++++++++++++++++++++---------- compiler/main.nim | 8 ++-- compiler/nim.nim | 2 +- compiler/options.nim | 2 +- koch.nim | 8 ++-- 6 files changed, 81 insertions(+), 28 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 869fc682a7..1bf8ec5505 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -499,7 +499,7 @@ proc parseCommand*(command: string): Command = of "nop", "help": cmdNop of "jsonscript": cmdJsonscript of "nifc": cmdNifC # generate C from NIF files - of "deps": cmdDeps # generate .build.nif for nifmake + of "ic": cmdIc # generate .build.nif for nifmake else: cmdUnknown proc setCmd*(conf: ConfigRef, cmd: Command) = diff --git a/compiler/deps.nim b/compiler/deps.nim index aa5322ecb6..7f3dfb797b 100644 --- a/compiler/deps.nim +++ b/compiler/deps.nim @@ -11,7 +11,7 @@ ## This enables incremental and parallel compilation using the `m` switch. import std / [os, tables, sets, times, osproc, strutils] -import options, msgs, lineinfos +import options, msgs, lineinfos, pathutils import "../dist/nimony/src/lib" / [nifstreams, nifcursors, bitabs, nifreader, nifbuilder] import "../dist/nimony/src/gear2" / modnames @@ -47,15 +47,18 @@ proc semmedFile(c: DepContext; f: FilePair): string = proc findNifler(): string = # Look for nifler in common locations - result = findExe("nifler") - if result.len == 0: - # Try relative to nim executable - let nimDir = getAppDir() - result = nimDir / "nifler" - if not fileExists(result): - result = nimDir / ".." / "nimony" / "bin" / "nifler" - if not fileExists(result): - result = "" + let nimDir = getAppDir() + result = nimDir / "nifler" + if not fileExists(result): + result = findExe("nifler") + +proc findNifmake(): string = + # Look for nifmake in common locations + # Try relative to nim executable + let nimDir = getAppDir() + result = nimDir / "nifmake" + if not fileExists(result): + result = findExe("nifmake") proc runNifler(c: DepContext; nimFile: string): bool = ## Run nifler deps on a file if needed. Returns true on success. @@ -220,12 +223,14 @@ proc traverseDeps(c: var DepContext; pair: FilePair; current: Node) = proc generateBuildFile(c: DepContext): string = ## Generate the .build.nif file for nifmake - result = getNimcacheDir(c.config).string / c.nodes[0].files[0].modname & ".build.nif" + createDir("nifcache") + result = "nifcache" / c.nodes[0].files[0].modname & ".build.nif" + #getNimcacheDir(c.config).string / c.nodes[0].files[0].modname & ".build.nif" var b = nifbuilder.open(result) defer: b.close() - b.addHeader("nim deps", "nifmake") + b.addHeader("nim ic", "nifmake") b.addTree "stmts" # Define nifler command @@ -245,6 +250,22 @@ proc generateBuildFile(c: DepContext): string = b.addSymbolDef "nim_m" b.addStrLit getAppFilename() b.addStrLit "m" + b.addStrLit "--nimcache:nifcache" + # Add search paths + for p in c.config.searchPaths: + b.addStrLit "--path:" & p.string + b.addTree "args" + b.endTree() + b.withTree "input": + b.addIntLit 0 # main parsed file + b.endTree() + + # Define nim nifc command + b.addTree "cmd" + b.addSymbolDef "nim_nifc" + b.addStrLit getAppFilename() + b.addStrLit "nifc" + b.addStrLit "--nimcache:nifcache" # Add search paths for p in c.config.searchPaths: b.addStrLit "--path:" & p.string @@ -279,6 +300,8 @@ proc generateBuildFile(c: DepContext): string = b.addTree "do" b.addIdent "nim_m" # Input: all parsed files for this module + b.withTree "input": + b.addStrLit node.files[0].nimFile for f in node.files: b.addTree "input" b.addStrLit c.parsedFile(f) @@ -292,15 +315,26 @@ proc generateBuildFile(c: DepContext): string = b.addTree "output" b.addStrLit c.semmedFile(pair) b.endTree() - b.addTree "args" - b.addStrLit pair.nimFile - b.endTree() b.endTree() + # Final compilation step: generate executable from main module + let mainNif = c.nodes[0].files[0].nimFile + let exeFile = changeFileExt(c.nodes[0].files[0].nimFile, ExeExt) + b.addTree "do" + b.addIdent "nim_nifc" + # Input: .nim file (expanded as argument) and .nif file (dependency) + b.addTree "input" + b.addStrLit mainNif + b.endTree() + b.addTree "output" + b.addStrLit exeFile + b.endTree() + b.endTree() + b.endTree() # stmts -proc commandDeps*(conf: ConfigRef) = - ## Main entry point for `nim deps` +proc commandIc*(conf: ConfigRef) = + ## Main entry point for `nim ic` when not defined(nimKochBootstrap): let nifler = findNifler() if nifler.len == 0: @@ -329,12 +363,27 @@ proc commandDeps*(conf: ConfigRef) = c.nodes.add rootNode c.processedModules[rootPair.modname] = 0 + # model the system.nim dependency: + let sysNode = Node(files: @[toPair(c, (conf.libpath / RelativeFile"system.nim").string)], id: 1) + c.nodes.add sysNode + rootNode.deps.add sysNode.id + # Process dependencies traverseDeps(c, rootPair, rootNode) # Generate build file let buildFile = generateBuildFile(c) rawMessage(conf, hintSuccess, "generated: " & buildFile) - rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile) + + # Automatically run nifmake + let nifmake = findNifmake() + if nifmake.len == 0: + rawMessage(conf, hintSuccess, "run: nifmake run " & buildFile) + else: + let cmd = quoteShell(nifmake) & " run " & quoteShell(buildFile) + rawMessage(conf, hintExecuting, cmd) + let exitCode = execShellCmd(cmd) + if exitCode != 0: + rawMessage(conf, errGenerated, "nifmake failed with exit code: " & $exitCode) else: - rawMessage(conf, errGenerated, "nim deps not available in bootstrap build") + rawMessage(conf, errGenerated, "nim ic not available in bootstrap build") diff --git a/compiler/main.nim b/compiler/main.nim index d63c7d3fbf..c0276d058c 100644 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -442,18 +442,20 @@ proc mainCommand*(graph: ModuleGraph) = of cmdM: # cmdM uses NIF files, not ROD files graph.config.symbolFiles = disabledSf - setUseIc(false) + setUseIc(true) commandCheck(graph) of cmdNifC: + setUseIc(true) # Generate C code from NIF files wantMainModule(conf) setOutFile(conf) commandNifC(graph) - of cmdDeps: + of cmdIc: # Generate .build.nif for nifmake + setUseIc(true) wantMainModule(conf) when not defined(nimKochBootstrap): - commandDeps(conf) + commandIc(conf) else: rawMessage(conf, errGenerated, "nim deps not available in bootstrap build") of cmdParse: diff --git a/compiler/nim.nim b/compiler/nim.nim index 72302a186e..ed6774983c 100644 --- a/compiler/nim.nim +++ b/compiler/nim.nim @@ -118,7 +118,7 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) = if conf.selectedGC == gcUnselected: if conf.backend in {backendC, backendCpp, backendObjc} or (conf.cmd in cmdDocLike and conf.backend != backendJs) or - conf.cmd in {cmdGendepend, cmdNifC, cmdDeps, cmdM}: + conf.cmd in {cmdGendepend, cmdNifC, cmdIc, cmdM}: initOrcDefines(conf) mainCommand(graph) diff --git a/compiler/options.nim b/compiler/options.nim index 28e7014497..086954563d 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -177,7 +177,7 @@ type # old unused: cmdInterpret, cmdDef: def feature (find definition for IDEs) cmdCompileToNif cmdNifC # generate C code from NIF files - cmdDeps # generate .build.nif for nifmake + cmdIc # generate .build.nif for nifmake const cmdBackends* = {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, diff --git a/koch.nim b/koch.nim index 0a7cd2ece4..7d7123abdd 100644 --- a/koch.nim +++ b/koch.nim @@ -16,7 +16,7 @@ const ChecksumsStableCommit = "0b8e46379c5bc1bf73d8b3011908389c60fb9b98" # 2.0.1 SatStableCommit = "faf1617f44d7632ee9601ebc13887644925dcc01" - NimonyStableCommit = "e2cd6eadcaa68eb8ab380cb4d3bdd7fd260677b4" # unversioned \ + NimonyStableCommit = "fc8baa61b9911caf4666685a5f5ed41b9c04f6f8" # unversioned \ # Note that Nimony uses Nim as a git submodule but we don't want to install # Nimony's dependency to Nim as we are Nim. So a `git clone` without --recursive # is **required** here. @@ -188,8 +188,10 @@ proc bundleChecksums(latest: bool) = let nimonyCommit = if latest: "HEAD" else: NimonyStableCommit cloneDependency(distDir, "https://github.com/nim-lang/nimony.git", nimonyCommit, allowBundled = true) - nimCompileFold("Compile nifler", "dist/nimony/src/nifler/nifler.nim", options = "-d:release") - nimCompileFold("Compile nifmake", "dist/nimony/src/nifmake/nifmake.nim", options = "-d:release") + if not fileExists("bin/nifler".exe): + nimCompileFold("Compile nifler", "dist/nimony/src/nifler/nifler.nim", options = "-d:release") + if not fileExists("bin/nifmake".exe): + nimCompileFold("Compile nifmake", "dist/nimony/src/nifmake/nifmake.nim", options = "-d:release") proc bundleNimsuggest(args: string) = bundleChecksums(false) From b3273e732dd628a0881448bc82ebedf103776ece Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Wed, 7 Jan 2026 17:35:07 +0100 Subject: [PATCH 28/31] IC: progress (#25417) --- compiler/ast2nif.nim | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/compiler/ast2nif.nim b/compiler/ast2nif.nim index 01830b65fe..dc7a422228 100644 --- a/compiler/ast2nif.nim +++ b/compiler/ast2nif.nim @@ -894,6 +894,9 @@ proc getOffset(c: var DecodeContext; module: FileIndex; nifName: string): NifInd proc loadNode(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]): PNode +proc loadSymFromCursor(c: var DecodeContext; s: PSym; n: var Cursor; thisModule: string; + localSyms: var Table[string, PSym]) + proc createTypeStub(c: var DecodeContext; t: SymId): PType = let name = pool.syms[t] assert name.startsWith("`t") @@ -919,8 +922,8 @@ proc createTypeStub(c: var DecodeContext; t: SymId): PType = proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: string; localSyms: var Table[string, PSym]) = ## Scan a tree for local symbol definitions (sdef tags) and add them to localSyms. - ## This doesn't fully load the symbols, just pre-registers them so references - ## can find them. After this proc returns, n is positioned AFTER the tree. + ## For local symbols, fully load them immediately since they have no index offsets. + ## After this proc returns, n is positioned AFTER the tree. # Handle atoms (non-compound nodes) - just skip them if n.kind != ParLe: inc n @@ -935,7 +938,8 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s let symName = pool.syms[name.symId] let sn = parseSymName(symName) if sn.module.len == 0 and symName notin localSyms: - # Local symbol - create a stub entry in localSyms + # Local symbol - create stub and immediately load it fully + # since local symbols have no index offsets for lazy loading let module = moduleId(c, thisModule) let val = addr c.mods[module].symCounter inc val[] @@ -943,6 +947,17 @@ proc extractLocalSymsFromTree(c: var DecodeContext; n: var Cursor; thisModule: s let sym = PSym(itemId: id, kindImpl: skStub, name: c.cache.getIdent(sn.name), disamb: sn.count.int32, state: Complete) localSyms[symName] = sym + # Load the full symbol definition immediately + # We're currently at the `(sd` position, need to skip to SymbolDef + inc n # skip past `sd` tag to get to SymbolDef + inc depth # account for the opening `(` of the sdef + loadSymFromCursor(c, sym, n, thisModule, localSyms) + sym.state = Sealed # mark as fully loaded + # loadSymFromCursor consumed everything including the closing `)`, + # so we need to account for it in depth tracking + dec depth + # Continue processing - loadSymFromCursor already advanced n past the closing `)` + continue inc depth elif n.kind == ParRi: dec depth From 01eedd916c914f7aa2346826016332cc4931286d Mon Sep 17 00:00:00 2001 From: Andreas Rumpf Date: Fri, 9 Jan 2026 13:10:04 +0100 Subject: [PATCH 29/31] IC: progress (#25420) --- compiler/modulepaths.nim | 6 ++- compiler/pipelines.nim | 17 +++++++- testament/categories.nim | 43 +++---------------- .../ic/ic_disabled}/config.nims | 0 .../ic/ic_disabled}/mbaseobj.nim | 0 .../ic/ic_disabled}/mcompiletime_counter.nim | 0 .../ic/ic_disabled}/mdefconverter.nim | 0 .../ic/ic_disabled}/mimports.nim | 0 .../ic/ic_disabled}/mimportsb.nim | 0 .../ic/ic_disabled}/tcompiletime_counter.nim | 0 .../ic/ic_disabled}/tconverter.nim | 0 .../ic/ic_disabled}/tgenericinst.nim | 0 .../ic/ic_disabled}/tgenerics.nim | 0 .../ic/ic_disabled}/timports.nim | 0 .../ic/ic_disabled}/tmethods.nim | 0 .../ic_disabled}/tstdlib_import_changed.nim | 0 16 files changed, 26 insertions(+), 40 deletions(-) rename {tests/ic => tests_disabled/ic/ic_disabled}/config.nims (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/mbaseobj.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/mcompiletime_counter.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/mdefconverter.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/mimports.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/mimportsb.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/tcompiletime_counter.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/tconverter.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/tgenericinst.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/tgenerics.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/timports.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/tmethods.nim (100%) rename {tests/ic => tests_disabled/ic/ic_disabled}/tstdlib_import_changed.nim (100%) diff --git a/compiler/modulepaths.nim b/compiler/modulepaths.nim index 7279ae6ce2..35f19b4663 100644 --- a/compiler/modulepaths.nim +++ b/compiler/modulepaths.nim @@ -109,9 +109,11 @@ proc mangleModuleName*(conf: ConfigRef; path: AbsoluteFile): string = of FromSearchPath: "@p" of FromNimblePath: "@n" + # Note: We encode ".." specially as "@d" to avoid issues with changeFileExt + # which would misinterpret ".." as "name.ext" and strip the second part. prefix & best.multiReplace( - {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) + {"..": "@d", $os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"}) proc demangleModuleName*(path: string): string = ## Demangle a relative module path. - result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@p": "", "@n": "", "@c": ":"}) + result = path.multiReplace({"@@": "@", "@d": "..", "@h": "#", "@s": "/", "@m": "", "@p": "", "@n": "", "@c": ":"}) diff --git a/compiler/pipelines.nim b/compiler/pipelines.nim index fd1193bd3b..6c67f1268c 100644 --- a/compiler/pipelines.nim +++ b/compiler/pipelines.nim @@ -242,8 +242,11 @@ proc processPipelineModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator raiseAssert "use setPipeLinePass to set a proper PipelinePass" when not defined(nimKochBootstrap): - if (optCompress in graph.config.globalOptions or graph.config.cmd == cmdM) and - not graph.config.isDefined("nimscript"): + # For cmdM: only write NIF for the main module, not for imported modules + # (imported modules should be loaded from existing NIF files) + let shouldWriteNif = (optCompress in graph.config.globalOptions) or + (graph.config.cmd == cmdM and sfMainModule in module.flags) + if shouldWriteNif and not graph.config.isDefined("nimscript"): topLevelStmts.add finalNode # Collect replay actions from both pragma computations and VM state diff var replayActions: seq[PNode] = @[] @@ -294,6 +297,16 @@ proc compilePipelineModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymF "nim m requires precompiled NIF for import: " & toFullPath(graph.config, fileIdx) & " (expected: " & nifPath & ")") return nil # Don't fall through to compile from source + else: + # Module successfully loaded from NIF file - use it and skip processing + result = precomp.module + if sfSystemModule in flags: + graph.systemModule = result + partialInitModule(result, graph, fileIdx, AbsoluteFile(toFullPath(graph.config, fileIdx))) + # Replay state changes from the loaded NIF module + if result.ast != nil: + replayStateChanges(result, graph) + return result # Return early, don't process from source if result == nil and graph.config.cmd != cmdM: # Fall back to ROD file loading (not used for cmdM which uses NIF only) result = moduleFromRodFile(graph, fileIdx, cachedModules) diff --git a/testament/categories.nim b/testament/categories.nim index b16ddbb91d..a86541dc3f 100644 --- a/testament/categories.nim +++ b/testament/categories.nim @@ -27,6 +27,7 @@ const "io", "js", "ic", + "ic_disabled", "lib", "manyloc", "nimble-packages", @@ -489,46 +490,16 @@ proc testNimblePackages(r: var TResults; cat: Category; packageFilter: string) = proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; isNavigatorTest: bool) = - const - tooltests = ["compiler/nim.nim"] - writeOnly = " --incremental:writeonly " - readOnly = " --incremental:readonly " - incrementalOn = " --incremental:legacy -d:nimIcIntegrityChecks " - navTestConfig = " --ic:legacy -d:nimIcNavigatorTests --hint:Conf:off --warnings:off " - - template test(x: untyped) = - testSpecWithNimcache(r, makeRawTest(file, x & options, cat), nimcache) - - template editedTest(x: untyped) = - var test = makeTest(file, x & options, cat) - if isNavigatorTest: - test.spec.action = actionCompile - test.spec.targets = {getTestSpecTarget()} + template editedTest() = + var test = makeTest(file, options, cat) + test.spec.targets = {targetC} + test.spec.cmd = compilerPrefix & " ic --hint:Conf:off --warnings:off $options " & file testSpecWithNimcache(r, test, nimcache) - template checkTest() = - var test = makeRawTest(file, options, cat) - test.spec.cmd = compilerPrefix & " check --hint:Conf:off --warnings:off --ic:legacy $options " & file - testSpecWithNimcache(r, test, nimcache) - - if not isNavigatorTest: - for file in tooltests: - let nimcache = nimcacheDir(file, options, getTestSpecTarget()) - removeDir(nimcache) - - let oldPassed = r.passed - checkTest() - - if r.passed == oldPassed+1: - checkTest() - if r.passed == oldPassed+2: - checkTest() - const tempExt = "_temp.nim" for it in walkDirRec(testsDir): - # for it in ["tests/ic/timports.nim"]: # debugging: to try a specific test if isTestFile(it) and not it.endsWith(tempExt): - let nimcache = nimcacheDir(it, options, getTestSpecTarget()) + let nimcache = nimcacheDir(it, options, targetC) removeDir(nimcache) let content = readFile(it) @@ -536,7 +507,7 @@ proc icTests(r: var TResults; testsDir: string, cat: Category, options: string; let file = it.replace(".nim", tempExt) writeFile(file, fragment) let oldPassed = r.passed - editedTest(if isNavigatorTest: navTestConfig else: incrementalOn) + editedTest() if r.passed != oldPassed+1: break # ---------------------------------------------------------------------------- diff --git a/tests/ic/config.nims b/tests_disabled/ic/ic_disabled/config.nims similarity index 100% rename from tests/ic/config.nims rename to tests_disabled/ic/ic_disabled/config.nims diff --git a/tests/ic/mbaseobj.nim b/tests_disabled/ic/ic_disabled/mbaseobj.nim similarity index 100% rename from tests/ic/mbaseobj.nim rename to tests_disabled/ic/ic_disabled/mbaseobj.nim diff --git a/tests/ic/mcompiletime_counter.nim b/tests_disabled/ic/ic_disabled/mcompiletime_counter.nim similarity index 100% rename from tests/ic/mcompiletime_counter.nim rename to tests_disabled/ic/ic_disabled/mcompiletime_counter.nim diff --git a/tests/ic/mdefconverter.nim b/tests_disabled/ic/ic_disabled/mdefconverter.nim similarity index 100% rename from tests/ic/mdefconverter.nim rename to tests_disabled/ic/ic_disabled/mdefconverter.nim diff --git a/tests/ic/mimports.nim b/tests_disabled/ic/ic_disabled/mimports.nim similarity index 100% rename from tests/ic/mimports.nim rename to tests_disabled/ic/ic_disabled/mimports.nim diff --git a/tests/ic/mimportsb.nim b/tests_disabled/ic/ic_disabled/mimportsb.nim similarity index 100% rename from tests/ic/mimportsb.nim rename to tests_disabled/ic/ic_disabled/mimportsb.nim diff --git a/tests/ic/tcompiletime_counter.nim b/tests_disabled/ic/ic_disabled/tcompiletime_counter.nim similarity index 100% rename from tests/ic/tcompiletime_counter.nim rename to tests_disabled/ic/ic_disabled/tcompiletime_counter.nim diff --git a/tests/ic/tconverter.nim b/tests_disabled/ic/ic_disabled/tconverter.nim similarity index 100% rename from tests/ic/tconverter.nim rename to tests_disabled/ic/ic_disabled/tconverter.nim diff --git a/tests/ic/tgenericinst.nim b/tests_disabled/ic/ic_disabled/tgenericinst.nim similarity index 100% rename from tests/ic/tgenericinst.nim rename to tests_disabled/ic/ic_disabled/tgenericinst.nim diff --git a/tests/ic/tgenerics.nim b/tests_disabled/ic/ic_disabled/tgenerics.nim similarity index 100% rename from tests/ic/tgenerics.nim rename to tests_disabled/ic/ic_disabled/tgenerics.nim diff --git a/tests/ic/timports.nim b/tests_disabled/ic/ic_disabled/timports.nim similarity index 100% rename from tests/ic/timports.nim rename to tests_disabled/ic/ic_disabled/timports.nim diff --git a/tests/ic/tmethods.nim b/tests_disabled/ic/ic_disabled/tmethods.nim similarity index 100% rename from tests/ic/tmethods.nim rename to tests_disabled/ic/ic_disabled/tmethods.nim diff --git a/tests/ic/tstdlib_import_changed.nim b/tests_disabled/ic/ic_disabled/tstdlib_import_changed.nim similarity index 100% rename from tests/ic/tstdlib_import_changed.nim rename to tests_disabled/ic/ic_disabled/tstdlib_import_changed.nim From 83d7d8c6342e05ce8cb69fb78f18a6784251556f Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sat, 10 Jan 2026 21:18:39 +0800 Subject: [PATCH 30/31] Add test case for jsffi type mismatch error (#16726) (#25429) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Issue #16726 reported an internal compiler error (`semcall.nim(229, 18) nArg != nil`) when calling `toJs` with invalid arguments. The bug has been fixed in the current codebase but lacked a regression test. ```nim import std/jsffi let a = toJs(3) let b = a.toJs(int) # Now produces: Error: type mismatch # Previously: internal error ``` **Changes:** - Added `tests/js/t16726.nim` to verify proper type mismatch error is reported instead of internal compiler error
Original prompt > > ---- > > *This section details on the original issue you should resolve* > > internal error: semcall.nim(229, 18) nArg != nil > ### Example > the code below should give a clean CT error, not an internal error > ```nim > import std/jsffi > let a = toJs(3) > let b = a.toJs(int) > ``` > > ### Current Output > nim r -b:js main > compiler/semcall.nim(229, 18) `nArg != nil` > > ### Expected Output > proper CT error > > ### Additional Information > 1.5.1 41965880ce095da09a1f7e781a0c79e436432401 > > > adds a test case for it and verify it by `./koch temp js -r test.nim` before committing > > ## Comments on the Issue (you are @copilot in this section) > > > @ringabout > related: https://github.com/nim-lang/Nim/issues/15607 > >
- Fixes nim-lang/Nim#16726 --- 💬 We'd love your input! Share your thoughts on Copilot coding agent in our [2 minute survey](https://gh.io/copilot-coding-agent-survey). --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: ringabout <43030857+ringabout@users.noreply.github.com> --- tests/js/t16726.nim | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tests/js/t16726.nim diff --git a/tests/js/t16726.nim b/tests/js/t16726.nim new file mode 100644 index 0000000000..bd28af4cd0 --- /dev/null +++ b/tests/js/t16726.nim @@ -0,0 +1,9 @@ +discard """ + errormsg: "type mismatch" +""" + +# issue #16726 +# the code below should give a clean CT error, not an internal error +import std/jsffi +let a = toJs(3) +let b = a.toJs(int) From c1e381ae8d02036fa8707e0434338b4cbe29bf21 Mon Sep 17 00:00:00 2001 From: Jake Leahy Date: Sun, 11 Jan 2026 21:39:01 +1100 Subject: [PATCH 31/31] Raw switch for `jsondoc` (#24568) Implements #21928 Adds a `--raw` (since thats what the original issue used, suggestions welcome) switch which stops the jsondoc gen from rendering rst/markdown. Implemented by making `genComment` check if it needs to return the raw string or not. This required switching the related procs to using `Option` to handle how `nil` values were returned before. The `nil` returns were eventually ignored so just ignoring `none(T)` has the same effect. Doesn't support `runnableExamples` since jsondocs doesn't support them either --- changelog.md | 1 + compiler/commands.nim | 3 +++ compiler/docgen.nim | 13 ++++++++++--- compiler/options.nim | 4 ++++ doc/advopt.txt | 1 + tests/misc/mrawjson.nim | 5 +++++ tests/misc/trunner.nim | 12 ++++++++++++ 7 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 tests/misc/mrawjson.nim diff --git a/changelog.md b/changelog.md index 08aafba6b8..c8a9c39c5d 100644 --- a/changelog.md +++ b/changelog.md @@ -112,6 +112,7 @@ errors. ## Tool changes +- Added `--raw` flag when generating JSON docs to not render markup. - Added `--stdinfile` flag to name of the file used when running program from stdin (defaults to `stdinfile.nim`) ## Documentation changes diff --git a/compiler/commands.nim b/compiler/commands.nim index 1bf8ec5505..7de69f8840 100644 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1109,6 +1109,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo; of "shownonexports": expectNoArg(conf, switch, arg, pass, info) showNonExportedFields(conf) + of "raw": + expectNoArg(conf, switch, arg, pass, info) + docRawOutput(conf) of "exceptions": case arg.normalize of "cpp": conf.exc = excCpp diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8167fc4b68..159214e27f 100644 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -433,6 +433,9 @@ proc getVarIdx(varnames: openArray[string], id: string): int = proc genComment(d: PDoc, n: PNode): PRstNode = if n.comment.len > 0: + if optDocRaw in d.conf.globalOptions: + return newRstLeaf(n.comment) + d.sharedState.currFileIdx = addRstFileIndex(d, n.info) try: result = parseRst(n.comment, @@ -1176,8 +1179,12 @@ proc genJsonItem(d: PDoc, n, nameNode: PNode, k: TSymKind, nonExports = false): "col": %n.info.col} ) if comm != nil: - result.rst = comm - result.rstField = "description" + if optDocRaw in d.conf.globalOptions: + result.json["description"] = %comm.text + else: + result.rst = comm + result.rstField = "description" + if r.buf.len > 0: result.json["code"] = %r.buf if k in routineKinds: @@ -1418,7 +1425,7 @@ proc generateDoc*(d: PDoc, n, orig: PNode, config: ConfigRef, docFlags: DocFlags of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept" of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0]) of nkCallKinds: - var comm: ItemPre = default(ItemPre) + var comm = default(ItemPre) getAllRunnableExamples(d, n, comm) if comm.len != 0: d.modDescPre.add(comm) else: discard diff --git a/compiler/options.nim b/compiler/options.nim index 086954563d..3fdc8a99cc 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -110,6 +110,7 @@ type # please make sure we have under 32 options optEnableDeepCopy # ORC specific: enable 'deepcopy' for all types. optShowNonExportedFields # for documentation: show fields that are not exported optJsBigInt64 # use bigints for 64-bit integers in JS + optDocRaw # for documentation: Don't render markdown for JSON output optItaniumMangle # mangling follows the Itanium spec optCompress # turn on AST compression by converting it to NIF optWithinConfigSystem # we still compile within the configuration system @@ -1046,6 +1047,9 @@ proc isDynlibOverride*(conf: ConfigRef; lib: string): bool = proc showNonExportedFields*(conf: ConfigRef) = incl(conf.globalOptions, optShowNonExportedFields) +proc docRawOutput*(conf: ConfigRef) = + incl(conf.globalOptions, optDocRaw) + proc expandDone*(conf: ConfigRef): bool = result = conf.ideCmd == ideExpand and conf.expandLevels == 0 and conf.expandProgress diff --git a/doc/advopt.txt b/doc/advopt.txt index 4f0c664acf..5b822e07fa 100644 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -115,6 +115,7 @@ Advanced options: --docSeeSrcUrl:url activate 'see source' for doc command (see doc.item.seesrc in config/nimdoc.cfg) --docInternal also generate documentation for non-exported symbols + --raw turn off markup rendering for JSON docs --lineDir:on|off generation of #line directive on|off --embedsrc:on|off embeds the original source code as comments in the generated output diff --git a/tests/misc/mrawjson.nim b/tests/misc/mrawjson.nim new file mode 100644 index 0000000000..d824a43a83 --- /dev/null +++ b/tests/misc/mrawjson.nim @@ -0,0 +1,5 @@ +## Module description. See [someProc] +## another line + +proc someProc*(a, b: int) = + ## Code should be used like `someProc(1, 2)` diff --git a/tests/misc/trunner.nim b/tests/misc/trunner.nim index 6e5487d1b7..ac13bc5723 100644 --- a/tests/misc/trunner.nim +++ b/tests/misc/trunner.nim @@ -251,6 +251,18 @@ sub/mmain.idx""", context doAssert doSomething["col"].getInt == 0 doAssert doSomething["code"].getStr == "proc doSomething(x, y: int): int {.raises: [], tags: [], forbids: [].}" + block: # nim jsondoc --raw switch + let file = testsDir / "misc/mrawjson.nim" + let output = "nimcache_tjsondoc.json" + defer: removeFile(output) + let (msg, exitCode) = execCmdEx(fmt"{nim} jsondoc --raw -o:{output} {file}") + doAssert exitCode == 0, msg + + let data = parseFile(output) + doAssert data["moduleDescription"].getStr == "Module description. See [someProc]\nanother line" + let someProc = data["entries"][0] + doAssert someProc["description"].getStr == "Code should be used like `someProc(1, 2)`" + block: # further issues with `--backend` let file = testsDir / "misc/mbackend.nim" var cmd = fmt"{nim} doc -b:cpp --hints:off --nimcache:{nimcache} {file}"