progress

closes #25294

changelog.md

@@ -81,6 +81,9 @@ parameter and result types, not just their source-level shape. Use
 - `std/re` and `std/nre` are deprecated as PCRE library is obsolete.
   Use https://github.com/nitely/nim-regex or `std/nre2`.
   See: https://github.com/nim-lang/Nim/issues/23668.
+- `std/pegs` now correctly lexes UTF-8 bytes inside bare identifier-style
+  terminals, so case-insensitive matching of non-ASCII terms (e.g. ``\i café``)
+  works without single-quoting.
 
 ## Language changes
 

compiler/ccgexprs.nim

@@ -2816,9 +2816,9 @@ proc genWasMoved(p: BProc; n: PNode) =
     #  [addrLoc(p.config, a), getTypeDesc(p.module, a.t)])
 
 proc genMove(p: BProc; n: PNode; d: var TLoc) =
-  var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
   if n.len == 4:
     # generated by liftdestructors:
+    var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
     var src: TLoc = initLocExpr(p, n[2])
     let destVal = rdLoc(a)
     let srcVal = rdLoc(src)
@@ -2838,29 +2838,16 @@ proc genMove(p: BProc; n: PNode; d: var TLoc) =
   else:
     if d.k == locNone: d = getTemp(p, n.typ)
     if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}:
-      genAssignment(p, d, a, {})
       var op = getAttachedOp(p.module.g.graph, n.typ, attachedWasMoved)
-      if op == nil:
+      if op == nil or sfOverridden notin op.flags:
+        var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
+        genAssignment(p, d, a, {})
         resetLoc(p, a)
       else:
-        var b = initLocExpr(p, newSymNode(op))
-        case skipTypes(a.t, abstractVar+{tyStatic}).kind
-        of tyOpenArray, tyVarargs: # todo fixme generated `wasMoved` hooks for
-                                   # openarrays, but it probably shouldn't?
-          let ra = rdLoc(a)
-          var s: string
-          if reifiedOpenArray(a.lode):
-            if a.t.kind in {tyVar, tyLent}:
-              s = derefField(ra, "Field0") & cArgumentSeparator & derefField(ra, "Field1")
-            else:
-              s = dotField(ra, "Field0") & cArgumentSeparator & dotField(ra, "Field1")
-          else:
-            s = ra & cArgumentSeparator & ra & "Len_0"
-          p.s(cpsStmts).addCallStmt(rdLoc(b), s)
-        else:
-          let val = if p.module.compileToCpp: rdLoc(a) else: byRefLoc(p, a)
-          p.s(cpsStmts).addCallStmt(rdLoc(b), val)
+        n[1] = makeAddr(n[1], p.module.idgen)
+        genCall(p, n, d)
     else:
+      var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
       genAssignment(p, d, a, {})
       resetLoc(p, a)
 

compiler/ccgstmts.nim

@@ -341,9 +341,9 @@ proc genCppParamsForCtor(p: BProc; call: PNode; didGenTemp: var bool): Snippet =
           call[i][0]
         else:
           call[i]
-      if param.kind != nkBracketExpr or param.typ.kind in
+      if not param.typ.isCompileTimeOnly and (param.kind != nkBracketExpr or param.typ.kind in
         {tyRef, tyPtr, tyUncheckedArray, tyArray, tyOpenArray,
-          tyVarargs, tySequence, tyString, tyCstring, tyTuple}:
+          tyVarargs, tySequence, tyString, tyCstring, tyTuple}):
         let tempLoc = initLocExprSingleUse(p, param)
         didGenTemp = didGenTemp or tempLoc.k == locTemp
       genOtherArg(p, call, i, typ, res, argBuilder)

compiler/lambdalifting.nim

@@ -520,6 +520,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
       detectCapturedVars(n[namePos], owner, c)
+  of nkClosure:
+    detectCapturedVars(n[1], owner, c)
   of nkReturnStmt:
     detectCapturedVars(n[0], owner, c)
   of nkIdentDefs:
@@ -769,6 +771,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
         let oldInContainer = c.inContainer
         c.inContainer = 0
         var body = transformBody(d.graph, d.idgen, s, {})
+        if not d.processed.containsOrIncl(s.id):
+          detectCapturedVars(body, s, d)
         body = liftCapturedVars(body, s, d, c)
         if c.envVars.getOrDefault(s.id).isNil:
           s.transformedBody = body

compiler/sem.nim

@@ -454,26 +454,6 @@ proc resetSemFlag(n: PNode) =
     for i in 0..<n.safeLen:
       resetSemFlag(n[i])
 
-proc normalizeTypedescMacroResult(c: PContext, n: PNode): PNode =
-  result = n
-  if result.kind == nkStmtList:
-    result.transitionSonsKind(nkStmtListType)
-
-  const maxTypedescMacroNormalizationPasses = 32
-  # Resolve surviving compile-time branches so later passes don't walk
-  # unevaluated type AST for a typedesc expression.
-  for _ in 0..<maxTypedescMacroNormalizationPasses:
-    if result.kind == nkWhenStmt:
-      result = semWhen(c, result, false)
-      if result.kind == nkStmtList:
-        result.transitionSonsKind(nkStmtListType)
-    elif result.kind == nkStmtListType and result.len > 0 and result[^1].kind == nkWhenStmt:
-      result[^1] = semWhen(c, result[^1], false)
-      if result[^1].kind == nkStmtList:
-        result[^1].transitionSonsKind(nkStmtListType)
-    else:
-      break
-
 proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
                        s: PSym, flags: TExprFlags; expectedType: PType = nil): PNode =
   ## Semantically check the output of a macro.
@@ -504,7 +484,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
       # More restrictive version.
       result = semExprWithType(c, result, flags, expectedType)
     of tyTypeDesc:
-      result = normalizeTypedescMacroResult(c, result)
+      if result.kind == nkStmtList: result.transitionSonsKind(nkStmtListType)
       var typ = semTypeNode(c, result, nil)
       if typ == nil:
         localError(c.config, result.info, "expression has no type: " &
@@ -512,7 +492,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
         result = newSymNode(errorSym(c, result))
       else:
         result.typ = makeTypeDesc(c, typ)
-        #result = symNodeFromType(c, typ, n.info)
+      #result = symNodeFromType(c, typ, n.info)
     else:
       if s.ast[genericParamsPos] != nil and retType.isMetaType:
         # The return type may depend on the Macro arguments

compiler/spawn.nim

@@ -10,7 +10,7 @@
 ## This module implements threadpool's ``spawn``.
 
 import ast, types, idents, magicsys, msgs, options, modulegraphs,
-  lowerings, liftdestructors, renderer
+  lowerings, liftdestructors, renderer, trees
 from trees import getMagic, getRoot
 
 proc callProc(a: PNode): PNode =
@@ -53,6 +53,24 @@ proc typeNeedsNoDeepCopy(t: PType): bool =
   if t.kind in {tyVar, tyLent, tySequence}: t = t.elementType
   result = not containsGarbageCollectedRef(t)
 
+proc newSpawnMoveStmt(g: ModuleGraph; idgen: IdGenerator; le, ri: PNode): PNode =
+  let op = getAttachedOp(g, ri.typ.skipTypes({tyGenericInst, tyAlias, tyVar, tySink}), attachedWasMoved)
+  if op != nil and sfOverridden in op.flags:
+    result = newNodeI(nkStmtList, le.info)
+    result.add newFastAsgnStmt(le, ri)
+
+    let wasMovedCall = newNodeI(nkCall, ri.info)
+    wasMovedCall.add newSymNode(op)
+    
+    if op.typ != nil and op.typ.signatureLen > 1 and op.typ.firstParamType.kind != tyVar:
+      wasMovedCall.add ri.skipAddr
+    else:
+      wasMovedCall.add makeAddr(ri.skipAddr, idgen)
+    
+    result.add wasMovedCall
+  else:
+    result = newFastMoveStmt(g, le, ri)
+
 proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; idgen: IdGenerator; owner: PSym; typ: PType;
                  v: PNode; useShallowCopy=false): PSym =
   result = newSym(skTemp, getIdent(g.cache, genPrefix), idgen, owner, varSection.info,
@@ -68,10 +86,10 @@ proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; idgen: IdGenerator;
   if varInit != nil:
     if g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc, gcYrc}:
       # inject destructors pass will do its own analysis
-      varInit.add newFastMoveStmt(g, newSymNode(result), v)
+      varInit.add newSpawnMoveStmt(g, idgen, newSymNode(result), v)
     else:
       if useShallowCopy and typeNeedsNoDeepCopy(typ) or optTinyRtti in g.config.globalOptions:
-        varInit.add newFastMoveStmt(g, newSymNode(result), v)
+        varInit.add newSpawnMoveStmt(g, idgen, newSymNode(result), v)
       else:
         let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
         deepCopyCall[0] = newSymNode(getSysMagic(g, varSection.info, "deepCopy", mDeepCopy))

lib/pure/collections/critbits.nim

@@ -495,13 +495,16 @@ func `$`*[T](c: CritBitTree[T]): string =
       const avgItemLen = 16
     result = newStringOfCap(c.count * avgItemLen)
     result.add("{")
+    var first = true
     when T is void:
       for key in keys(c):
-        if result.len > 1: result.add(", ")
+        if first: first = false
+        else: result.add(", ")
         result.addQuoted(key)
     else:
       for key, val in pairs(c):
-        if result.len > 1: result.add(", ")
+        if first: first = false
+        else: result.add(", ")
         result.addQuoted(key)
         result.add(": ")
         result.addQuoted(val)

lib/pure/collections/deques.nim

@@ -454,8 +454,10 @@ proc `$`*[T](deq: Deque[T]): string =
     assert $a == "[10, 20, 30]"
 
   result = "["
+  var first = true
   for x in deq:
-    if result.len > 1: result.add(", ")
+    if first: first = false
+    else: result.add(", ")
     result.addQuoted(x)
   result.add("]")
 

lib/pure/collections/heapqueue.nim

@@ -260,7 +260,9 @@ proc `$`*[T](heap: HeapQueue[T]): string =
     assert $heap == "[1, 2]"
 
   result = "["
+  var first = true
   for x in heap.data:
-    if result.len > 1: result.add(", ")
+    if first: first = false
+    else: result.add(", ")
     result.addQuoted(x)
   result.add("]")

lib/pure/collections/lists.nim

@@ -304,8 +304,10 @@ proc `$`*[T](L: SomeLinkedCollection[T]): string =
     assert $a == "[1, 2, 3, 4]"
 
   result = "["
+  var first = true
   for x in nodes(L):
-    if result.len > 1: result.add(", ")
+    if first: first = false
+    else: result.add(", ")
     result.addQuoted(x.value)
   result.add("]")
 

lib/pure/pegs.nim

@@ -1668,7 +1668,10 @@ func getSymbol(c: var PegLexer, tok: var Token) =
   while pos < c.buf.len:
     add(tok.literal, c.buf[pos])
     inc(pos)
-    if pos < c.buf.len and c.buf[pos] notin strutils.IdentChars: break
+    if pos < c.buf.len:
+      let ch = c.buf[pos]
+      # Keep non-ASCII bytes so UTF-8 terminals reach the rune-aware matchers.
+      if ch notin strutils.IdentChars and ord(ch) < 0x80: break
   c.bufpos = pos
   tok.kind = tkIdentifier
 

lib/pure/strtabs.nim

@@ -380,8 +380,10 @@ proc `$`*(t: StringTableRef): string {.rtlFunc, extern: "nstDollar".} =
     result = "{:}"
   else:
     result = "{"
+    var first = true
     for key, val in pairs(t):
-      if result.len > 1: result.add(", ")
+      if first: first = false
+      else: result.add(", ")
       result.add(key)
       result.add(": ")
       result.add(val)

lib/system.nim

@@ -166,7 +166,7 @@ proc wasMoved*[T](obj: var T) {.magic: "WasMoved", noSideEffect.}
   ## it was "moved" and to signify its destructor should do nothing and
   ## ideally be optimized away.
 
-proc move*[T](x: var T): T {.magic: "Move", noSideEffect.} =
+proc move*[T](x: var T): T {.magic: "Move", noSideEffect, nodestroy.} =
   result = x
   {.cast(raises: []), cast(tags: []).}:
     `=wasMoved`(x)

tests/ccgbugs2/m25294/c.nim

@@ -0,0 +1,13 @@
+template a(T: type): int =
+  when T is uint64: 1 else: 2
+
+type
+  M*[T] = object
+    data*: seq[T]
+    b: seq[int]
+    indices*: array[a(T), int64]
+  U = distinct uint64
+  D* = object
+    c: M[U]
+    v: array[180000, int64]
+    g*: M[uint64]

tests/ccgbugs2/m25294/t.nim

@@ -0,0 +1,5 @@
+import ./c
+
+proc p*(): D =
+  let c = M[uint64](data: @[0], indices: [1])
+  result = D(g: c)

tests/ccgbugs2/m25800.h

@@ -0,0 +1,7 @@
+/*TYPESECTION*/
+struct CppRef {
+  int* data;
+  CppRef() : data(new int(42)) {}
+  ~CppRef() { delete data; data = nullptr; }
+  void reset() { delete data; data = nullptr; }
+};

tests/ccgbugs2/t25294.nim

@@ -0,0 +1,18 @@
+discard """
+  matrix: "--mm:refc; --mm:orc"
+"""
+
+import ./m25294/[c, t]
+
+block:
+  let a = new D
+  a[] = p()
+  discard a[]
+block:
+  let a = new D
+  a[] = p()
+  discard a[]
+block:
+  let a = new D
+  a[] = p()
+  discard a[]

tests/ccgbugs2/t25800.nim

@@ -0,0 +1,23 @@
+discard """
+  cmd: "nim cpp $file"
+  action: "compile"
+"""
+
+# Bug Report 1: {.importcpp.} on =wasMoved generates invalid preprocessor directive #.
+
+
+type CppRef* {.importcpp, bycopy, noInit, header: "m25800.h".} = object
+
+proc `=destroy`(x: var CppRef) {.importcpp: "#.~CppRef()".}
+proc `=wasMoved`(x: var CppRef) {.importcpp: "#.reset()".}
+proc `=copy`(dest: var CppRef; src: CppRef) {.importcpp: "dest = src".}
+proc `=sink`(dest: var CppRef; src: CppRef) {.importcpp: "dest = std::move(src)".}
+
+# This triggers =wasMoved when passing to sink parameter
+proc consume(x: sink CppRef) = discard
+
+proc test() =
+  var x: CppRef
+  consume(move(x))  # =wasMoved MUST be called here after the move
+
+test()

tests/destructor/twasmoved.nim

@@ -12,3 +12,51 @@ proc foo =
   doAssert m.id == 999
 
 foo()
+
+block:
+  type Foo = object
+    a,b,c: int
+
+  var dest: Foo
+
+  # proc `=wasMoved`(x: var Foo) =
+  #   debugEcho "wasMoved called"
+
+  proc main() =
+    var x = Foo(a:11, b:12, c:13)
+    dest = move(x)
+
+  main()
+
+block:
+  type Foo = object
+    a,b,c: int
+
+  var dest: Foo
+
+  proc `=wasMoved`(x: var Foo) =
+    discard "wasMoved called"
+
+  proc main() =
+    var x = Foo(a:11, b:12, c:13)
+    dest = move(x)
+
+  main()
+
+
+import std/threadpool
+
+block:
+  type Foo = object
+    data: string
+
+  proc `=wasMoved`(x: var Foo) =
+    discard
+
+  proc work(x: Foo) =
+    discard
+
+  var x = Foo(data: "hello")
+  spawn work(x)
+  sync()
+

tests/iter/titer_issues.nim

@@ -239,6 +239,17 @@ block t2023_objiter:
   var o = init()
   echo(o.iter())
 
+  block: # bug #25591
+    iterator h(): int =
+      let n = 0
+      (proc() = discard n)()
+      yield 0
+
+    proc a() =
+      iterator m(): int {.closure.} = (for _ in h(): discard)
+      let _ = m
+
+    a()
 
 block:
   # bug #13739

tests/stdlib/tdeques.nim

@@ -241,3 +241,12 @@ proc main() =
 
 static: main()
 main()
+
+# https://github.com/nim-lang/Nim/issues/18583
+# $ separator must be emitted even when the item's string repr is empty
+type EmptyStr18583 = object
+proc `$`(x: EmptyStr18583): string = ""
+
+block:
+  var d = [EmptyStr18583(), EmptyStr18583()].toDeque
+  doAssert $d == "[, ]", "got: " & $d

tests/stdlib/theapqueue.nim

@@ -104,3 +104,15 @@ template main() =
 
 static: main()
 main()
+
+# https://github.com/nim-lang/Nim/issues/18583
+type EmptyStr18583HeapQ = object
+proc `$`(x: EmptyStr18583HeapQ): string = ""
+proc `<`(a, b: EmptyStr18583HeapQ): bool = false
+
+block:
+  var h = initHeapQueue[EmptyStr18583HeapQ]()
+  push(h, EmptyStr18583HeapQ())
+  push(h, EmptyStr18583HeapQ())
+  let s = $h
+  doAssert s == "[, ]", "got: " & s

tests/stdlib/tlists.nim

@@ -287,3 +287,14 @@ template main =
 
 static: main()
 main()
+
+# https://github.com/nim-lang/Nim/issues/18583
+type EmptyStr18583List = object
+proc `$`(x: EmptyStr18583List): string = ""
+
+block:
+  var L: SinglyLinkedList[EmptyStr18583List]
+  L.prepend(EmptyStr18583List())
+  L.prepend(EmptyStr18583List())
+  let s = $L
+  doAssert s == "[, ]", "got: " & s

tests/stdlib/tpegs.nim

@@ -259,6 +259,11 @@ block:
     doAssert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+")
     doAssert(not match("456678", peg"(\letter)+"))
 
+    block:
+      doAssert match("CAFÉ", peg"\i café")
+      doAssert match("Café", peg"\i café")
+      doAssert "two cafés: Café and CAFÉ".findAll(peg"\i café").len == 3
+
     doAssert("var1 = key; var2 = key2".replacef(
       peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") ==
            "var1<-keykey;var2<-key2key2")

tests/whenstmt/twhen.nim

@@ -45,12 +45,3 @@ elif compiles(nonexistent):
 else:
   output("whenElse")
 
-
-template test(): typedesc =
-  when true:
-    int
-  else:
-    bool
-
-const c = default(test())
-echo c