progress

closes #25294

changelog.md

@@ -81,6 +81,9 @@ parameter and result types, not just their source-level shape. Use
 - `std/re` and `std/nre` are deprecated as PCRE library is obsolete.
   Use https://github.com/nitely/nim-regex or `std/nre2`.
   See: https://github.com/nim-lang/Nim/issues/23668.
+- `std/pegs` now correctly lexes UTF-8 bytes inside bare identifier-style
+  terminals, so case-insensitive matching of non-ASCII terms (e.g. ``\i café``)
+  works without single-quoting.
 
 ## Language changes
 

compiler/ccgexprs.nim

@@ -2816,9 +2816,9 @@ proc genWasMoved(p: BProc; n: PNode) =
     #  [addrLoc(p.config, a), getTypeDesc(p.module, a.t)])
 
 proc genMove(p: BProc; n: PNode; d: var TLoc) =
-  var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
   if n.len == 4:
     # generated by liftdestructors:
+    var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
     var src: TLoc = initLocExpr(p, n[2])
     let destVal = rdLoc(a)
     let srcVal = rdLoc(src)
@@ -2838,29 +2838,16 @@ proc genMove(p: BProc; n: PNode; d: var TLoc) =
   else:
     if d.k == locNone: d = getTemp(p, n.typ)
     if p.config.selectedGC in {gcArc, gcAtomicArc, gcOrc, gcYrc}:
-      genAssignment(p, d, a, {})
       var op = getAttachedOp(p.module.g.graph, n.typ, attachedWasMoved)
-      if op == nil:
+      if op == nil or sfOverridden notin op.flags:
+        var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
+        genAssignment(p, d, a, {})
         resetLoc(p, a)
       else:
-        var b = initLocExpr(p, newSymNode(op))
-        case skipTypes(a.t, abstractVar+{tyStatic}).kind
-        of tyOpenArray, tyVarargs: # todo fixme generated `wasMoved` hooks for
-                                   # openarrays, but it probably shouldn't?
-          let ra = rdLoc(a)
-          var s: string
-          if reifiedOpenArray(a.lode):
-            if a.t.kind in {tyVar, tyLent}:
-              s = derefField(ra, "Field0") & cArgumentSeparator & derefField(ra, "Field1")
-            else:
-              s = dotField(ra, "Field0") & cArgumentSeparator & dotField(ra, "Field1")
-          else:
-            s = ra & cArgumentSeparator & ra & "Len_0"
-          p.s(cpsStmts).addCallStmt(rdLoc(b), s)
-        else:
-          let val = if p.module.compileToCpp: rdLoc(a) else: byRefLoc(p, a)
-          p.s(cpsStmts).addCallStmt(rdLoc(b), val)
+        n[1] = makeAddr(n[1], p.module.idgen)
+        genCall(p, n, d)
     else:
+      var a: TLoc = initLocExpr(p, n[1].skipAddr, {lfEnforceDeref, lfPrepareForMutation})
       genAssignment(p, d, a, {})
       resetLoc(p, a)
 

compiler/lambdalifting.nim

@@ -520,6 +520,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
       detectCapturedVars(n[namePos], owner, c)
+  of nkClosure:
+    detectCapturedVars(n[1], owner, c)
   of nkReturnStmt:
     detectCapturedVars(n[0], owner, c)
   of nkIdentDefs:
@@ -769,6 +771,8 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: var DetectionPass;
         let oldInContainer = c.inContainer
         c.inContainer = 0
         var body = transformBody(d.graph, d.idgen, s, {})
+        if not d.processed.containsOrIncl(s.id):
+          detectCapturedVars(body, s, d)
         body = liftCapturedVars(body, s, d, c)
         if c.envVars.getOrDefault(s.id).isNil:
           s.transformedBody = body

compiler/spawn.nim

@@ -10,7 +10,7 @@
 ## This module implements threadpool's ``spawn``.
 
 import ast, types, idents, magicsys, msgs, options, modulegraphs,
-  lowerings, liftdestructors, renderer
+  lowerings, liftdestructors, renderer, trees
 from trees import getMagic, getRoot
 
 proc callProc(a: PNode): PNode =
@@ -53,6 +53,24 @@ proc typeNeedsNoDeepCopy(t: PType): bool =
   if t.kind in {tyVar, tyLent, tySequence}: t = t.elementType
   result = not containsGarbageCollectedRef(t)
 
+proc newSpawnMoveStmt(g: ModuleGraph; idgen: IdGenerator; le, ri: PNode): PNode =
+  let op = getAttachedOp(g, ri.typ.skipTypes({tyGenericInst, tyAlias, tyVar, tySink}), attachedWasMoved)
+  if op != nil and sfOverridden in op.flags:
+    result = newNodeI(nkStmtList, le.info)
+    result.add newFastAsgnStmt(le, ri)
+
+    let wasMovedCall = newNodeI(nkCall, ri.info)
+    wasMovedCall.add newSymNode(op)
+    
+    if op.typ != nil and op.typ.signatureLen > 1 and op.typ.firstParamType.kind != tyVar:
+      wasMovedCall.add ri.skipAddr
+    else:
+      wasMovedCall.add makeAddr(ri.skipAddr, idgen)
+    
+    result.add wasMovedCall
+  else:
+    result = newFastMoveStmt(g, le, ri)
+
 proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; idgen: IdGenerator; owner: PSym; typ: PType;
                  v: PNode; useShallowCopy=false): PSym =
   result = newSym(skTemp, getIdent(g.cache, genPrefix), idgen, owner, varSection.info,
@@ -68,10 +86,10 @@ proc addLocalVar(g: ModuleGraph; varSection, varInit: PNode; idgen: IdGenerator;
   if varInit != nil:
     if g.config.selectedGC in {gcArc, gcOrc, gcAtomicArc, gcYrc}:
       # inject destructors pass will do its own analysis
-      varInit.add newFastMoveStmt(g, newSymNode(result), v)
+      varInit.add newSpawnMoveStmt(g, idgen, newSymNode(result), v)
     else:
       if useShallowCopy and typeNeedsNoDeepCopy(typ) or optTinyRtti in g.config.globalOptions:
-        varInit.add newFastMoveStmt(g, newSymNode(result), v)
+        varInit.add newSpawnMoveStmt(g, idgen, newSymNode(result), v)
       else:
         let deepCopyCall = newNodeI(nkCall, varInit.info, 3)
         deepCopyCall[0] = newSymNode(getSysMagic(g, varSection.info, "deepCopy", mDeepCopy))

compiler/transf.nim

@@ -90,20 +90,11 @@ proc getCurrOwner(c: PTransf): PSym =
   if c.transCon != nil: result = c.transCon.owner
   else: result = c.module
 
-proc freshOwnedSym(c: PTransf; s, owner: PSym): PNode =
-  # We need to copy the symbol here because we might need to change its owner and
-  # we don't want to mess with the original symbol which might be used in other places.
-  # This can happen for example for iterators which are transformed multiple times when
-  # they are used in different contexts.
-  var fresh = copySym(s, c.idgen)
-  incl(fresh.flagsImpl, sfFromGeneric)
-  setOwner(fresh, owner)
-  result = newSymNode(fresh)
-
 proc newTemp(c: PTransf, typ: PType, info: TLineInfo): PNode =
   let r = newSym(skTemp, getIdent(c.graph.cache, genPrefix), c.idgen, getCurrOwner(c), info)
   r.typ = typ #skipTypes(typ, {tyGenericInst, tyAlias, tySink})
   incl(r.flagsImpl, sfFromGeneric)
+  let owner = getCurrOwner(c)
   result = newSymNode(r)
 
 proc transform(c: PTransf, n: PNode, noConstFold = false): PNode
@@ -194,39 +185,11 @@ proc transformSym(c: PTransf, n: PNode): PNode =
   result = transformSymAux(c, n)
 
 proc freshVar(c: PTransf; v: PSym): PNode =
-  result = freshOwnedSym(c, v, getCurrOwner(c))
-
-proc introduceNewRoutineHeaderSyms(c: PTransf; n: PNode; oldOwner, newOwner: PSym) =
-  # We need to introduce new symbols for the parameters and result of a routine when
-  # we copy it for inlining or closure generation.
-  # Otherwise, we would have multiple nodes referring to the same parameter symbols which
-  # can lead to problems when we need to change the owner of these symbols.
-  case n.kind
-  of nkSym:
-    if n.sym.owner == oldOwner:
-      c.transCon.mapping[n.sym.itemId] = freshOwnedSym(c, n.sym, newOwner)
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit:
-    discard
-  else:
-    for i in 0..<n.len:
-      introduceNewRoutineHeaderSyms(c, n[i], oldOwner, newOwner)
-
-proc copyRoutineTypeHeader(c: PTransf; oldProc, newProc: PSym) =
-  # We need to copy the routine type header to ensure that
-  # modifications to the newProc do not affect the oldProc.
-  if oldProc.typ != nil and oldProc.typ.kind == tyProc and oldProc.typ.n != nil:
-    newProc.typ = copyType(oldProc.typ, c.idgen, newProc)
-    newProc.typ.n = newNodeI(oldProc.typ.n.kind, oldProc.typ.n.info)
-    if oldProc.typ.n.len > 0:
-      newProc.typ.n.add copyNode(oldProc.typ.n[0])
-      for i in 1..<oldProc.typ.n.len:
-        let oldParam = oldProc.typ.n[i].sym
-        var newParam = getOrDefault(c.transCon.mapping, oldParam.itemId)
-        if newParam == nil:
-          newParam = freshOwnedSym(c, oldParam, newProc)
-          c.transCon.mapping[oldParam.itemId] = newParam
-        doAssert newParam.kind == nkSym
-        newProc.typ.addParam newParam.sym
+  let owner = getCurrOwner(c)
+  var newVar = copySym(v, c.idgen)
+  incl(newVar.flagsImpl, sfFromGeneric)
+  setOwner(newVar, owner)
+  result = newSymNode(newVar)
 
 proc transformVarSection(c: PTransf, v: PNode): PNode =
   result = newTransNode(v)
@@ -375,13 +338,8 @@ proc introduceNewLocalVars(c: PTransf, n: PNode): PNode =
     return n
   of nkLambdaKinds, nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef: # todo optimize nosideeffects?
     result = newTransNode(n)
-    let oldProc = n[namePos].sym
-    let x = freshOwnedSym(c, oldProc, oldProc.owner)
-    c.transCon.mapping[oldProc.itemId] = x
-    introduceNewRoutineHeaderSyms(c, n[paramsPos], oldProc, x.sym)
-    if resultPos < n.len and n[resultPos] != nil:
-      introduceNewRoutineHeaderSyms(c, n[resultPos], oldProc, x.sym)
-    copyRoutineTypeHeader(c, oldProc, x.sym)
+    let x = newSymNode(copySym(n[namePos].sym, c.idgen))
+    c.transCon.mapping[n[namePos].sym.itemId] = x
     result[namePos] = x # we have to copy proc definitions for iters
     for i in 1..<n.len:
       result[i] = introduceNewLocalVars(c, n[i])

lib/pure/collections/critbits.nim

@@ -495,13 +495,16 @@ func `$`*[T](c: CritBitTree[T]): string =
       const avgItemLen = 16
     result = newStringOfCap(c.count * avgItemLen)
     result.add("{")
+    var first = true
     when T is void:
       for key in keys(c):
-        if result.len > 1: result.add(", ")
+        if first: first = false
+        else: result.add(", ")
         result.addQuoted(key)
     else:
       for key, val in pairs(c):
-        if result.len > 1: result.add(", ")
+        if first: first = false
+        else: result.add(", ")
         result.addQuoted(key)
         result.add(": ")
         result.addQuoted(val)

lib/pure/collections/deques.nim

@@ -454,8 +454,10 @@ proc `$`*[T](deq: Deque[T]): string =
     assert $a == "[10, 20, 30]"
 
   result = "["
+  var first = true
   for x in deq:
-    if result.len > 1: result.add(", ")
+    if first: first = false
+    else: result.add(", ")
     result.addQuoted(x)
   result.add("]")
 

lib/pure/collections/heapqueue.nim

@@ -260,7 +260,9 @@ proc `$`*[T](heap: HeapQueue[T]): string =
     assert $heap == "[1, 2]"
 
   result = "["
+  var first = true
   for x in heap.data:
-    if result.len > 1: result.add(", ")
+    if first: first = false
+    else: result.add(", ")
     result.addQuoted(x)
   result.add("]")

lib/pure/collections/lists.nim

@@ -304,8 +304,10 @@ proc `$`*[T](L: SomeLinkedCollection[T]): string =
     assert $a == "[1, 2, 3, 4]"
 
   result = "["
+  var first = true
   for x in nodes(L):
-    if result.len > 1: result.add(", ")
+    if first: first = false
+    else: result.add(", ")
     result.addQuoted(x.value)
   result.add("]")
 

lib/pure/pegs.nim

@@ -1668,7 +1668,10 @@ func getSymbol(c: var PegLexer, tok: var Token) =
   while pos < c.buf.len:
     add(tok.literal, c.buf[pos])
     inc(pos)
-    if pos < c.buf.len and c.buf[pos] notin strutils.IdentChars: break
+    if pos < c.buf.len:
+      let ch = c.buf[pos]
+      # Keep non-ASCII bytes so UTF-8 terminals reach the rune-aware matchers.
+      if ch notin strutils.IdentChars and ord(ch) < 0x80: break
   c.bufpos = pos
   tok.kind = tkIdentifier
 

lib/pure/strtabs.nim

@@ -380,8 +380,10 @@ proc `$`*(t: StringTableRef): string {.rtlFunc, extern: "nstDollar".} =
     result = "{:}"
   else:
     result = "{"
+    var first = true
     for key, val in pairs(t):
-      if result.len > 1: result.add(", ")
+      if first: first = false
+      else: result.add(", ")
       result.add(key)
       result.add(": ")
       result.add(val)

lib/system.nim

@@ -166,7 +166,7 @@ proc wasMoved*[T](obj: var T) {.magic: "WasMoved", noSideEffect.}
   ## it was "moved" and to signify its destructor should do nothing and
   ## ideally be optimized away.
 
-proc move*[T](x: var T): T {.magic: "Move", noSideEffect.} =
+proc move*[T](x: var T): T {.magic: "Move", noSideEffect, nodestroy.} =
   result = x
   {.cast(raises: []), cast(tags: []).}:
     `=wasMoved`(x)

tests/ccgbugs2/m25294/c.nim

@@ -0,0 +1,13 @@
+template a(T: type): int =
+  when T is uint64: 1 else: 2
+
+type
+  M*[T] = object
+    data*: seq[T]
+    b: seq[int]
+    indices*: array[a(T), int64]
+  U = distinct uint64
+  D* = object
+    c: M[U]
+    v: array[180000, int64]
+    g*: M[uint64]

tests/ccgbugs2/m25294/t.nim

@@ -0,0 +1,5 @@
+import ./c
+
+proc p*(): D =
+  let c = M[uint64](data: @[0], indices: [1])
+  result = D(g: c)

tests/ccgbugs2/m25800.h

@@ -0,0 +1,7 @@
+/*TYPESECTION*/
+struct CppRef {
+  int* data;
+  CppRef() : data(new int(42)) {}
+  ~CppRef() { delete data; data = nullptr; }
+  void reset() { delete data; data = nullptr; }
+};

tests/ccgbugs2/t25294.nim

@@ -0,0 +1,18 @@
+discard """
+  matrix: "--mm:refc; --mm:orc"
+"""
+
+import ./m25294/[c, t]
+
+block:
+  let a = new D
+  a[] = p()
+  discard a[]
+block:
+  let a = new D
+  a[] = p()
+  discard a[]
+block:
+  let a = new D
+  a[] = p()
+  discard a[]

tests/ccgbugs2/t25800.nim

@@ -0,0 +1,23 @@
+discard """
+  cmd: "nim cpp $file"
+  action: "compile"
+"""
+
+# Bug Report 1: {.importcpp.} on =wasMoved generates invalid preprocessor directive #.
+
+
+type CppRef* {.importcpp, bycopy, noInit, header: "m25800.h".} = object
+
+proc `=destroy`(x: var CppRef) {.importcpp: "#.~CppRef()".}
+proc `=wasMoved`(x: var CppRef) {.importcpp: "#.reset()".}
+proc `=copy`(dest: var CppRef; src: CppRef) {.importcpp: "dest = src".}
+proc `=sink`(dest: var CppRef; src: CppRef) {.importcpp: "dest = std::move(src)".}
+
+# This triggers =wasMoved when passing to sink parameter
+proc consume(x: sink CppRef) = discard
+
+proc test() =
+  var x: CppRef
+  consume(move(x))  # =wasMoved MUST be called here after the move
+
+test()

tests/destructor/twasmoved.nim

@@ -12,3 +12,51 @@ proc foo =
   doAssert m.id == 999
 
 foo()
+
+block:
+  type Foo = object
+    a,b,c: int
+
+  var dest: Foo
+
+  # proc `=wasMoved`(x: var Foo) =
+  #   debugEcho "wasMoved called"
+
+  proc main() =
+    var x = Foo(a:11, b:12, c:13)
+    dest = move(x)
+
+  main()
+
+block:
+  type Foo = object
+    a,b,c: int
+
+  var dest: Foo
+
+  proc `=wasMoved`(x: var Foo) =
+    discard "wasMoved called"
+
+  proc main() =
+    var x = Foo(a:11, b:12, c:13)
+    dest = move(x)
+
+  main()
+
+
+import std/threadpool
+
+block:
+  type Foo = object
+    data: string
+
+  proc `=wasMoved`(x: var Foo) =
+    discard
+
+  proc work(x: Foo) =
+    discard
+
+  var x = Foo(data: "hello")
+  spawn work(x)
+  sync()
+

tests/iter/titer_issues.nim

@@ -239,6 +239,17 @@ block t2023_objiter:
   var o = init()
   echo(o.iter())
 
+  block: # bug #25591
+    iterator h(): int =
+      let n = 0
+      (proc() = discard n)()
+      yield 0
+
+    proc a() =
+      iterator m(): int {.closure.} = (for _ in h(): discard)
+      let _ = m
+
+    a()
 
 block:
   # bug #13739
@@ -465,24 +476,4 @@ block: # bug #25724
     else: yield 1
   for w in c():
     let n = w
-    (proc() = discard n)()
-
-block:
-  iterator c(): int =
-    yield 1
-    yield 1
-
-  for w in c():
-    proc p(s: int) =
-      let sap = s
-    p(0)
-
-block: # bug #25725
-  iterator c(): int =
-    when nimvm: yield 0
-    else: yield 1
-  for w in c():
-    let n = w
-    proc p(s: int) =
-      let s = s; discard n
-    p(0)
+    (proc() = discard n)()

tests/stdlib/tdeques.nim

@@ -241,3 +241,12 @@ proc main() =
 
 static: main()
 main()
+
+# https://github.com/nim-lang/Nim/issues/18583
+# $ separator must be emitted even when the item's string repr is empty
+type EmptyStr18583 = object
+proc `$`(x: EmptyStr18583): string = ""
+
+block:
+  var d = [EmptyStr18583(), EmptyStr18583()].toDeque
+  doAssert $d == "[, ]", "got: " & $d

tests/stdlib/theapqueue.nim

@@ -104,3 +104,15 @@ template main() =
 
 static: main()
 main()
+
+# https://github.com/nim-lang/Nim/issues/18583
+type EmptyStr18583HeapQ = object
+proc `$`(x: EmptyStr18583HeapQ): string = ""
+proc `<`(a, b: EmptyStr18583HeapQ): bool = false
+
+block:
+  var h = initHeapQueue[EmptyStr18583HeapQ]()
+  push(h, EmptyStr18583HeapQ())
+  push(h, EmptyStr18583HeapQ())
+  let s = $h
+  doAssert s == "[, ]", "got: " & s

tests/stdlib/tlists.nim

@@ -287,3 +287,14 @@ template main =
 
 static: main()
 main()
+
+# https://github.com/nim-lang/Nim/issues/18583
+type EmptyStr18583List = object
+proc `$`(x: EmptyStr18583List): string = ""
+
+block:
+  var L: SinglyLinkedList[EmptyStr18583List]
+  L.prepend(EmptyStr18583List())
+  L.prepend(EmptyStr18583List())
+  let s = $L
+  doAssert s == "[, ]", "got: " & s

tests/stdlib/tpegs.nim

@@ -259,6 +259,11 @@ block:
     doAssert match("EINE ÜBERSICHT UND AUSSERDEM", peg"(\upper \white*)+")
     doAssert(not match("456678", peg"(\letter)+"))
 
+    block:
+      doAssert match("CAFÉ", peg"\i café")
+      doAssert match("Café", peg"\i café")
+      doAssert "two cafés: Café and CAFÉ".findAll(peg"\i café").len == 3
+
     doAssert("var1 = key; var2 = key2".replacef(
       peg"\skip(\s*) {\ident}'='{\ident}", "$1<-$2$2") ==
            "var1<-keykey;var2<-key2key2")