allow replacing captured syms in macro calls in generics (#23091)

fixes #22605, separated from #22744

This marks symbol captures in macro calls in generic contexts as
`nfOpenSym`, which means if there is a new symbol in the local
instantiatied body during instantiation time, this symbol replaces the
captured symbol. We have to be careful not to consider symbols outside
of the instantiation body during instantiation, because this will leak
symbols from the instantiation context scope rather than the original
declaration scope. This is done by checking if the local context owner
(maybe should be the symbol of the proc currently getting instantiated
instead? not sure how to get this) is the same as or a parent owner of
the owner of the replacement candidate symbol.

This solution is distinct from the symchoice mechanisms which we
originally assumed had to be related, if this assumption was wrong it
would explain why this solution took so long to arrive at.

compiler/ast.nim

@@ -520,6 +520,7 @@ type
     nfFirstWrite # this node is a first write
     nfHasComment # node has a comment
     nfSkipFieldChecking # node skips field visable checking
+    nfOpenSym # node is a captured sym but can be overriden by local symbols
 
   TNodeFlags* = set[TNodeFlag]
   TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 47)
@@ -1095,7 +1096,8 @@ const
                                       nfIsRef, nfIsPtr, nfPreventCg, nfLL,
                                       nfFromTemplate, nfDefaultRefsParam,
                                       nfExecuteOnReload, nfLastRead,
-                                      nfFirstWrite, nfSkipFieldChecking}
+                                      nfFirstWrite, nfSkipFieldChecking,
+                                      nfOpenSym}
   namePos* = 0
   patternPos* = 1    # empty except for term rewriting macros
   genericParamsPos* = 2

compiler/ic/ic.nim

@@ -433,11 +433,11 @@ proc addModuleRef(n: PNode; ir: var PackedTree; c: var PackedEncoder; m: var Pac
   let info = n.info.toPackedInfo(c, m)
   if n.typ != n.sym.typ:
     ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total
-               info = info,
+               info = info, flags = n.flags,
                typeId = storeTypeLater(n.typ, c, m))
   else:
     ir.addNode(kind = nkModuleRef, operand = 3.int32, # spans 3 nodes in total
-              info = info)
+              info = info, flags = n.flags)
   ir.addNode(kind = nkNone, info = info,
              operand = toLitId(n.sym.itemId.module.FileIndex, c, m).int32)
   ir.addNode(kind = nkNone, info = info,
@@ -829,7 +829,8 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
     result.ident = getIdent(c.cache, g[thisModule].fromDisk.strings[n.litId])
   of nkSym:
     result.sym = loadSym(c, g, thisModule, PackedItemId(module: LitId(0), item: tree[n].soperand))
-    if result.typ == nil: result.typ = result.sym.typ
+    if result.typ == nil and nfOpenSym notin result.flags:
+      result.typ = result.sym.typ
   of externIntLit:
     result.intVal = g[thisModule].fromDisk.numbers[n.litId]
   of nkStrLit..nkTripleStrLit:
@@ -842,7 +843,8 @@ proc loadNodes*(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int;
     assert n2.kind == nkNone
     transitionNoneToSym(result)
     result.sym = loadSym(c, g, thisModule, PackedItemId(module: n1.litId, item: tree[n2].soperand))
-    if result.typ == nil: result.typ = result.sym.typ
+    if result.typ == nil and nfOpenSym notin result.flags:
+      result.typ = result.sym.typ
   else:
     for n0 in sonsReadonly(tree, n):
       result.addAllowNil loadNodes(c, g, thisModule, tree, n0)

compiler/semexprs.nim

@@ -1024,7 +1024,10 @@ proc afterCallActions(c: PContext; n, orig: PNode, flags: TExprFlags; expectedTy
   of skMacro: result = semMacroExpr(c, result, orig, callee, flags, expectedType)
   of skTemplate: result = semTemplateExpr(c, result, callee, flags, expectedType)
   else:
-    semFinishOperands(c, result)
+    if callee.magic notin {mArrGet, mArrPut, mNBindSym}:
+      # calls to `[]` can be explicit generic instantiations,
+      # don't sem every operand now, leave it to semmagic
+      semFinishOperands(c, result)
     activate(c, result)
     fixAbstractType(c, result)
     analyseIfAddressTakenInCall(c, result)
@@ -3063,9 +3066,22 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}, expectedType: PType
   of nkClosedSymChoice, nkOpenSymChoice:
     result = semSymChoice(c, result, flags, expectedType)
   of nkSym:
+    let s = n.sym
+    if nfOpenSym in n.flags:
+      let id = newIdentNode(s.name, n.info)
+      c.isAmbiguous = false
+      let s2 = qualifiedLookUp(c, id, {})
+      if s2 != nil and s2 != s and not c.isAmbiguous:
+        # only consider symbols defined under current proc:
+        var o = s2.owner
+        while o != nil:
+          if o == c.p.owner:
+            result = semExpr(c, id, flags, expectedType)
+            return
+          o = o.owner
     # because of the changed symbol binding, this does not mean that we
     # don't have to check the symbol for semantics here again!
-    result = semSym(c, n, n.sym, flags)
+    result = semSym(c, n, s, flags)
   of nkEmpty, nkNone, nkCommentStmt, nkType:
     discard
   of nkNilLit:

compiler/semgnrc.nim

@@ -69,6 +69,9 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
         result.transitionSonsKind(nkClosedSymChoice)
     else:
       result = symChoice(c, n, s, scOpen)
+      if {withinMixin, withinConcept} * flags == {withinMixin} and result.kind == nkSym:
+        result.flags.incl nfOpenSym
+        result.typ = nil
   case s.kind
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
@@ -96,6 +99,9 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
         result = n
     else:
       result = newSymNodeTypeDesc(s, c.idgen, n.info)
+      if {withinMixin, withinConcept} * flags == {withinMixin}:
+        result.flags.incl nfOpenSym
+        result.typ = nil
     onUse(n.info, s)
   of skParam:
     result = n
@@ -104,11 +110,17 @@ proc semGenericStmtSymbol(c: PContext, n: PNode, s: PSym,
     if (s.typ != nil) and
        (s.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} == {}):
       result = newSymNodeTypeDesc(s, c.idgen, n.info)
+      if {withinMixin, withinConcept} * flags == {withinMixin}:
+        result.flags.incl nfOpenSym
+        result.typ = nil
     else:
       result = n
     onUse(n.info, s)
   else:
     result = newSymNode(s, n.info)
+    if {withinMixin, withinConcept} * flags == {withinMixin}:
+      result.flags.incl nfOpenSym
+      result.typ = nil
     onUse(n.info, s)
 
 proc lookup(c: PContext, n: PNode, flags: TSemGenericFlags,
@@ -148,6 +160,7 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
 
   var s = qualifiedLookUp(c, n, luf)
   if s != nil:
+    isMacro = s.kind in {skTemplate, skMacro}
     result = semGenericStmtSymbol(c, n, s, ctx, flags)
   else:
     n[0] = semGenericStmt(c, n[0], flags, ctx)

tests/generics/tmacroinjectedsym.nim

@@ -0,0 +1,86 @@
+block: # issue #22605, normal call syntax
+  const error = "bad"
+
+  template valueOr(self: int, def: untyped): untyped =
+    case false
+    of true: ""
+    of false:
+      template error: untyped {.used, inject.} = "good"
+      def
+
+  proc g(T: type): string =
+    let x = valueOr 123:
+      return $error
+
+    "ok"
+
+  doAssert g(int) == "good"
+
+block: # issue #22605, method call syntax
+  const error = "bad"
+
+  template valueOr(self: int, def: untyped): untyped =
+    case false
+    of true: ""
+    of false:
+      template error: untyped {.used, inject.} = "good"
+      def
+
+  proc g(T: type): string =
+    let x = 123.valueOr:
+      return $error
+
+    "ok"
+
+  doAssert g(int) == "good"
+
+block: # issue #22605, original complex example
+  type Xxx = enum
+    error
+    value
+
+  type
+    Result[T, E] = object
+      when T is void:
+        when E is void:
+          oResultPrivate*: bool
+        else:
+          case oResultPrivate*: bool
+          of false:
+            eResultPrivate*: E
+          of true:
+            discard
+      else:
+        when E is void:
+          case oResultPrivate*: bool
+          of false:
+            discard
+          of true:
+            vResultPrivate*: T
+        else:
+          case oResultPrivate*: bool
+          of false:
+            eResultPrivate*: E
+          of true:
+            vResultPrivate*: T
+
+  template valueOr[T: not void, E](self: Result[T, E], def: untyped): untyped =
+    let s = (self) # TODO avoid copy
+    case s.oResultPrivate
+    of true:
+      s.vResultPrivate
+    of false:
+      when E isnot void:
+        template error: untyped {.used, inject.} = s.eResultPrivate
+      def
+
+  proc f(): Result[int, cstring] =
+    Result[int, cstring](oResultPrivate: false, eResultPrivate: "f")
+
+  proc g(T: type): string =
+    let x = f().valueOr:
+      return $error
+
+    "ok"
+
+  doAssert g(int) == "f"