typRel and sumGeneric adjustments (#23137)

Filling in some more logic in `typeRel` that I came across when poking
the compiler in another PR. Some of the cases where `typeRel` returns an
"incorrect" result are actually common, but `sumGeneric` ends up
breaking the tie correctly. There isn't anything wrong with that
necessarily, but I assume that it's preferred these functions behave
just as well in isolation as they do when integrated.

I will be following up this description with specific examples.

(cherry picked from commit ccc7c45d71)

compiler/sigmatch.nim

@@ -167,19 +167,19 @@ proc newCandidate*(ctx: PContext, callee: PSym,
 proc newCandidate*(ctx: PContext, callee: PType): TCandidate =
   initCandidate(ctx, result, callee)
 
-proc copyCandidate(a: var TCandidate, b: TCandidate) =
-  a.c = b.c
-  a.exactMatches = b.exactMatches
-  a.subtypeMatches = b.subtypeMatches
-  a.convMatches = b.convMatches
-  a.intConvMatches = b.intConvMatches
-  a.genericMatches = b.genericMatches
-  a.state = b.state
-  a.callee = b.callee
-  a.calleeSym = b.calleeSym
-  a.call = copyTree(b.call)
-  a.baseTypeMatch = b.baseTypeMatch
-  copyIdTable(a.bindings, b.bindings)
+proc copyCandidate(dest: var TCandidate, src: TCandidate) =
+  dest.c = src.c
+  dest.exactMatches = src.exactMatches
+  dest.subtypeMatches = src.subtypeMatches
+  dest.convMatches = src.convMatches
+  dest.intConvMatches = src.intConvMatches
+  dest.genericMatches = src.genericMatches
+  dest.state = src.state
+  dest.callee = src.callee
+  dest.calleeSym = src.calleeSym
+  dest.call = copyTree(src.call)
+  dest.baseTypeMatch = src.baseTypeMatch
+  copyIdTable(dest.bindings, src.bindings)
 
 proc typeRel*(c: var TCandidate, f, aOrig: PType,
               flags: TTypeRelFlags = {}): TTypeRelation
@@ -194,10 +194,10 @@ proc checkGeneric(a, b: TCandidate): int =
     let tra = typeRel(ma, bb[i], aa[i], {trDontBind})
     var mb = newCandidate(c, aa[i])
     let trb = typeRel(mb, aa[i], bb[i], {trDontBind})
-    if tra == isGeneric and trb == isNone:
+    if tra == isGeneric and trb in {isNone, isInferred, isInferredConvertible}:
       if winner == -1: return 0
       winner = 1
-    if trb == isGeneric and tra == isNone:
+    if trb == isGeneric and tra in {isNone, isInferred, isInferredConvertible}:
       if winner == 1: return 0
       winner = -1
   result = winner
@@ -207,14 +207,23 @@ proc sumGeneric(t: PType): int =
   # and Foo[T] has the value 2 so that we know Foo[Foo[T]] is more
   # specific than Foo[T].
   var t = t
-  var isvar = 0
   while true:
     case t.kind
-    of tyGenericInst, tyArray, tyRef, tyPtr, tyDistinct, tyUncheckedArray,
+    of tyAlias, tySink, tyNot: t = t.lastSon
+    of tyArray, tyRef, tyPtr, tyDistinct, tyUncheckedArray,
         tyOpenArray, tyVarargs, tySet, tyRange, tySequence, tyGenericBody,
-        tyLent, tyOwned:
+        tyLent, tyOwned, tyVar:
       t = t.lastSon
       inc result
+    of tyBool, tyChar, tyEnum, tyObject, tyPointer, tyVoid,
+        tyString, tyCstring, tyInt..tyInt64, tyFloat..tyFloat128,
+        tyUInt..tyUInt64, tyCompositeTypeClass, tyBuiltInTypeClass,
+        tyGenericParam:
+      inc result
+      break
+    of tyGenericInst, tyStatic:
+      t = t[0]
+      inc result
     of tyOr:
       var maxBranch = 0
       for branch in t.sons:
@@ -222,32 +231,19 @@ proc sumGeneric(t: PType): int =
         if branchSum > maxBranch: maxBranch = branchSum
       inc result, maxBranch
       break
-    of tyVar:
-      t = t[0]
-      inc result
-      inc isvar
     of tyTypeDesc:
       t = t.lastSon
       if t.kind == tyEmpty: break
       inc result
+    of tyUntyped, tyTyped: break
     of tyGenericInvocation, tyTuple, tyProc, tyAnd:
-      result += ord(t.kind in {tyGenericInvocation, tyAnd})
+      result += ord(t.kind == tyAnd)
       for i in 0..<t.len:
         if t[i] != nil:
           result += sumGeneric(t[i])
       break
-    of tyStatic:
-      return sumGeneric(t[0]) + 1
-    of tyGenericParam, tyUntyped, tyTyped: break
-    of tyAlias, tySink: t = t.lastSon
-    of tyBool, tyChar, tyEnum, tyObject, tyPointer,
-        tyString, tyCstring, tyInt..tyInt64, tyFloat..tyFloat128,
-        tyUInt..tyUInt64, tyCompositeTypeClass:
-      return isvar + 1
-    of tyBuiltInTypeClass:
-      return isvar
     else:
-      return 0
+      break
 
 proc complexDisambiguation(a, b: PType): int =
   # 'a' matches better if *every* argument matches better or equal than 'b'.
@@ -449,33 +445,34 @@ proc handleFloatRange(f, a: PType): TTypeRelation =
       else: result = isIntConv
     else: result = isNone
 
-proc getObjectTypeOrNil(f: PType): PType =
+proc getObjectType(f: PType): PType =
   #[
     Returns a type that is f's effective typeclass. This is usually just one level deeper
     in the hierarchy of generality for a type. `object`, `ref object`, `enum` and user defined
     tyObjects are common return values.
   ]#
-  if f == nil: return nil
   case f.kind:
-  of tyGenericInvocation, tyCompositeTypeClass, tyAlias:
+  of tyGenericInvocation:
+    result = getObjectType(f[0])
+  of tyCompositeTypeClass, tyAlias:
     if f.len <= 0 or f[0] == nil:
-      result = nil
+      result = f
     else:
-      result = getObjectTypeOrNil(f[0])
+      result = getObjectType(f[0])
   of tyGenericBody, tyGenericInst:
-    result = getObjectTypeOrNil(f.lastSon)
+    result = getObjectType(f.lastSon)
   of tyUserTypeClass:
     if f.isResolvedUserTypeClass:
       result = f.base  # ?? idk if this is right
     else:
       result = f.lastSon
   of tyStatic, tyOwned, tyVar, tyLent, tySink:
-    result = getObjectTypeOrNil(f.base)
+    result = getObjectType(f.base)
   of tyInferred:
     # This is not true "After a candidate type is selected"
-    result = getObjectTypeOrNil(f.base)
+    result = getObjectType(f.base)
   of tyTyped, tyUntyped, tyFromExpr:
-    result = nil
+    result = f
   of tyRange:
     result = f.lastSon
   else:
@@ -559,7 +556,7 @@ proc minRel(a, b: TTypeRelation): TTypeRelation =
   if a <= b: result = a
   else: result = b
 
-proc recordRel(c: var TCandidate, f, a: PType): TTypeRelation =
+proc recordRel(c: var TCandidate, f, a: PType, flags: TTypeRelFlags): TTypeRelation =
   result = isNone
   if sameType(f, a):
     result = isEqual
@@ -568,7 +565,7 @@ proc recordRel(c: var TCandidate, f, a: PType): TTypeRelation =
     let firstField = if f.kind == tyTuple: 0
                      else: 1
     for i in firstField..<f.len:
-      var m = typeRel(c, f[i], a[i])
+      var m = typeRel(c, f[i], a[i], flags)
       if m < isSubtype: return isNone
       result = minRel(result, m)
     if f.n != nil and a.n != nil:
@@ -579,7 +576,7 @@ proc recordRel(c: var TCandidate, f, a: PType): TTypeRelation =
         else:
           var x = f.n[i].sym
           var y = a.n[i].sym
-          if f.kind == tyObject and typeRel(c, x.typ, y.typ) < isSubtype:
+          if f.kind == tyObject and typeRel(c, x.typ, y.typ, flags) < isSubtype:
             return isNone
           if x.name.id != y.name.id: return isNone
 
@@ -1234,6 +1231,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       result = typeRel(c, f.base, aOrig, flags + {trNoCovariance})
     subtypeCheck()
   of tyArray:
+    a = getObjectType(a)
     case a.kind
     of tyArray:
       var fRange = f[0]
@@ -1354,13 +1352,12 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
     skipOwned(a)
     if a.kind == f.kind: result = isEqual
   of tyTuple:
-    if a.kind == tyTuple: result = recordRel(c, f, a)
+    if a.kind == tyTuple: result = recordRel(c, f, a, flags)
   of tyObject:
     let effectiveArgType = if useTypeLoweringRuleInTypeClass:
         a
       else:
-        getObjectTypeOrNil(a)
-    if effectiveArgType == nil: return isNone
+        getObjectType(a)
     if effectiveArgType.kind == tyObject:
       if sameObjectTypes(f, effectiveArgType):
         result = isEqual
@@ -1391,7 +1388,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
             result = isNone
 
   of tyPtr, tyRef:
-    skipOwned(a)
+    a = getObjectType(a)
     if a.kind == f.kind:
       # ptr[R, T] can be passed to ptr[T], but not the other way round:
       if a.len < f.len: return isNone
@@ -1692,8 +1689,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
     considerPreviousT:
       let target = f[0]
       let targetKind = target.kind
-      var effectiveArgType = a.getObjectTypeOrNil()
-      if effectiveArgType == nil: return isNone
+      var effectiveArgType = getObjectType(a)
       effectiveArgType = effectiveArgType.skipTypes({tyBuiltInTypeClass})
       if targetKind == effectiveArgType.kind:
         if effectiveArgType.isEmptyContainer: