fix some issues overloading with generics and inheritance (#15211)

* fix some issues overloading with generics and inheritance

* fix passing procs with subtype matches

compiler/sigmatch.nim

@@ -199,6 +199,27 @@ proc copyCandidate(a: var TCandidate, b: TCandidate) =
   a.baseTypeMatch = b.baseTypeMatch
   copyIdTable(a.bindings, b.bindings)
 
+proc typeRel*(c: var TCandidate, f, aOrig: PType,
+              flags: TTypeRelFlags = {}): TTypeRelation
+
+proc checkGeneric(a, b: TCandidate): int =
+  let c = a.c
+  let aa = a.callee
+  let bb = b.callee
+  var winner = 0
+  for i in 1..<min(aa.len, bb.len):
+    var ma = newCandidate(c, bb[i])
+    let tra = typeRel(ma, bb[i], aa[i], {trDontBind})
+    var mb = newCandidate(c, aa[i])
+    let trb = typeRel(mb, aa[i], bb[i], {trDontBind})
+    if tra == isGeneric and trb == isNone:
+      if winner == -1: return 0
+      winner = 1
+    if trb == isGeneric and tra == isNone:
+      if winner == 1: return 0
+      winner = -1
+  result = winner
+
 proc sumGeneric(t: PType): int =
   # count the "genericness" so that Foo[Foo[T]] has the value 3
   # and Foo[T] has the value 2 so that we know Foo[Foo[T]] is more
@@ -296,6 +317,9 @@ proc cmpCandidates*(a, b: TCandidate): int =
   # the other way round because of other semantics:
   result = b.inheritancePenalty - a.inheritancePenalty
   if result != 0: return
+  # check for generic subclass relation
+  result = checkGeneric(a, b)
+  if result != 0: return
   # prefer more specialized generic over more general generic:
   result = complexDisambiguation(a.callee, b.callee)
   # only as a last resort, consider scoping:
@@ -336,9 +360,6 @@ proc describeArgs*(c: PContext, n: PNode, startIdx = 1;
     result.add(argTypeToString(arg, prefer))
     if i != n.len - 1: result.add(", ")
 
-proc typeRel*(c: var TCandidate, f, aOrig: PType,
-              flags: TTypeRelFlags = {}): TTypeRelation
-
 proc concreteType(c: TCandidate, t: PType; f: PType = nil): PType =
   case t.kind
   of tyTypeDesc:
@@ -357,8 +378,7 @@ proc concreteType(c: TCandidate, t: PType; f: PType = nil): PType =
         # proc sort[T](cmp: proc(a, b: T): int = cmp)
       if result.kind != tyGenericParam: break
   of tyGenericInvocation:
-    result = t
-    doAssert(false, "cannot resolve type: " & typeToString(t))
+    result = nil
   of tyOwned:
     # bug #11257: the comparison system.`==`[T: proc](x, y: T) works
     # better without the 'owned' type:
@@ -599,7 +619,8 @@ proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
     # if f is metatype.
     result = typeRel(c, f, a)
 
-  if result <= isSubtype or inconsistentVarTypes(f, a):
+  #               v--- is this correct?
+  if result <= isIntConv or inconsistentVarTypes(f, a):
     result = isNone
 
   #if result == isEqual:
@@ -1465,7 +1486,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
           if baseType != nil:
             c.inheritancePenalty += 1
             let ret = typeRel(c, f, baseType)
-            return if ret == isEqual: isSubtype else: ret
+            return if ret in {isEqual,isGeneric}: isSubtype else: ret
 
         result = isNone
     else:
@@ -1496,10 +1517,12 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       #echo "inferred ", typeToString(inst), " for ", f
       return typeRel(c, inst, a)
 
-    if x.kind == tyGenericInvocation or f[0].kind != tyGenericBody:
-      #InternalError("typeRel: tyGenericInvocation -> tyGenericInvocation")
-      # simply no match for now:
-      discard
+    if x.kind == tyGenericInvocation:
+      if f[0] == x[0]:
+        for i in 1..<f.len:
+          let tr = typeRel(c, f[i], x[i])
+          if tr <= isSubtype: return
+        result = isGeneric
     elif x.kind == tyGenericInst and f[0] == x[0] and
           x.len - 1 == f.len:
       for i in 1..<f.len:

tests/overload/toverload_various.nim

@@ -411,7 +411,7 @@ block:
   test(d, 2)
 
 
-# inheritance depth
+# inheritance and generics
 block:
   type
     Foo[T] = object of RootObj
@@ -421,17 +421,43 @@ block:
     Baz[T] = object of Bar[T]
       z: T
 
-  template t0[T](x: Foo[T]): int = 0
-  template t0[T](x: Bar[T]): int = 1
-  proc p0[T](x: Foo[T]): int = 0
-  proc p0[T](x: Bar[T]): int = 1
+  template t0(x: Foo[int]): int = 0
+  template t0(x: Bar[int]): int = 1
+  template t0(x: Foo[bool or int]): int = 10
+  template t0(x: Bar[bool or int]): int = 11
+  template t0[T](x: Foo[T]): int = 20
+  template t0[T](x: Bar[T]): int = 21
+  proc p0(x: Foo[int]): int = 0
+  proc p0(x: Bar[int]): int = 1
+  #proc p0(x: Foo[bool or int]): int = 10
+  #proc p0(x: Bar[bool or int]): int = 11
+  proc p0[T](x: Foo[T]): int = 20
+  proc p0[T](x: Bar[T]): int = 21
 
   var a: Foo[int]
   var b: Bar[int]
   var c: Baz[int]
+  var d: Foo[bool]
+  var e: Bar[bool]
+  var f: Baz[bool]
+  var g: Foo[float]
+  var h: Bar[float]
+  var i: Baz[float]
   doAssert(t0(a) == 0)
   doAssert(t0(b) == 1)
   doAssert(t0(c) == 1)
+  doAssert(t0(d) == 10)
+  doAssert(t0(e) == 11)
+  doAssert(t0(f) == 11)
+  doAssert(t0(g) == 20)
+  doAssert(t0(h) == 21)
+  #doAssert(t0(i) == 21)
   doAssert(p0(a) == 0)
   doAssert(p0(b) == 1)
   doAssert(p0(c) == 1)
+  #doAssert(p0(d) == 10)
+  #doAssert(p0(e) == 11)
+  #doAssert(p0(f) == 11)
+  doAssert(p0(g) == 20)
+  doAssert(p0(h) == 21)
+  doAssert(p0(i) == 21)