Of operator in vm fixes [backport] (#9717)

* fixes #9701
*  fixes #9702
* optimize of statement to bool

(cherry picked from commit 8c1083d3b7)

compiler/lineinfos.nim

@@ -41,7 +41,7 @@ type
     hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded,
     hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
     hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
-    hintConditionAlwaysTrue, hintName, hintPattern,
+    hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
     hintExecuting, hintLinking, hintDependency,
     hintSource, hintPerformance, hintStackTrace, hintGCStats,
     hintGlobalVar,
@@ -107,6 +107,7 @@ const
     hintConf: "used config file '$1'",
     hintPath: "added path: '$1'",
     hintConditionAlwaysTrue: "condition is always true: '$1'",
+    hintConditionAlwaysFalse: "condition is always false: '$1'",
     hintName: "name should be: '$1'",
     hintPattern: "$1",
     hintExecuting: "$1",
@@ -140,7 +141,7 @@ const
     "Success", "SuccessX", "CC", "LineTooLong",
     "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
     "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
-    "Path", "CondTrue", "Name", "Pattern", "Exec", "Link", "Dependency",
+    "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
     "Source", "Performance", "StackTrace", "GCStats", "GlobalVar",
     "User", "UserRaw", "ExtendedContext",
   ]

compiler/semmagic.nim

@@ -299,7 +299,13 @@ proc semOf(c: PContext, n: PNode): PNode =
         result.typ = getSysType(c.graph, n.info, tyBool)
         return result
       elif diff == high(int):
-        localError(c.config, n.info, "'$1' cannot be of this subtype" % typeToString(a))
+        if commonSuperclass(a, b) == nil:
+          localError(c.config, n.info, "'$1' cannot be of this subtype" % typeToString(a))
+        else:
+          message(c.config, n.info, hintConditionAlwaysFalse, renderTree(n))
+          result = newIntNode(nkIntLit, 0)
+          result.info = n.info
+          result.typ = getSysType(c.graph, n.info, tyBool)
   else:
     localError(c.config, n.info, "'of' takes 2 arguments")
   n.typ = getSysType(c.graph, n.info, tyBool)

compiler/types.nim

@@ -1079,8 +1079,8 @@ proc inheritanceDiff*(a, b: PType): int =
   # | returns: +x iff `a` is the x'th direct subclass of `b`
   # | returns: `maxint` iff `a` and `b` are not compatible at all
   if a == b or a.kind == tyError or b.kind == tyError: return 0
-  assert a.kind == tyObject
-  assert b.kind == tyObject
+  assert a.kind in {tyObject} + skipPtrs
+  assert b.kind in {tyObject} + skipPtrs
   var x = a
   result = 0
   while x != nil:

compiler/vm.nim

@@ -1201,7 +1201,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcOf:
       decodeBC(rkInt)
       let typ = c.types[regs[rc].intVal.int]
-      regs[ra].intVal = ord(inheritanceDiff(regs[rb].node.typ, typ) >= 0)
+      regs[ra].intVal = ord(inheritanceDiff(regs[rb].node.typ, typ) <= 0)
     of opcIs:
       decodeBC(rkInt)
       let t1 = regs[rb].node.typ.skipTypes({tyTypeDesc})

compiler/vmgen.nim

@@ -1069,7 +1069,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     var tmp = c.genx(n.sons[1])
     var idx = c.getTemp(getSysType(c.graph, n.info, tyInt))
     var typ = n.sons[2].typ
-    if m == mOf: typ = typ.skipTypes(abstractPtrs-{tyTypeDesc})
+    if m == mOf: typ = typ.skipTypes(abstractPtrs)
     c.gABx(n, opcLdImmInt, idx, c.genType(typ))
     c.gABC(n, if m == mOf: opcOf else: opcIs, dest, tmp, idx)
     c.freeTemp(tmp)

tests/vm/tinheritance.nim

@@ -27,3 +27,54 @@ proc test() =
 
 static:
   test()
+
+#----------------------------------------------
+# Bugs #9701 and #9702
+type
+  MyKind = enum
+    kA, kB, kC
+
+  Base = ref object of RootObj
+    x: string
+
+  A = ref object of Base
+    a: string
+
+  B = ref object of Base
+    b: string
+
+  C = ref object of B
+    c: string
+
+template check_templ(n: Base, k: MyKind) =
+  if k == kA: doAssert(n of A) else: doAssert(not (n of A))
+  if k in {kB, kC}: doAssert(n of B) else: doAssert(not (n of B))
+  if k == kC: doAssert(n of C) else: doAssert(not (n of C))
+  doAssert(n of Base) 
+
+proc check_proc(n: Base, k: MyKind) =
+  if k == kA: doAssert(n of A) else: doAssert(not (n of A))
+  if k in {kB, kC}: doAssert(n of B) else: doAssert(not (n of B))
+  if k == kC: doAssert(n of C) else: doAssert(not (n of C))
+  doAssert(n of Base) 
+
+static:
+  let aa = new(A)
+  check_templ(aa, kA)
+  check_proc(aa, kA)
+  let bb = new(B)
+  check_templ(bb, kB)
+  check_proc(bb, kB)
+  let cc = new(C)
+  check_templ(cc, kC)
+  check_proc(cc, kC)
+    
+let aa = new(A)
+check_templ(aa, kA)
+check_proc(aa, kA)
+let bb = new(B)
+check_templ(bb, kB)
+check_proc(bb, kB)
+let cc = new(C)
+check_templ(cc, kC)
+check_proc(cc, kC)