From 7acdacb25316ffa34c6d0cc65c8106312bcbdae3 Mon Sep 17 00:00:00 2001
From: ringabout <43030857+ringabout@users.noreply.github.com>
Date: Wed, 24 Aug 2022 01:58:50 +0800
Subject: [PATCH] fixes #18983 #5282 #13008; recursive types casue infinite
 type (#20172)

* fixes #18983 #5282 #13008; recursive types casue infinite type

* re

* add testcases

(cherry picked from commit a895bbf714894b059361d0c63a0e46e54eb22da3)
---
 compiler/semtypes.nim    | 29 +++++++++++++++++++++++++++--
 tests/errmsgs/t18983.nim |  7 +++++++
 tests/errmsgs/t5282.nim  |  5 +++++
 3 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 tests/errmsgs/t18983.nim
 create mode 100644 tests/errmsgs/t5282.nim

diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim
index 5b96c2ff2d..f5bbd46d6d 100644
--- a/compiler/semtypes.nim
+++ b/compiler/semtypes.nim
@@ -212,10 +212,34 @@ proc semVarOutType(c: PContext, n: PNode, prev: PType; kind: TTypeKind): PType =
   else:
     result = newConstraint(c, kind)
 
+proc isRecursiveType(t: PType, cycleDetector: var IntSet): bool =
+  if t == nil:
+    return false
+  if cycleDetector.containsOrIncl(t.id):
+    return true
+  case t.kind
+  of tyAlias, tyGenericInst, tyDistinct:
+    return isRecursiveType(t.lastSon, cycleDetector)
+  else:
+    return false
+
+proc isRecursiveType*(t: PType): bool =
+  # handle simple recusive types before typeFinalPass
+  var cycleDetector = initIntSet()
+  isRecursiveType(t, cycleDetector)
+
+proc addSonSkipIntLitChecked(c: PContext; father, son: PType; it: PNode, id: IdGenerator) =
+  let s = son.skipIntLit(id)
+  father.sons.add(s)
+  if isRecursiveType(s):
+    localError(c.config, it.info, "illegal recursion in type '" & typeToString(s) & "'")
+  else:
+    propagateToOwner(father, s)
+
 proc semDistinct(c: PContext, n: PNode, prev: PType): PType =
   if n.len == 0: return newConstraint(c, tyDistinct)
   result = newOrPrevType(tyDistinct, prev, c)
-  addSonSkipIntLit(result, semTypeNode(c, n[0], nil), c.idgen)
+  addSonSkipIntLitChecked(c, result, semTypeNode(c, n[0], nil), n[0], c.idgen)
   if n.len > 1: result.n = n[1]
 
 proc semRangeAux(c: PContext, n: PNode, prev: PType): PType =
@@ -447,7 +471,8 @@ proc semAnonTuple(c: PContext, n: PNode, prev: PType): PType =
     localError(c.config, n.info, errTypeExpected)
   result = newOrPrevType(tyTuple, prev, c)
   for it in n:
-    addSonSkipIntLit(result, semTypeNode(c, it, nil), c.idgen)
+    let t = semTypeNode(c, it, nil)
+    addSonSkipIntLitChecked(c, result, t, it, c.idgen)
 
 proc semTuple(c: PContext, n: PNode, prev: PType): PType =
   var typ: PType
diff --git a/tests/errmsgs/t18983.nim b/tests/errmsgs/t18983.nim
new file mode 100644
index 0000000000..3451875cb4
--- /dev/null
+++ b/tests/errmsgs/t18983.nim
@@ -0,0 +1,7 @@
+discard """
+  errormsg: "illegal recursion in type 'A'"
+"""
+
+type
+  A* = A
+  B = (A,)
diff --git a/tests/errmsgs/t5282.nim b/tests/errmsgs/t5282.nim
new file mode 100644
index 0000000000..4da49d1551
--- /dev/null
+++ b/tests/errmsgs/t5282.nim
@@ -0,0 +1,5 @@
+discard """
+  errormsg: "illegal recursion in type 'x'"
+"""
+
+type x = distinct x