fix #17630: Implement cycle detection for recursive concepts (#25353)

fixes #17630

## Recursive Concept Cycle Detection

- Track (conceptId, typeId) pairs during matching to detect cycles
- Changed marker from IntSet to HashSet[ConceptTypePair]
- Removed unused depthCount field
- Added recursive concepts documentation to manual
- Added tests for recursive concepts, distinct chains, and co-dependent
concepts

## Fix Flaky `tasyncclosestall` Test

The macOS ARM64 CI jobs were failing due to a flaky async socket test
(unrelated to concepts).

The test only accepted `EBADF` as a valid error code when closing a
socket with pending writes. However, depending on timing, the kernel may
report `ECONNRESET` or `EPIPE` instead:

- **EBADF**: Socket was closed locally before kernel detected remote
state
- **ECONNRESET**: Remote peer sent RST packet (detected first)  
- **EPIPE**: Socket is no longer connected (broken pipe)

All three are valid disconnection errors. The fix accepts any of them,
making the test reliable across platforms.

---------

Co-authored-by: Andreas Rumpf <araq4k@proton.me>

compiler/concepts.nim

@@ -13,7 +13,7 @@
 
 import ast, astalgo, semdata, lookups, lineinfos, idents, msgs, renderer, types, layeredtable
 
-import std/intsets
+import std/[intsets, sets]
 
 when defined(nimPreviewSlimSystem):
   import std/assertions
@@ -73,18 +73,20 @@ type
   MatchFlags* = enum
     mfDontBind  # Do not bind generic parameters
     mfCheckGeneric  # formal <- formal comparison as opposed to formal <- operand
-  
+
+  ConceptTypePair = tuple[conceptId, typeId: ItemId]
+    ## Pair of (concept type id, implementation type id) used for cycle detection
+
   MatchCon = object ## Context we pass around during concept matching.
     bindings: LayeredIdTable
-    marker: IntSet ## Some protection against wild runaway recursions.
+    marker: HashSet[ConceptTypePair] ## Tracks (concept, type) pairs being checked to detect cycles.
     potentialImplementation: PType ## the concrete type that might match the concept we try to match.
     magic: TMagic  ## mArrGet and mArrPut is wrong in system.nim and
                    ## cannot be fixed that easily.
                    ## Thus we special case it here.
     concpt: PType  ## current concept being evaluated
-    depthCount = 0
     flags: set[MatchFlags]
-    
+
   MatchKind = enum
     mkNoMatch, mkSubset, mkSame
 
@@ -188,32 +190,48 @@ iterator traverseTyOr(t: PType): PType {. closure .}=
 proc matchConceptToImpl(c: PContext, f, potentialImpl: PType; m: var MatchCon): bool =
   assert not(potentialImpl.reduceToBase.kind == tyConcept)
   let concpt = f.reduceToBase
-  if m.depthCount > 0:
-    # concepts that are more then 2 levels deep are treated like
-    # tyAnything to stop dependencies from getting out of control
+
+  # Handle self-referential concepts: when a concept references itself in its body
+  # (e.g., `A = concept; proc test(x: Self, y: A)`), the inner type A has n=nil.
+  # We detect this by checking if the concept has the same symbol name as the
+  # one we're currently matching and has no body (n=nil).
+  if concpt.n.isNil:
+    if concpt.sym != nil and m.concpt.sym != nil and
+       concpt.sym == m.concpt.sym:
+      # Self-reference: check if potentialImpl matches what we're already checking
+      return potentialImpl.id == m.potentialImplementation.id
+    # Concept without body that's not a self-reference - cannot match
+    return false
+
+  # Cycle detection: track (concept, type) pairs to prevent infinite recursion.
+  # Returns true on cycle (coinductive semantics) to support co-dependent concepts.
+  let pair: ConceptTypePair = (concpt.itemId, potentialImpl.itemId)
+  if pair in m.marker:
     return true
+  m.marker.incl pair
+
   var efPot = potentialImpl
   if potentialImpl.isSelf:
     if m.concpt.n == concpt.n:
+      m.marker.excl pair
       return true
     efPot = m.potentialImplementation
-  
+
   var oldBindings = m.bindings
   m.bindings = newTypeMapLayer(m.bindings)
   let oldPotentialImplementation = m.potentialImplementation
   m.potentialImplementation = efPot
   let oldConcept = m.concpt
   m.concpt = concpt
-  
+
   var invocation: PType = nil
   if f.kind in {tyGenericInvocation, tyGenericInst}:
     invocation = f
-  inc m.depthCount
   result = processConcept(c, concpt, invocation, oldBindings, m)
-  dec m.depthCount
   m.potentialImplementation = oldPotentialImplementation
   m.concpt = oldConcept
   m.bindings = oldBindings
+  m.marker.excl pair
 
 proc cmpConceptDefs(c: PContext, fn, an: PNode, m: var MatchCon): bool=
   if fn.kind != an.kind:
@@ -610,7 +628,7 @@ proc conceptMatch*(c: PContext; concpt, arg: PType; bindings: var LayeredIdTable
   ## `C[S, T]` parent type that we look for. We need this because we need to store bindings
   ## for 'S' and 'T' inside 'bindings' on a successful match. It is very important that
   ## we do not add any bindings at all on an unsuccessful match!
-  var m = MatchCon(bindings: bindings, potentialImplementation: arg, concpt: concpt, flags: flags)
+  var m = MatchCon(bindings: bindings, potentialImplementation: arg, concpt: concpt, flags: flags, marker: initHashSet[ConceptTypePair]())
   if arg.isConcept:
     result = conceptsMatch(c, concpt.reduceToBase, arg.reduceToBase, m) >= mkSubset
   elif arg.acceptsAllTypes:

doc/manual.md

@@ -3025,6 +3025,44 @@ If neither of them are subsets of one another, then the disambiguation proceeds
 and the concept with the most definitions wins, if any. No definite winner is an ambiguity error at
 compile time.
 
+Recursive concepts
+------------------
+
+Concepts can reference themselves in their definitions, enabling recursive type constraints.
+This is useful for matching `distinct` types that should inherit traits from their base type:
+
+```nim
+import std/typetraits
+
+type
+  PrimitiveBase = SomeNumber | bool | ptr | pointer | enum
+
+  # Matches PrimitiveBase directly, or any distinct type whose base is Primitive
+  Primitive = concept x
+    x is PrimitiveBase or distinctBase(x) is Primitive
+
+  # Application: a handle type that should be treated like a primitive
+  Handle = distinct int
+  SpecialHandle = distinct Handle
+
+assert int is Primitive
+assert Handle is Primitive
+assert SpecialHandle is Primitive  # works through 2 levels
+assert not (string is Primitive)
+```
+
+Concepts can also be mutually recursive (co-dependent):
+
+```nim
+type
+  Serializable = concept
+    proc serialize(s: Self; writer: var Writer)
+  Writer = concept
+    proc write(w: var Self; data: Serializable)
+```
+
+The compiler uses cycle detection to handle these cases without infinite recursion.
+
 Statements and expressions
 ==========================
 

tests/concepts/t17630.nim

@@ -0,0 +1,15 @@
+discard """
+  action: "compile"
+"""
+
+# https://github.com/nim-lang/Nim/issues/17630
+# A concept that references itself in a proc signature
+# should not cause infinite recursion / stack overflow
+
+type
+  A = concept
+    proc test(x: Self, y: A)
+
+proc test(x: int, y: int) = discard
+
+discard (int is A)

tests/concepts/trecursive_concepts.nim

@@ -0,0 +1,132 @@
+discard """
+action: "run"
+output: '''
+int is Primitive: true
+Handle is Primitive: true
+SpecialHandle is Primitive: true
+FileDescriptor is Primitive: true
+float is Primitive: false
+string is Primitive: false
+char is PrimitiveBase: true
+ptr int is PrimitiveBase: true
+'''
+"""
+
+# Test recursive concepts with cycle detection
+# This tests concepts that reference themselves via distinctBase
+
+import std/typetraits
+
+block: # Basic recursive concept with distinctBase
+  type
+    PrimitiveBase = SomeInteger | bool | char | ptr | pointer
+
+    # Recursive concept: matches PrimitiveBase or any distinct type whose base is Primitive
+    Primitive = concept x
+      x is PrimitiveBase or distinctBase(x) is Primitive
+
+    # Real-world example: handle types that wrap integers
+    Handle = distinct int
+    SpecialHandle = distinct Handle
+    FileDescriptor = distinct SpecialHandle
+
+  # Direct base types
+  echo "int is Primitive: ", int is Primitive
+
+  # Single-level distinct (like a simple handle type)
+  echo "Handle is Primitive: ", Handle is Primitive
+
+  # Two-level distinct
+  echo "SpecialHandle is Primitive: ", SpecialHandle is Primitive
+
+  # Three-level distinct
+  echo "FileDescriptor is Primitive: ", FileDescriptor is Primitive
+
+  # Non-primitive types should NOT match
+  echo "float is Primitive: ", float is Primitive
+  echo "string is Primitive: ", string is Primitive
+
+block: # Ensure base type matching still works
+  type
+    PrimitiveBase = SomeInteger | bool | char | ptr | pointer
+
+  echo "char is PrimitiveBase: ", char is PrimitiveBase
+  echo "ptr int is PrimitiveBase: ", (ptr int) is PrimitiveBase
+
+block: # Test that cycle detection doesn't break normal concept matching
+  type
+    Addable = concept x, y
+      x + y is typeof(x)
+
+  doAssert int is Addable
+  doAssert float is Addable
+
+block: # Test non-matching recursive case
+  type
+    IntegerBase = SomeInteger
+
+    IntegerLike = concept x
+      x is IntegerBase or distinctBase(x) is IntegerLike
+
+    Percentage = distinct float  # float base, not integer
+
+  doAssert int is IntegerLike
+  doAssert not(float is IntegerLike)
+  doAssert not(Percentage is IntegerLike)  # float base doesn't match
+
+block: # Test deep distinct chains (5+ levels) - e.g., layered ID types
+  type
+    IdBase = SomeInteger
+
+    IdLike = concept x
+      x is IdBase or distinctBase(x) is IdLike
+
+    EntityId = distinct int
+    UserId = distinct EntityId
+    AdminId = distinct UserId
+    SuperAdminId = distinct AdminId
+    RootId = distinct SuperAdminId
+
+  doAssert int is IdLike
+  doAssert EntityId is IdLike
+  doAssert UserId is IdLike
+  doAssert AdminId is IdLike
+  doAssert SuperAdminId is IdLike
+  doAssert RootId is IdLike
+  doAssert not(float is IdLike)
+
+block: # Test 3-way mutual recursion (co-dependent concepts)
+  # This tests that cycle detection properly handles A -> B -> C -> A cycles
+  type
+    Serializable = concept
+      proc serialize(x: Self): Bytes
+
+    Bytes = concept
+      proc compress(x: Self): Compressed
+
+    Compressed = concept
+      proc decompress(x: Self): Serializable
+
+    Data = object
+      value: int
+
+  proc serialize(x: Data): Data = x
+  proc compress(x: Data): Data = x
+  proc decompress(x: Data): Data = x
+
+  # Data should satisfy all three mutually recursive concepts
+  doAssert Data is Serializable
+  doAssert Data is Bytes
+  doAssert Data is Compressed
+
+block: # Test concept with method returning same type
+  type
+    Cloneable = concept
+      proc clone(x: Self): Self
+
+    Document = object
+      content: string
+
+  proc clone(x: Document): Document = x
+
+  doAssert Document is Cloneable