Fix #17509: Continue instead of return with unfinished generics (#22563)

Close #17509

Current knowledge:
- delaying cache fixes the issue
- changing return of `if inst.len < key.len:` in `searchInstTypes` to
`continue` fixes the issue. With return the broken types are also cached
over and over

Related issues are completely unaffected as of now, so there must be
something deeper.

I am also still trying to find the true cause, so feel free to ignore
for now

---------

Co-authored-by: SirOlaf <>

compiler/semtypinst.nim

@@ -45,7 +45,8 @@ proc searchInstTypes*(g: ModuleGraph; key: PType): PType =
       # XXX: This happens for prematurely cached
       # types such as Channel[empty]. Why?
       # See the notes for PActor in handleGenericInvocation
-      return
+      # if this is return the same type gets cached more than it needs to
+      continue
     if not sameFlags(inst, key):
       continue
 

tests/generics/t17509.nim

@@ -0,0 +1,25 @@
+type List[O] = object
+  next: ptr List[O]
+
+proc initList[O](l: ptr List[O]) =
+  l[].next = l
+
+type
+  PolytopeVertex[R] = object
+    list: List[PolytopeVertex[R]]
+
+  PolytopeEdge[R] = object
+    list: List[PolytopeEdge[R]]
+
+  Polytope[R] = object
+    vertices: List[PolytopeVertex[R]]
+    edges: List[PolytopeEdge[R]]
+
+var pt: Polytope[float]
+
+static:
+  doAssert pt.vertices.next is (ptr List[PolytopeVertex[float]])
+  doAssert pt.edges.next is (ptr List[PolytopeEdge[float]])
+
+initList(addr pt.vertices)
+initList(addr pt.edges)