mostly fixes #1339

The compiler hangs were caused by the interaction of tyError and the instantiation caches.
For procs, the cache wasn't able to find previously compiled proc featuring tyError in the signature.
For types, the unresolved type parameters leading to tyError were not replaced everywhere leading
to endless replaceTypeVarsT recursion for cyclic types.

The fix is still not perfect, because the handling of tyError in other places in the compiler doesn't seem
to be complete and the first test case now results in internal error (still, much better than a hang blocking
your IDE).

compiler/semtypinst.nim

@@ -216,12 +216,16 @@ proc replaceTypeVarsS(cl: var TReplTypeVars, s: PSym): PSym =
     result.typ = replaceTypeVarsT(cl, s.typ)
     result.ast = replaceTypeVarsN(cl, s.ast)
     
-proc lookupTypeVar(cl: TReplTypeVars, t: PType): PType = 
+proc lookupTypeVar(cl: var TReplTypeVars, t: PType): PType =
   result = PType(idTableGet(cl.typeMap, t))
   if result == nil:
     if cl.allowMetaTypes or tfRetType in t.flags: return
     localError(t.sym.info, errCannotInstantiateX, typeToString(t))
     result = errorType(cl.c)
+    # In order to prevent endless recursions, we must remember
+    # this bad lookup and replace it with errorType everywhere.
+    # These code paths are only active in nimrod check
+    idTablePut(cl.typeMap, t, result)
   elif result.kind == tyGenericParam and not cl.allowMetaTypes:
     internalError(cl.info, "substitution with generic parameter")
 
@@ -353,7 +357,7 @@ proc replaceTypeVarsTAux(cl: var TReplTypeVars, t: PType): PType =
 
   of tyGenericBody:
     localError(cl.info, errCannotInstantiateX, typeToString(t))
-    result = t
+    result = errorType(cl.c)
     #result = replaceTypeVarsT(cl, lastSon(t))
 
   of tyFromExpr:

compiler/types.nim

@@ -913,9 +913,11 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
           result = sameTypeAux(a.sons[0], b.sons[0], c)     
     else: 
       result = sameTypeAux(a.sons[0], b.sons[0], c) and sameFlags(a, b)
-  of tyEnum, tyForward, tyProxy:
+  of tyEnum, tyForward:
     # XXX generic enums do not make much sense, but require structural checking
     result = a.id == b.id and sameFlags(a, b)
+  of tyError:
+    result = b.kind == tyError
   of tyTuple:
     cycleCheck()
     result = sameTuple(a, b, c) and sameFlags(a, b)