From c26cb470a22e1985bbb7ec878a402ad8f78ef75e Mon Sep 17 00:00:00 2001
From: gingerBill <bill@gingerbill.org>
Date: Mon, 9 Nov 2020 10:27:27 +0000
Subject: [PATCH] Fix LLVM-API type cycle for procedures of named procedures

---
 src/check_type.cpp   | 24 +++++++++++++++++++++---
 src/llvm_backend.cpp |  3 ++-
 src/types.cpp        |  1 +
 3 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/check_type.cpp b/src/check_type.cpp
index 93040e493..62b5fd8e1 100644
--- a/src/check_type.cpp
+++ b/src/check_type.cpp
@@ -2209,6 +2209,11 @@ Type *type_to_abi_compat_param_type(gbAllocator a, Type *original_type, ProcCall
 		return new_type;
 	}
 
+	if (is_type_proc(original_type)) {
+		// NOTE(bill): Force a cast to prevent a possible type cycle
+		return t_rawptr;
+	}
+
 	if (cc == ProcCC_None || cc == ProcCC_PureNone || cc == ProcCC_InlineAsm) {
 		return new_type;
 	}
@@ -2332,6 +2337,11 @@ Type *type_to_abi_compat_result_type(gbAllocator a, Type *original_type, ProcCal
 		return new_type;
 	}
 
+	if (is_type_proc(single_type)) {
+		// NOTE(bill): Force a cast to prevent a possible type cycle
+		return t_rawptr;
+	}
+
 	if (is_type_simd_vector(single_type)) {
 		return new_type;
 	}
@@ -2451,22 +2461,29 @@ void set_procedure_abi_types(gbAllocator allocator, Type *type) {
 		return;
 	}
 
-	if (type->Proc.abi_types_set) {
+	if (type->Proc.abi_types_set || type->flags & TypeFlag_InProcessOfCheckingABI) {
 		return;
 	}
 
+	u32 flags = type->flags;
+	type->flags |= TypeFlag_InProcessOfCheckingABI;
+
 	type->Proc.abi_compat_params = array_make<Type *>(allocator, cast(isize)type->Proc.param_count);
 	for (i32 i = 0; i < type->Proc.param_count; i++) {
 		Entity *e = type->Proc.params->Tuple.variables[i];
 		if (e->kind == Entity_Variable) {
 			Type *original_type = e->type;
+			if (is_type_named(original_type) && is_type_proc(original_type)) {
+				continue;
+			}
+
 			Type *new_type = type_to_abi_compat_param_type(allocator, original_type, type->Proc.calling_convention);
 			type->Proc.abi_compat_params[i] = new_type;
 			switch (type->Proc.calling_convention) {
 			case ProcCC_Odin:
 			case ProcCC_Contextless:
 			case ProcCC_Pure:
-				if (is_type_pointer(new_type) & !is_type_pointer(e->type)) {
+				if (is_type_pointer(new_type) && !is_type_pointer(e->type) && !is_type_proc(e->type)) {
 					e->flags |= EntityFlag_ImplicitReference;
 				}
 				break;
@@ -2474,7 +2491,7 @@ void set_procedure_abi_types(gbAllocator allocator, Type *type) {
 
 			if (build_context.ODIN_OS == "linux" ||
 			    build_context.ODIN_OS == "darwin") {
-				if (is_type_pointer(new_type) & !is_type_pointer(e->type)) {
+				if (is_type_pointer(new_type) & !is_type_pointer(e->type) && !is_type_proc(e->type)) {
 					e->flags |= EntityFlag_ByVal;
 				}
 			}
@@ -2499,6 +2516,7 @@ void set_procedure_abi_types(gbAllocator allocator, Type *type) {
 	type->Proc.return_by_pointer = abi_compat_return_by_pointer(allocator, type->Proc.calling_convention, type->Proc.abi_compat_result_type);
 
 	type->Proc.abi_types_set = true;
+	type->flags = flags;
 }
 
 // NOTE(bill): 'operands' is for generating non generic procedure type
diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp
index 4abc65ab4..c6fee8ab8 100644
--- a/src/llvm_backend.cpp
+++ b/src/llvm_backend.cpp
@@ -1370,7 +1370,8 @@ LLVMTypeRef lb_type_internal(lbModule *m, Type *type) {
 							// Sanity check
 							continue;
 						}
-						array_add(&param_types, lb_type(m, v->type));
+						LLVMTypeRef t = lb_type(m, v->type);
+						array_add(&param_types, t);
 					}
 				} else {
 					array_add(&param_types, lb_type(m, param));
diff --git a/src/types.cpp b/src/types.cpp
index f4d375f4f..acc1c7b2e 100644
--- a/src/types.cpp
+++ b/src/types.cpp
@@ -324,6 +324,7 @@ enum TypeFlag : u32 {
 	TypeFlag_Polymorphic     = 1<<1,
 	TypeFlag_PolySpecialized = 1<<2,
 	TypeFlag_InProcessOfCheckingPolymorphic = 1<<3,
+	TypeFlag_InProcessOfCheckingABI = 1<<4,
 };
 
 struct Type {