mirrors/Odin
1
0
Fork 0
mirror of https://github.com/odin-lang/Odin.git synced 2026-01-08 14:03:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3644 from odin-lang/foreign-import-improvements

Browse Source 
Allow `foreign import` import paths to be evaluated in the semantic phase rather than parsing
This commit is contained in:
gingerBill
2024-05-28 12:18:15 +01:00
committed by GitHub
parent 23852c16be d91054b615
commit 74524b6050
8 changed files with 152 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
core/odin/ast/ast.odin
View File

@@ -538,7 +538,7 @@ Foreign_Import_Decl :: struct {
import_tok: tokenizer.Token,
name: ^Ident,
collection_name: string,
fullpaths: []string,
fullpaths: []^Expr,
comment: ^Comment_Group,
}

10
core/odin/parser/parser.odin
View File

@@ -1190,12 +1190,12 @@ parse_foreign_decl :: proc(p: ^Parser) -> ^ast.Decl {
error(p, name.pos, "illegal foreign import name: '_'")
}
fullpaths: [dynamic]string
fullpaths: [dynamic]^ast.Expr
if allow_token(p, .Open_Brace) {
for p.curr_tok.kind != .Close_Brace &&
p.curr_tok.kind != .EOF {
path := expect_token(p, .String)
append(&fullpaths, path.text)
path := parse_expr(p, false)
append(&fullpaths, path)
allow_token(p, .Comma) or_break
}
@@ -1203,7 +1203,9 @@ parse_foreign_decl :: proc(p: ^Parser) -> ^ast.Decl {
} else {
path := expect_token(p, .String)
reserve(&fullpaths, 1)
append(&fullpaths, path.text)
bl := ast.new(ast.Basic_Lit, path.pos, end_pos(path))
bl.tok = tok
append(&fullpaths, bl)
}
if len(fullpaths) == 0 {

139
src/checker.cpp
View File

@@ -1283,6 +1283,7 @@ gb_internal void init_checker_info(CheckerInfo *i) {
mpsc_init(&i->definition_queue, a); //); // 1<<20);
mpsc_init(&i->required_global_variable_queue, a); // 1<<10);
mpsc_init(&i->required_foreign_imports_through_force_queue, a); // 1<<10);
mpsc_init(&i->foreign_imports_to_check_fullpaths, a); // 1<<10);
mpsc_init(&i->intrinsics_entry_point_usage, a); // 1<<10); // just waste some memory here, even if it probably never used
string_map_init(&i->load_directory_cache);
@@ -1307,6 +1308,7 @@ gb_internal void destroy_checker_info(CheckerInfo *i) {
mpsc_destroy(&i->definition_queue);
mpsc_destroy(&i->required_global_variable_queue);
mpsc_destroy(&i->required_foreign_imports_through_force_queue);
mpsc_destroy(&i->foreign_imports_to_check_fullpaths);
map_destroy(&i->objc_msgSend_types);
string_map_destroy(&i->load_file_cache);
@@ -4874,6 +4876,83 @@ gb_internal DECL_ATTRIBUTE_PROC(foreign_import_decl_attribute) {
return false;
}
gb_internal void check_foreign_import_fullpaths(Checker *c) {
CheckerContext ctx = make_checker_context(c);
UntypedExprInfoMap untyped = {};
defer (map_destroy(&untyped));
for (Entity *e = nullptr; mpsc_dequeue(&c->info.foreign_imports_to_check_fullpaths, &e); /**/) {
GB_ASSERT(e != nullptr);
GB_ASSERT(e->kind == Entity_LibraryName);
Ast *decl = e->LibraryName.decl;
ast_node(fl, ForeignImportDecl, decl);
AstFile *f = decl->file();
reset_checker_context(&ctx, f, &untyped);
ctx.collect_delayed_decls = false;
GB_ASSERT(ctx.scope == e->scope);
if (fl->fullpaths.count == 0) {
String base_dir = dir_from_path(decl->file()->fullpath);
auto fullpaths = array_make<String>(permanent_allocator(), 0, fl->filepaths.count);
for (Ast *fp_node : fl->filepaths) {
Operand op = {};
check_expr(&ctx, &op, fp_node);
if (op.mode != Addressing_Constant && op.value.kind != ExactValue_String) {
gbString s = expr_to_string(op.expr);
error(fp_node, "Expected a constant string value, got '%s'", s);
gb_string_free(s);
continue;
}
if (!is_type_string(op.type)) {
gbString s = type_to_string(op.type);
error(fp_node, "Expected a constant string value, got value of type '%s'", s);
gb_string_free(s);
continue;
}
String file_str = op.value.value_string;
file_str = string_trim_whitespace(file_str);
String fullpath = file_str;
if (allow_check_foreign_filepath()) {
String foreign_path = {};
bool ok = determine_path_from_string(nullptr, decl, base_dir, file_str, &foreign_path, /*use error not syntax_error*/true);
if (ok) {
fullpath = foreign_path;
}
}
array_add(&fullpaths, fullpath);
}
fl->fullpaths = slice_from_array(fullpaths);
}
for (String const &path : fl->fullpaths) {
String ext = path_extension(path);
if (str_eq_ignore_case(ext, ".c") ||
str_eq_ignore_case(ext, ".cpp") ||
str_eq_ignore_case(ext, ".cxx") ||
str_eq_ignore_case(ext, ".h") ||
str_eq_ignore_case(ext, ".hpp") ||
str_eq_ignore_case(ext, ".hxx") ||
false
) {
error(fl->token, "With 'foreign import', you cannot import a %.*s file/directory, you must precompile the library and link against that", LIT(ext));
break;
}
}
add_untyped_expressions(ctx.info, &untyped);
e->LibraryName.paths = fl->fullpaths;
}
}
gb_internal void check_add_foreign_import_decl(CheckerContext *ctx, Ast *decl) {
if (decl->state_flags & StateFlag_BeenHandled) return;
decl->state_flags |= StateFlag_BeenHandled;
@@ -4883,59 +4962,26 @@ gb_internal void check_add_foreign_import_decl(CheckerContext *ctx, Ast *decl) {
Scope *parent_scope = ctx->scope;
GB_ASSERT(parent_scope->flags&ScopeFlag_File);
GB_ASSERT(fl->fullpaths.count > 0);
String fullpath = fl->fullpaths[0];
String library_name = path_to_entity_name(fl->library_name.string, fullpath);
if (is_blank_ident(library_name)) {
error(fl->token, "File name, %.*s, cannot be as a library name as it is not a valid identifier", LIT(fl->library_name.string));
String library_name = fl->library_name.string;
if (library_name.len == 0 && fl->fullpaths.count != 0) {
String fullpath = fl->fullpaths[0];
library_name = path_to_entity_name(fl->library_name.string, fullpath);
}
if (library_name.len == 0 || is_blank_ident(library_name)) {
error(fl->token, "File name, '%.*s', cannot be as a library name as it is not a valid identifier", LIT(library_name));
return;
}
for (String const &path : fl->fullpaths) {
String ext = path_extension(path);
if (str_eq_ignore_case(ext, ".c") ||
str_eq_ignore_case(ext, ".cpp") ||
str_eq_ignore_case(ext, ".cxx") ||
str_eq_ignore_case(ext, ".h") ||
str_eq_ignore_case(ext, ".hpp") ||
str_eq_ignore_case(ext, ".hxx") ||
false
) {
error(fl->token, "With 'foreign import', you cannot import a %.*s file directory, you must precompile the library and link against that", LIT(ext));
break;
}
}
// if (fl->collection_name != "system") {
// char *c_str = gb_alloc_array(heap_allocator(), char, fullpath.len+1);
// defer (gb_free(heap_allocator(), c_str));
// gb_memmove(c_str, fullpath.text, fullpath.len);
// c_str[fullpath.len] = '\0';
// gbFile f = {};
// gbFileError file_err = gb_file_open(&f, c_str);
// defer (gb_file_close(&f));
// switch (file_err) {
// case gbFileError_Invalid:
// error(decl, "Invalid file or cannot be found ('%.*s')", LIT(fullpath));
// return;
// case gbFileError_NotExists:
// error(decl, "File cannot be found ('%.*s')", LIT(fullpath));
// return;
// }
// }
GB_ASSERT(fl->library_name.pos.line != 0);
fl->library_name.string = library_name;
Entity *e = alloc_entity_library_name(parent_scope, fl->library_name, t_invalid,
fl->fullpaths, library_name);
e->LibraryName.decl = decl;
add_entity_flags_from_file(ctx, e, parent_scope);
add_entity(ctx, parent_scope, nullptr, e);
AttributeContext ac = {};
check_decl_attributes(ctx, fl->attributes, foreign_import_decl_attribute, &ac);
if (ac.require_declaration) {
@@ -4950,12 +4996,8 @@ gb_internal void check_add_foreign_import_decl(CheckerContext *ctx, Ast *decl) {
e->LibraryName.extra_linker_flags = extra_linker_flags;
}
if (has_asm_extension(fullpath)) {
if (build_context.metrics.arch != TargetArch_amd64 && build_context.metrics.os != TargetOs_darwin) {
error(decl, "Assembly files are not yet supported on this platform: %.*s_%.*s",
LIT(target_os_names[build_context.metrics.os]), LIT(target_arch_names[build_context.metrics.arch]));
}
}
mpsc_enqueue(&ctx->info->foreign_imports_to_check_fullpaths, e);
}
// Returns true if a new package is present
@@ -6317,6 +6359,9 @@ gb_internal void check_parsed_files(Checker *c) {
TIME_SECTION("check procedure bodies");
check_procedure_bodies(c);
TIME_SECTION("check foreign import fullpaths");
check_foreign_import_fullpaths(c);
TIME_SECTION("add entities from procedure bodies");
check_merge_queues_into_arrays(c);

3
src/checker.hpp
View File

@@ -414,6 +414,7 @@ struct CheckerInfo {
MPSCQueue<Entity *> entity_queue;
MPSCQueue<Entity *> required_global_variable_queue;
MPSCQueue<Entity *> required_foreign_imports_through_force_queue;
MPSCQueue<Entity *> foreign_imports_to_check_fullpaths;
MPSCQueue<Ast *> intrinsics_entry_point_usage;
@@ -434,6 +435,8 @@ struct CheckerInfo {
BlockingMutex load_directory_mutex;
StringMap<LoadDirectoryCache *> load_directory_cache;
PtrMap<Ast *, LoadDirectoryCache *> load_directory_map; // Key: Ast_CallExpr *
};
struct CheckerContext {

1
src/entity.cpp
View File

@@ -266,6 +266,7 @@ struct Entity {
Scope *scope;
} ImportName;
struct {
Ast *decl;
Slice<String> paths;
String name;
i64 priority_index;

77
src/parser.cpp
View File

@@ -1284,14 +1284,16 @@ gb_internal Ast *ast_import_decl(AstFile *f, Token token, Token relpath, Token i
return result;
}
gb_internal Ast *ast_foreign_import_decl(AstFile *f, Token token, Array<Token> filepaths, Token library_name,
CommentGroup *docs, CommentGroup *comment) {
gb_internal Ast *ast_foreign_import_decl(AstFile *f, Token token, Array<Ast *> filepaths, Token library_name,
bool multiple_filepaths,
CommentGroup *docs, CommentGroup *comment) {
Ast *result = alloc_ast_node(f, Ast_ForeignImportDecl);
result->ForeignImportDecl.token = token;
result->ForeignImportDecl.filepaths = slice_from_array(filepaths);
result->ForeignImportDecl.library_name = library_name;
result->ForeignImportDecl.docs = docs;
result->ForeignImportDecl.comment = comment;
result->ForeignImportDecl.multiple_filepaths = multiple_filepaths;
result->ForeignImportDecl.attributes.allocator = ast_allocator(f);
return result;
@@ -4882,14 +4884,17 @@ gb_internal Ast *parse_foreign_decl(AstFile *f) {
if (is_blank_ident(lib_name)) {
syntax_error(lib_name, "Illegal foreign import name: '_'");
}
Array<Token> filepaths = {};
bool multiple_filepaths = false;
Array<Ast *> filepaths = {};
if (allow_token(f, Token_OpenBrace)) {
multiple_filepaths = true;
array_init(&filepaths, ast_allocator(f));
while (f->curr_token.kind != Token_CloseBrace &&
f->curr_token.kind != Token_EOF) {
Token path = expect_token(f, Token_String);
Ast *path = parse_expr(f, false);
array_add(&filepaths, path);
if (!allow_field_separator(f)) {
@@ -4898,9 +4903,10 @@ gb_internal Ast *parse_foreign_decl(AstFile *f) {
}
expect_closing_brace_of_field_list(f);
} else {
filepaths = array_make<Token>(ast_allocator(f), 0, 1);
filepaths = array_make<Ast *>(ast_allocator(f), 0, 1);
Token path = expect_token(f, Token_String);
array_add(&filepaths, path);
Ast *lit = ast_basic_lit(f, path);
array_add(&filepaths, lit);
}
Ast *s = nullptr;
@@ -4909,9 +4915,9 @@ gb_internal Ast *parse_foreign_decl(AstFile *f) {
s = ast_bad_decl(f, lib_name, f->curr_token);
} else if (f->curr_proc != nullptr) {
syntax_error(lib_name, "You cannot use foreign import within a procedure. This must be done at the file scope");
s = ast_bad_decl(f, lib_name, filepaths[0]);
s = ast_bad_decl(f, lib_name, ast_token(filepaths[0]));
} else {
s = ast_foreign_import_decl(f, token, filepaths, lib_name, docs, f->line_comment);
s = ast_foreign_import_decl(f, token, filepaths, lib_name, multiple_filepaths, docs, f->line_comment);
}
expect_semicolon(f);
return s;
@@ -5648,9 +5654,19 @@ gb_internal bool is_package_name_reserved(String const &name) {
}
gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node, String base_dir, String const &original_string, String *path) {
gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node, String base_dir, String const &original_string, String *path, bool use_check_errors=false) {
GB_ASSERT(path != nullptr);
void (*do_error)(Ast *, char const *, ...);
void (*do_warning)(Token const &, char const *, ...);
do_error = &syntax_error;
do_warning = &syntax_warning;
if (use_check_errors) {
do_error = &error;
do_error = &warning;
}
// NOTE(bill): if file_mutex == nullptr, this means that the code is used within the semantics stage
String collection_name = {};
@@ -5677,7 +5693,7 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
String file_str = {};
if (colon_pos == 0) {
syntax_error(node, "Expected a collection name");
do_error(node, "Expected a collection name");
return false;
}
@@ -5692,11 +5708,11 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
if (has_windows_drive) {
String sub_file_path = substring(file_str, 3, file_str.len);
if (!is_import_path_valid(sub_file_path)) {
syntax_error(node, "Invalid import path: '%.*s'", LIT(file_str));
do_error(node, "Invalid import path: '%.*s'", LIT(file_str));
return false;
}
} else if (!is_import_path_valid(file_str)) {
syntax_error(node, "Invalid import path: '%.*s'", LIT(file_str));
do_error(node, "Invalid import path: '%.*s'", LIT(file_str));
return false;
}
@@ -5718,16 +5734,16 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
}
if (replace_with_base) {
if (ast_file_vet_deprecated(node->file())) {
syntax_error(node, "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
do_error(node, "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
} else {
syntax_warning(ast_token(node), "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
do_warning(ast_token(node), "import \"core:%.*s\" has been deprecated in favour of \"base:%.*s\"", LIT(file_str), LIT(file_str));
}
}
}
if (collection_name == "system") {
if (node->kind != Ast_ForeignImportDecl) {
syntax_error(node, "The library collection 'system' is restrict for 'foreign_library'");
do_error(node, "The library collection 'system' is restrict for 'foreign import'");
return false;
} else {
*path = file_str;
@@ -5735,7 +5751,7 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
}
} else if (!find_library_collection_path(collection_name, &base_dir)) {
// NOTE(bill): It's a naughty name
syntax_error(node, "Unknown library collection: '%.*s'", LIT(collection_name));
do_error(node, "Unknown library collection: '%.*s'", LIT(collection_name));
return false;
}
} else {
@@ -5759,7 +5775,7 @@ gb_internal bool determine_path_from_string(BlockingMutex *file_mutex, Ast *node
if (collection_name == "core" || collection_name == "base") {
return true;
} else {
syntax_error(node, "The package '%.*s' must be imported with the 'base' library collection: 'base:%.*s'", LIT(file_str), LIT(file_str));
do_error(node, "The package '%.*s' must be imported with the 'base' library collection: 'base:%.*s'", LIT(file_str), LIT(file_str));
return false;
}
}
@@ -5844,30 +5860,29 @@ gb_internal void parse_setup_file_decls(Parser *p, AstFile *f, String const &bas
} else if (node->kind == Ast_ForeignImportDecl) {
ast_node(fl, ForeignImportDecl, node);
auto fullpaths = array_make<String>(permanent_allocator(), 0, fl->filepaths.count);
for (Token const &fp : fl->filepaths) {
String file_str = string_trim_whitespace(string_value_from_token(f, fp));
if (fl->filepaths.count == 0) {
syntax_error(decls[i], "No foreign paths found");
decls[i] = ast_bad_decl(f, ast_token(fl->filepaths[0]), ast_end_token(fl->filepaths[fl->filepaths.count-1]));
goto end;
} else if (!fl->multiple_filepaths &&
fl->filepaths.count == 1) {
Ast *fp = fl->filepaths[0];
GB_ASSERT(fp->kind == Ast_BasicLit);
Token fp_token = fp->BasicLit.token;
String file_str = string_trim_whitespace(string_value_from_token(f, fp_token));
String fullpath = file_str;
if (allow_check_foreign_filepath()) {
String foreign_path = {};
bool ok = determine_path_from_string(&p->file_decl_mutex, node, base_dir, file_str, &foreign_path);
if (!ok) {
decls[i] = ast_bad_decl(f, fp, fl->filepaths[fl->filepaths.count-1]);
decls[i] = ast_bad_decl(f, fp_token, fp_token);
goto end;
}
fullpath = foreign_path;
}
array_add(&fullpaths, fullpath);
fl->fullpaths = slice_make<String>(permanent_allocator(), 1);
fl->fullpaths[0] = fullpath;
}
if (fullpaths.count == 0) {
syntax_error(decls[i], "No foreign paths found");
decls[i] = ast_bad_decl(f, fl->filepaths[0], fl->filepaths[fl->filepaths.count-1]);
goto end;
}
fl->fullpaths = slice_from_array(fullpaths);
} else if (node->kind == Ast_WhenStmt) {
ast_node(ws, WhenStmt, node);

3
src/parser.hpp
View File

@@ -631,7 +631,8 @@ AST_KIND(_DeclBegin, "", bool) \
}) \
AST_KIND(ForeignImportDecl, "foreign import declaration", struct { \
Token token; \
Slice<Token> filepaths; \
Slice<Ast *> filepaths; \
bool multiple_filepaths; \
Token library_name; \
String collection_name; \
Slice<String> fullpaths; \

2
src/parser_pos.cpp
View File

@@ -278,7 +278,7 @@ Token ast_end_token(Ast *node) {
case Ast_ImportDecl: return node->ImportDecl.relpath;
case Ast_ForeignImportDecl:
if (node->ForeignImportDecl.filepaths.count > 0) {
return node->ForeignImportDecl.filepaths[node->ForeignImportDecl.filepaths.count-1];
return ast_end_token(node->ForeignImportDecl.filepaths[node->ForeignImportDecl.filepaths.count-1]);
}
if (node->ForeignImportDecl.library_name.kind != Token_Invalid) {
return node->ForeignImportDecl.library_name;