From b7858a66b94e87830c0c777547c4f411c4d8d259 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Mon, 28 May 2018 12:06:50 +0100 Subject: [PATCH] Parallelize per file rather than per package --- src/check_decl.cpp | 4 +- src/checker.cpp | 54 ++++--- src/checker.hpp | 3 +- src/entity.cpp | 2 +- src/ir.cpp | 10 +- src/main.cpp | 5 +- src/parser.cpp | 345 +++++++++++++++++++++------------------------ src/parser.hpp | 20 ++- 8 files changed, 221 insertions(+), 222 deletions(-) diff --git a/src/check_decl.cpp b/src/check_decl.cpp index 7a85175d8..64f59269d 100644 --- a/src/check_decl.cpp +++ b/src/check_decl.cpp @@ -534,7 +534,7 @@ void check_proc_decl(Checker *c, Entity *e, DeclInfo *d) { e->deprecated_message = ac.deprecated_message; ac.link_name = handle_link_name(c, e->token, ac.link_name, ac.link_prefix); - if (e->package != nullptr && e->token.string == "main") { + if (e->pkg != nullptr && e->token.string == "main") { if (pt->param_count != 0 || pt->result_count != 0) { gbString str = type_to_string(proc_type); @@ -546,7 +546,7 @@ void check_proc_decl(Checker *c, Entity *e, DeclInfo *d) { error(e->token, "Procedure 'main' cannot have a custom calling convention"); } pt->calling_convention = ProcCC_Contextless; - if (e->package->kind == Package_Init) { + if (e->pkg->kind == Package_Init) { if (c->info.entry_point != nullptr) { error(e->token, "Redeclaration of the entry pointer procedure 'main'"); } else { diff --git a/src/checker.cpp b/src/checker.cpp index c70256b45..6e20545ff 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -212,13 +212,13 @@ bool decl_info_has_init(DeclInfo *d) { -Scope *create_scope(Scope *parent, gbAllocator allocator) { +Scope *create_scope(Scope *parent, gbAllocator allocator, isize init_elements_capacity=16) { Scope *s = gb_alloc_item(allocator, Scope); s->parent = parent; - map_init(&s->elements, heap_allocator()); - ptr_set_init(&s->implicit, heap_allocator()); - ptr_set_init(&s->imported, heap_allocator()); - ptr_set_init(&s->exported, heap_allocator()); + map_init(&s->elements, heap_allocator(), init_elements_capacity); + ptr_set_init(&s->implicit, heap_allocator(), 0); + ptr_set_init(&s->imported, heap_allocator(), 0); + ptr_set_init(&s->exported, heap_allocator(), 0); s->delayed_imports.allocator = heap_allocator(); s->delayed_asserts.allocator = heap_allocator(); @@ -231,10 +231,13 @@ Scope *create_scope(Scope *parent, gbAllocator allocator) { Scope *create_scope_from_file(Checker *c, AstFile *f) { GB_ASSERT(f != nullptr); - GB_ASSERT(f->package != nullptr); - GB_ASSERT(f->package->scope != nullptr); + GB_ASSERT(f->pkg != nullptr); + GB_ASSERT(f->pkg->scope != nullptr); - Scope *s = create_scope(f->package->scope, c->allocator); + Scope *s = create_scope(f->pkg->scope, c->allocator); + + array_reserve(&s->delayed_imports, f->imports.count); + array_reserve(&s->delayed_asserts, f->assert_decl_count); s->is_file = true; s->file = f; @@ -246,7 +249,13 @@ Scope *create_scope_from_file(Checker *c, AstFile *f) { Scope *create_scope_from_package(Checker *c, AstPackage *p) { GB_ASSERT(p != nullptr); - Scope *s = create_scope(universal_scope, c->allocator); + isize decl_count = 0; + for_array(i, p->files) { + decl_count += p->files[i]->decls.count; + } + isize init_elements_capacity = 2*decl_count; + + Scope *s = create_scope(universal_scope, c->allocator, init_elements_capacity); s->is_package = true; s->package = p; @@ -658,7 +667,9 @@ void init_checker(Checker *c, Parser *parser) { // c->allocator = gb_arena_allocator(&c->arena); c->tmp_allocator = gb_arena_allocator(&c->tmp_arena); - map_init(&c->package_scopes, heap_allocator()); + isize pkg_cap = 2*c->parser->packages.count; + + map_init(&c->package_scopes, heap_allocator(), pkg_cap); array_init(&c->package_order, heap_allocator(), 0, c->parser->packages.count); @@ -943,10 +954,10 @@ void add_entity_and_decl_info(Checker *c, AstNode *identifier, Entity *e, DeclIn // NOTE(bill): Entities local to file rather than package break; default: { - AstPackage *p = scope->file->package; - GB_ASSERT(p->scope == scope->parent); - GB_ASSERT(c->context.package == p); - scope = p->scope; + AstPackage *pkg = scope->file->pkg; + GB_ASSERT(pkg->scope == scope->parent); + GB_ASSERT(c->context.pkg == pkg); + scope = pkg->scope; break; } } @@ -959,7 +970,7 @@ void add_entity_and_decl_info(Checker *c, AstNode *identifier, Entity *e, DeclIn e->decl_info = d; array_add(&c->info.entities, e); e->order_in_src = c->info.entities.count; - e->package = c->context.package; + e->pkg = c->context.pkg; } @@ -1161,11 +1172,10 @@ void add_curr_ast_file(Checker *c, AstFile *file) { if (file != nullptr) { TokenPos zero_pos = {}; global_error_collector.prev = zero_pos; - c->curr_ast_file = file; - c->context.decl = file->package->decl_info; - c->context.scope = file->scope; - c->context.package = file->package; - c->context.package_scope = file->package->scope; + c->curr_ast_file = file; + c->context.decl = file->pkg->decl_info; + c->context.scope = file->scope; + c->context.pkg = file->pkg; } } @@ -2239,7 +2249,7 @@ void check_all_global_entities(Checker *c) { GB_ASSERT(d->scope->is_file); AstFile *file = d->scope->file; add_curr_ast_file(c, file); - Scope *package_scope = file->package->scope; + Scope *package_scope = file->pkg->scope; if (e->token.string == "main") { if (e->kind != Entity_Procedure) { @@ -2338,7 +2348,7 @@ String path_to_entity_name(String name, String fullpath) { #if 1 void add_import_dependency_node(Checker *c, AstNode *decl, Map *M) { - Scope *parent_package_scope = decl->file->package->scope; + Scope *parent_package_scope = decl->file->pkg->scope; switch (decl->kind) { case_ast_node(id, ImportDecl, decl); diff --git a/src/checker.hpp b/src/checker.hpp index 705f32116..896231a4c 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -274,8 +274,7 @@ struct ForeignContext { typedef Array CheckerTypePath; struct CheckerContext { - AstPackage * package; - Scope * package_scope; + AstPackage * pkg; Scope * scope; DeclInfo * decl; u32 stmt_state_flags; diff --git a/src/entity.cpp b/src/entity.cpp index 31081e5ce..aea01d6ac 100644 --- a/src/entity.cpp +++ b/src/entity.cpp @@ -76,7 +76,7 @@ struct Entity { AstNode * identifier; // Can be nullptr DeclInfo * decl_info; DeclInfo * parent_proc_decl; // nullptr if in file/global scope - AstPackage *package; + AstPackage *pkg; // TODO(bill): Cleanup how `using` works for entities Entity * using_parent; diff --git a/src/ir.cpp b/src/ir.cpp index 9b9d6d3bf..87525028d 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -1575,8 +1575,8 @@ void ir_emit_zero_init(irProcedure *p, irValue *address, AstNode *expr) { auto args = array_make(a, 2); args[0] = ir_emit_conv(p, address, t_rawptr); args[1] = ir_const_int(a, type_size_of(t)); - AstPackage *package = get_core_package(p->module->info, str_lit("mem")); - if (p->entity->token.string != "zero" && p->entity->package != package) { + AstPackage *pkg = get_core_package(p->module->info, str_lit("mem")); + if (p->entity->token.string != "zero" && p->entity->pkg != pkg) { ir_emit_package_call(p, "mem", "zero", args, expr); } ir_emit(p, ir_instr_zero_init(p, address)); @@ -3833,8 +3833,8 @@ String ir_mangle_name(irGen *s, Entity *e) { return make_string(new_name, new_name_len-1); #else - GB_ASSERT(e->package != nullptr); - String pkg = e->package->name; + GB_ASSERT(e->pkg != nullptr); + String pkg = e->pkg->name; GB_ASSERT(!rune_is_digit(pkg[0])); String name = e->token.string; @@ -8310,7 +8310,7 @@ void ir_gen_tree(irGen *s) { Entity *e = info->entities[i]; String name = e->token.string; - bool is_global = e->package != nullptr; + bool is_global = e->pkg != nullptr; if (e->kind == Entity_Variable) { global_variable_max_count++; diff --git a/src/main.cpp b/src/main.cpp index 4dd8b3983..703fa42a9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -580,7 +580,9 @@ void show_timings(Checker *c, Timings *t) { for_array(i, p->packages) { files += p->packages[i]->files.count; } - +#if 1 + timings_print_all(t); +#else { timings_print_all(t); gb_printf("\n"); @@ -610,6 +612,7 @@ void show_timings(Checker *c, Timings *t) { gb_printf("us/Token - %.3f\n", 1.0e6*total_time/cast(f64)tokens); gb_printf("\n"); } +#endif } void remove_temp_files(String output_base) { diff --git a/src/parser.cpp b/src/parser.cpp index 4f35568d7..b1b1f8e05 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -3885,10 +3885,11 @@ void destroy_ast_file(AstFile *f) { bool init_parser(Parser *p) { GB_ASSERT(p != nullptr); - map_init(&p->imported_files, heap_allocator()); + string_set_init(&p->imported_files, heap_allocator()); map_init(&p->package_map, heap_allocator()); array_init(&p->packages, heap_allocator()); - array_init(&p->imports, heap_allocator()); + array_init(&p->package_imports, heap_allocator()); + array_init(&p->files_to_process, heap_allocator()); gb_mutex_init(&p->file_add_mutex); gb_mutex_init(&p->file_decl_mutex); return true; @@ -3898,47 +3899,144 @@ void destroy_parser(Parser *p) { GB_ASSERT(p != nullptr); // TODO(bill): Fix memory leak for_array(i, p->packages) { - AstPackage *package = p->packages[i]; - for_array(j, package->files) { - destroy_ast_file(package->files[j]); + AstPackage *pkg = p->packages[i]; + for_array(j, pkg->files) { + destroy_ast_file(pkg->files[j]); } - array_free(&package->files); + array_free(&pkg->files); } #if 0 - for_array(i, p->imports) { - // gb_free(heap_allocator(), p->imports[i].text); + for_array(i, p->package_imports) { + // gb_free(heap_allocator(), p->package_imports[i].text); } #endif array_free(&p->packages); - array_free(&p->imports); - map_destroy(&p->imported_files); + array_free(&p->package_imports); + array_free(&p->files_to_process); + string_set_destroy(&p->imported_files); map_destroy(&p->package_map); gb_mutex_destroy(&p->file_add_mutex); gb_mutex_destroy(&p->file_decl_mutex); } + +void parser_add_package(Parser *p, AstPackage *pkg) { + pkg->id = p->packages.count+1; + array_add(&p->packages, pkg); + if (pkg->name.len > 0) { + HashKey key = hash_string(pkg->name); + auto found = map_get(&p->package_map, key); + if (found) { + GB_ASSERT(pkg->files.count > 0); + AstFile *f = pkg->files[0]; + error(f->package_token, "Non-unique package name '%.*s'", LIT(pkg->name)); + GB_ASSERT((*found)->files.count > 0); + TokenPos pos = (*found)->files[0]->package_token.pos; + gb_printf_err("\tpreviously declared at %.*s(%td:%td)", LIT(pos.file), pos.line, pos.column); + } else { + map_set(&p->package_map, key, pkg); + } + } +} + +void parser_add_file_to_process(Parser *p, AstPackage *pkg, FileInfo fi, TokenPos pos) { + ImportedFile f = {pkg, fi, pos, p->files_to_process.count}; + array_add(&p->files_to_process, f); +} + + // NOTE(bill): Returns true if it's added -bool try_add_import_path(Parser *p, String path, String rel_path, TokenPos pos, PackageKind kind = Package_Normal) { +bool try_add_import_path(Parser *p, String const &path, String const &rel_path, TokenPos pos, PackageKind kind = Package_Normal) { if (build_context.generate_docs) { return false; } - path = string_trim_whitespace(path); - rel_path = string_trim_whitespace(rel_path); + String const FILE_EXT = str_lit(".odin"); - HashKey key = hash_string(path); - if (map_get(&p->imported_files, key) != nullptr) { + gb_mutex_lock(&p->file_add_mutex); + defer (gb_mutex_unlock(&p->file_add_mutex)); + + if (string_set_exists(&p->imported_files, path)) { return false; } - map_set(&p->imported_files, key, true); + string_set_add(&p->imported_files, path); - ImportedPackage item = {}; - item.kind = kind; - item.path = path; - item.rel_path = rel_path; - item.pos = pos; - item.index = p->imports.count; - array_add(&p->imports, item); + + AstPackage *pkg = gb_alloc_item(heap_allocator(), AstPackage); + pkg->kind = kind; + pkg->fullpath = path; + array_init(&pkg->files, heap_allocator()); + + // NOTE(bill): Single file initial package + if (kind == Package_Init && string_ends_with(path, FILE_EXT)) { + FileInfo fi = {}; + fi.name = filename_from_path(path); + fi.fullpath = path; + fi.size = get_file_size(path); + fi.is_dir = false; + + parser_add_file_to_process(p, pkg, fi, pos); + parser_add_package(p, pkg); + return true; + } + + + Array list = {}; + ReadDirectoryError rd_err = read_directory(path, &list); + defer (array_free(&list)); + + if (list.count == 1) { + GB_ASSERT(path != list[0].fullpath); + } + + + if (rd_err != ReadDirectory_None) { + if (pos.line != 0) { + gb_printf_err("%.*s(%td:%td) ", LIT(pos.file), pos.line, pos.column); + } + gb_mutex_lock(&global_error_collector.mutex); + defer (gb_mutex_unlock(&global_error_collector.mutex)); + global_error_collector.count++; + + + switch (rd_err) { + case ReadDirectory_InvalidPath: + gb_printf_err("Invalid path: %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_NotExists: + gb_printf_err("Path does not exist: %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_NotDir: + gb_printf_err("Expected a directory for a package, got a file: %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_Unknown: + gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(rel_path)); + return false; + case ReadDirectory_Permission: + gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(rel_path)); + return false; + + case ReadDirectory_Empty: + gb_printf_err("Empty directory: %.*s\n", LIT(rel_path)); + return false; + } + } + + for_array(list_index, list) { + FileInfo fi = list[list_index]; + String name = fi.name; + if (string_ends_with(name, FILE_EXT)) { + if (is_excluded_target_filename(name)) { + continue; + } + parser_add_file_to_process(p, pkg, fi, pos); + } + } + + parser_add_package(p, pkg); return true; } @@ -4074,7 +4172,8 @@ void parse_setup_file_decls(Parser *p, AstFile *f, String base_dir, Arraykind == AstNode_CallExpr && expr->CallExpr.proc->kind == AstNode_BasicDirective && expr->CallExpr.proc->BasicDirective.name == "assert") { - // NOTE(bill): Okay! + + f->assert_decl_count += 1; continue; } } @@ -4083,13 +4182,14 @@ void parse_setup_file_decls(Parser *p, AstFile *f, String base_dir, Arraykind == AstNode_ImportDecl) { ast_node(id, ImportDecl, node); - String original_string = id->relpath.string; + String original_string = string_trim_whitespace(id->relpath.string); String import_path = {}; bool ok = determine_path_from_string(p, node, base_dir, original_string, &import_path); if (!ok) { decls[i] = ast_bad_decl(f, id->relpath, id->relpath); continue; } + import_path = string_trim_whitespace(import_path); id->fullpath = import_path; try_add_import_path(p, import_path, original_string, ast_node_token(node).pos); @@ -4141,7 +4241,7 @@ bool parse_file(Parser *p, AstFile *f) { if (package_name.kind == Token_Ident) { if (package_name.string == "_") { error(package_name, "Invalid package name '_'"); - } else if (f->package->kind != Package_Runtime && package_name.string == "runtime") { + } else if (f->pkg->kind != Package_Runtime && package_name.string == "runtime") { error(package_name, "Use of reserved package name '%.*s'", LIT(package_name.string)); } } @@ -4161,17 +4261,24 @@ bool parse_file(Parser *p, AstFile *f) { } -ParseFileError parse_imported_file(Parser *p, AstPackage *package, FileInfo *fi, TokenPos pos) { - AstFile *file = gb_alloc_item(heap_allocator(), AstFile); - file->package = package; +ParseFileError process_imported_file(Parser *p, ImportedFile imported_file) { + AstPackage *pkg = imported_file.pkg; + FileInfo *fi = &imported_file.fi; + TokenPos pos = imported_file.pos; - p->file_index += 1; - file->id = p->file_index; + AstFile *file = gb_alloc_item(heap_allocator(), AstFile); + file->pkg = pkg; + + file->id = imported_file.index+1; TokenPos err_pos = {0}; ParseFileError err = init_ast_file(file, fi->fullpath, &err_pos); if (err != ParseFile_None) { + gb_mutex_lock(&global_error_collector.mutex); + defer (gb_mutex_unlock(&global_error_collector.mutex)); + global_error_collector.count++; + if (err == ParseFile_EmptyFile) { if (fi->fullpath == p->init_fullpath) { gb_printf_err("Initial file is empty - %.*s\n", LIT(p->init_fullpath)); @@ -4206,9 +4313,6 @@ ParseFileError parse_imported_file(Parser *p, AstPackage *package, FileInfo *fi, } gb_printf_err("\n"); - gb_mutex_lock(&global_error_collector.mutex); - global_error_collector.count++; - gb_mutex_unlock(&global_error_collector.mutex); return err; } @@ -4217,164 +4321,39 @@ ParseFileError parse_imported_file(Parser *p, AstPackage *package, FileInfo *fi, skip: if (parse_file(p, file)) { gb_mutex_lock(&p->file_add_mutex); - array_add(&package->files, file); + defer (gb_mutex_unlock(&p->file_add_mutex)); - if (package->name.len == 0) { - package->name = file->package_name; - } else if (file->tokens.count > 0 && package->name != file->package_name) { - error(file->package_token, "Different package name, expected '%.*s', got '%.*s'", LIT(package->name), LIT(file->package_name)); + array_add(&pkg->files, file); + + if (pkg->name.len == 0) { + pkg->name = file->package_name; + } else if (file->tokens.count > 0 && pkg->name != file->package_name) { + error(file->package_token, "Different package name, expected '%.*s', got '%.*s'", LIT(pkg->name), LIT(file->package_name)); } p->total_line_count += file->tokenizer.line_count; p->total_token_count += file->tokens.count; - gb_mutex_unlock(&p->file_add_mutex); } return ParseFile_None; } -void parser_add_package(Parser *p, AstPackage *package) { - package->id = p->packages.count+1; - array_add(&p->packages, package); - if (package->name.len > 0) { - HashKey key = hash_string(package->name); - auto found = map_get(&p->package_map, key); - if (found) { - GB_ASSERT(package->files.count > 0); - AstFile *f = package->files[0]; - error(f->package_token, "Non-unique package name '%.*s'", LIT(package->name)); - GB_ASSERT((*found)->files.count > 0); - TokenPos pos = (*found)->files[0]->package_token.pos; - gb_printf_err("\tpreviously declared at %.*s(%td:%td)", LIT(pos.file), pos.line, pos.column); - } else { - map_set(&p->package_map, key, package); - } - } -} - -ParseFileError parse_import(Parser *p, ImportedPackage imported_package) { - String import_path = imported_package.path; - String import_rel_path = imported_package.rel_path; - TokenPos pos = imported_package.pos; - String const file_ext = str_lit(".odin"); - - // NOTE(bill): Single file initial package - if (imported_package.kind == Package_Init && string_ends_with(import_path, file_ext)) { - AstPackage *package = gb_alloc_item(heap_allocator(), AstPackage); - package->kind = imported_package.kind; - package->fullpath = import_path; - array_init(&package->files, heap_allocator()); - - FileInfo fi = {}; - fi.name = filename_from_path(import_path); - fi.fullpath = import_path; - fi.size = get_file_size(import_path); - fi.is_dir = false; - - ParseFileError err = parse_imported_file(p, package, &fi, pos); - if (err != ParseFile_None) { - return err; - } - - parser_add_package(p, package); - - return ParseFile_None; - } - - - Array list = {}; - ReadDirectoryError rd_err = read_directory(import_path, &list); - defer (array_free(&list)); - - if (list.count == 1) { - GB_ASSERT(import_path != list[0].fullpath); - } - - - if (rd_err != ReadDirectory_None) { - if (pos.line != 0) { - gb_printf_err("%.*s(%td:%td) ", LIT(pos.file), pos.line, pos.column); - } - gb_mutex_lock(&global_error_collector.mutex); - defer (gb_mutex_unlock(&global_error_collector.mutex)); - global_error_collector.count++; - - - switch (rd_err) { - case ReadDirectory_InvalidPath: - gb_printf_err("Invalid path: %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - - case ReadDirectory_NotExists: - gb_printf_err("Path does not exist: %.*s\n", LIT(import_rel_path)); - return ParseFile_NotFound; - - case ReadDirectory_NotDir: - gb_printf_err("Expected a directory for a package, got a file: %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - - case ReadDirectory_Unknown: - gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - case ReadDirectory_Permission: - gb_printf_err("Unknown error whilst reading path %.*s\n", LIT(import_rel_path)); - return ParseFile_InvalidFile; - - case ReadDirectory_Empty: - gb_printf_err("Empty directory: %.*s\n", LIT(import_rel_path)); - return ParseFile_EmptyFile; - } - } - - AstPackage *package = gb_alloc_item(heap_allocator(), AstPackage); - package->kind = imported_package.kind; - package->fullpath = import_path; - array_init(&package->files, heap_allocator()); - - // TODO(bill): Fix concurrency - for_array(list_index, list) { - FileInfo *fi = &list[list_index]; - String name = fi->name; - if (string_ends_with(name, file_ext)) { - if (is_excluded_target_filename(name)) { - continue; - } - ParseFileError err = parse_imported_file(p, package, fi, pos); - if (err != ParseFile_None) { - return err; - } - } - } - - parser_add_package(p, package); - - return ParseFile_None; -} - GB_THREAD_PROC(parse_worker_file_proc) { if (thread == nullptr) return 0; auto *p = cast(Parser *)thread->user_data; isize index = thread->user_index; - ImportedPackage imported_package = p->imports[index]; - ParseFileError err = parse_import(p, imported_package); + ParseFileError err = process_imported_file(p, p->files_to_process[index]); return cast(isize)err; } +void add_shared_package(Parser *p, String name, TokenPos pos) { -struct ParserThreadWork { - Parser *parser; - isize import_index; -}; - -void add_shared_package(Parser *p, String name, TokenPos pos, PackageKind kind) { - String s = get_fullpath_core(heap_allocator(), name); - try_add_import_path(p, s, s, pos, kind); } ParseFileError parse_packages(Parser *p, String init_filename) { GB_ASSERT(init_filename.text[init_filename.len] == 0); - char *fullpath_str = gb_path_get_full_name(heap_allocator(), cast(char *)&init_filename[0]); + char *fullpath_str = gb_path_get_full_name(heap_allocator(), cast(char const *)&init_filename[0]); String init_fullpath = string_trim_whitespace(make_string_c(fullpath_str)); if (!path_is_directory(init_fullpath)) { String const ext = str_lit(".odin"); @@ -4385,25 +4364,24 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } TokenPos init_pos = {}; - ImportedPackage init_imported_package = {Package_Init, init_fullpath, init_fullpath, init_pos}; - isize shared_package_count = 0; if (!build_context.generate_docs) { - add_shared_package(p, str_lit("runtime"), init_pos, Package_Runtime); shared_package_count++; + String s = get_fullpath_core(heap_allocator(), str_lit("runtime")); + try_add_import_path(p, s, s, init_pos, Package_Runtime); } - array_add(&p->imports, init_imported_package); + try_add_import_path(p, init_fullpath, init_fullpath, init_pos, Package_Init); p->init_fullpath = init_fullpath; // IMPORTANT TODO(bill): Figure out why this doesn't work on *nix sometimes -#if defined(GB_SYSTEM_WINDOWS) +#if 1 && defined(GB_SYSTEM_WINDOWS) isize thread_count = gb_max(build_context.thread_count, 1); if (thread_count > 1) { isize volatile curr_import_index = 0; - + isize initial_file_count = p->files_to_process.count; // NOTE(bill): Make sure that these are in parsed in this order - for (isize i = 0; i < shared_package_count; i++) { - ParseFileError err = parse_import(p, p->imports[i]); + for (isize i = 0; i < initial_file_count; i++) { + ParseFileError err = process_imported_file(p, p->files_to_process[i]); if (err != ParseFile_None) { return err; } @@ -4429,7 +4407,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { gbThread *t = &worker_threads[i]; if (gb_thread_is_running(t)) { are_any_alive = true; - } else if (curr_import_index < p->imports.count) { + } else if (curr_import_index < p->files_to_process.count) { auto curr_err = cast(ParseFileError)t->return_value; if (curr_err != ParseFile_None) { array_add(&errors, curr_err); @@ -4441,7 +4419,7 @@ ParseFileError parse_packages(Parser *p, String init_filename) { } } } - if (!are_any_alive && curr_import_index >= p->imports.count) { + if (!are_any_alive && curr_import_index >= p->files_to_process.count) { break; } } @@ -4450,16 +4428,17 @@ ParseFileError parse_packages(Parser *p, String init_filename) { return errors[errors.count-1]; } } else { - for_array(i, p->imports) { - ParseFileError err = parse_import(p, p->imports[i]); + for_array(i, p->files_to_process) { + ParseFileError err = process_imported_file(p, p->files_to_process[i]); if (err != ParseFile_None) { return err; } } } #else - for_array(i, p->imports) { - ParseFileError err = parse_import(p, p->imports[i]); + for_array(i, p->files_to_process) { + ImportedFile f = p->files_to_process[i]; + ParseFileError err = process_imported_file(p, f); if (err != ParseFile_None) { return err; } diff --git a/src/parser.hpp b/src/parser.hpp index ce0eed350..2f88240ce 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -39,9 +39,17 @@ struct ImportedPackage { isize index; }; + +struct ImportedFile { + AstPackage *pkg; + FileInfo fi; + TokenPos pos; // import + isize index; +}; + struct AstFile { isize id; - AstPackage * package; + AstPackage * pkg; Scope * scope; String fullpath; @@ -65,10 +73,10 @@ struct AstFile { Array decls; Array imports; // 'import' 'using import' + isize assert_decl_count; AstNode * curr_proc; - isize scope_level; // DeclInfo * decl_info; // NOTE(bill): Created in checker isize error_count; @@ -85,8 +93,8 @@ struct AstFile { struct AstPackage { - isize id; PackageKind kind; + isize id; String name; String fullpath; Array files; @@ -98,15 +106,15 @@ struct AstPackage { struct Parser { String init_fullpath; - Map imported_files; // Key: String (fullpath) + StringSet imported_files; // fullpath Map package_map; // Key: String (package name) Array packages; - Array imports; + Array package_imports; + Array files_to_process; isize total_token_count; isize total_line_count; gbMutex file_add_mutex; gbMutex file_decl_mutex; - isize file_index; }; enum ProcInlining {