diff --git a/src/build_settings.cpp b/src/build_settings.cpp index c8c83422f..28ca0f088 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -326,7 +326,17 @@ enum SanitizerFlags : u32 { SanitizerFlag_Thread = 1u<<2, }; +struct BuildCacheData { + u64 crc; + String cache_dir; + // manifests + String files_path; + String args_path; + String env_path; + + bool copy_already_done; +}; // This stores the information for the specify architecture of this build struct BuildContext { @@ -427,6 +437,9 @@ struct BuildContext { bool use_separate_modules; bool module_per_file; + bool cached; + BuildCacheData build_cache_data; + bool no_threaded_checker; bool show_debug_messages; diff --git a/src/cached.cpp b/src/cached.cpp new file mode 100644 index 000000000..02cb1ae96 --- /dev/null +++ b/src/cached.cpp @@ -0,0 +1,449 @@ +gb_internal GB_COMPARE_PROC(string_cmp) { + String const &x = *(String *)a; + String const &y = *(String *)b; + return string_compare(x, y); +} + +gb_internal bool recursively_delete_directory(wchar_t *wpath_c) { +#if defined(GB_SYSTEM_WINDOWS) + auto const is_dots_w = [](wchar_t const *str) -> bool { + if (!str) { + return false; + } + return wcscmp(str, L".") == 0 || wcscmp(str, L"..") == 0; + }; + + TEMPORARY_ALLOCATOR_GUARD(); + + wchar_t dir_path[MAX_PATH] = {}; + wchar_t filename[MAX_PATH] = {}; + wcscpy(dir_path, wpath_c); + wcscat(dir_path, L"\\*"); + + wcscpy(filename, wpath_c); + wcscat(filename, L"\\"); + + + WIN32_FIND_DATAW find_file_data = {}; + HANDLE hfind = FindFirstFileW(dir_path, &find_file_data); + if (hfind == INVALID_HANDLE_VALUE) { + return false; + } + defer (FindClose(hfind)); + + wcscpy(dir_path, filename); + + for (;;) { + if (FindNextFileW(hfind, &find_file_data)) { + if (is_dots_w(find_file_data.cFileName)) { + continue; + } + wcscat(filename, find_file_data.cFileName); + + if (find_file_data.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { + if (!recursively_delete_directory(filename)) { + return false; + } + RemoveDirectoryW(filename); + wcscpy(filename, dir_path); + } else { + if (find_file_data.dwFileAttributes & FILE_ATTRIBUTE_READONLY) { + _wchmod(filename, _S_IWRITE); + } + if (!DeleteFileW(filename)) { + return false; + } + wcscpy(filename, dir_path); + } + } else { + if (GetLastError() == ERROR_NO_MORE_FILES) { + break; + } + return false; + } + } + + + return RemoveDirectoryW(wpath_c); +#else + return false; +#endif +} + +gb_internal bool recursively_delete_directory(String const &path) { +#if defined(GB_SYSTEM_WINDOWS) + String16 wpath = string_to_string16(permanent_allocator(), path); + wchar_t *wpath_c = alloc_wstring(permanent_allocator(), wpath); + return recursively_delete_directory(wpath_c); +#else + return false; +#endif +} + +gb_internal bool try_clear_cache(void) { + return recursively_delete_directory(str_lit(".odin-cache")); +} + + +gb_internal u64 crc64_with_seed(void const *data, isize len, u64 seed) { + isize remaining; + u64 result = ~seed; + u8 const *c = cast(u8 const *)data; + for (remaining = len; remaining--; c++) { + result = (result >> 8) ^ (GB__CRC64_TABLE[(result ^ *c) & 0xff]); + } + return ~result; +} + +gb_internal bool check_if_exists_file_otherwise_create(String const &str) { + char const *str_c = alloc_cstring(permanent_allocator(), str); + if (!gb_file_exists(str_c)) { + gbFile f = {}; + gb_file_create(&f, str_c); + gb_file_close(&f); + return true; + } + return false; +} + + +gb_internal bool check_if_exists_directory_otherwise_create(String const &str) { +#if defined(GB_SYSTEM_WINDOWS) + String16 wstr = string_to_string16(permanent_allocator(), str); + wchar_t *wstr_c = alloc_wstring(permanent_allocator(), wstr); + return CreateDirectoryW(wstr_c, nullptr); +#else + char const *str_c = alloc_cstring(permanent_allocator(), str); + if (!gb_file_exists(str_c)) { + int status = mkdir(str_c, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + return status == 0; + } + return false; +#endif +} +gb_internal bool try_copy_executable_cache_internal(bool to_cache) { + String exe_name = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_Output]); + defer (gb_free(heap_allocator(), exe_name.text)); + + gbString cache_name = gb_string_make(heap_allocator(), ""); + defer (gb_string_free(cache_name)); + + String cache_dir = build_context.build_cache_data.cache_dir; + + cache_name = gb_string_append_length(cache_name, cache_dir.text, cache_dir.len); + cache_name = gb_string_appendc(cache_name, "/"); + + cache_name = gb_string_appendc(cache_name, "cached-exe"); + if (selected_target_metrics) { + cache_name = gb_string_appendc(cache_name, "-"); + cache_name = gb_string_append_length(cache_name, selected_target_metrics->name.text, selected_target_metrics->name.len); + } + if (selected_subtarget) { + String st = subtarget_strings[selected_subtarget]; + cache_name = gb_string_appendc(cache_name, "-"); + cache_name = gb_string_append_length(cache_name, st.text, st.len); + } + cache_name = gb_string_appendc(cache_name, ".bin"); + + if (to_cache) { + return gb_file_copy( + alloc_cstring(temporary_allocator(), exe_name), + cache_name, + false + ); + } else { + return gb_file_copy( + cache_name, + alloc_cstring(temporary_allocator(), exe_name), + false + ); + } +} + + + +gb_internal bool try_copy_executable_to_cache(void) { + debugf("Cache: try_copy_executable_to_cache\n"); + + if (try_copy_executable_cache_internal(true)) { + build_context.build_cache_data.copy_already_done = true; + return true; + } + return false; +} + +gb_internal bool try_copy_executable_from_cache(void) { + debugf("Cache: try_copy_executable_from_cache\n"); + + if (try_copy_executable_cache_internal(false)) { + build_context.build_cache_data.copy_already_done = true; + return true; + } + return false; +} + + +#if !defined(GB_SYSTEM_WINDOWS) +extern char **environ; +#endif + +// returns false if different, true if it is the same +gb_internal bool try_cached_build(Checker *c, Array const &args) { + TEMPORARY_ALLOCATOR_GUARD(); + + Parser *p = c->parser; + + auto files = array_make(heap_allocator()); + for (AstPackage *pkg : p->packages) { + for (AstFile *f : pkg->files) { + array_add(&files, f->fullpath); + } + } + + for (auto const &entry : c->info.load_file_cache) { + auto *cache = entry.value; + if (!cache || !cache->exists) { + continue; + } + array_add(&files, cache->path); + } + + array_sort(files, string_cmp); + + u64 crc = 0; + for (String const &path : files) { + crc = crc64_with_seed(path.text, path.len, crc); + } + + String base_cache_dir = build_context.build_paths[BuildPath_Output].basename; + base_cache_dir = concatenate_strings(permanent_allocator(), base_cache_dir, str_lit("/.odin-cache")); + (void)check_if_exists_directory_otherwise_create(base_cache_dir); + + gbString crc_str = gb_string_make_reserve(permanent_allocator(), 16); + crc_str = gb_string_append_fmt(crc_str, "%016llx", crc); + String cache_dir = concatenate3_strings(permanent_allocator(), base_cache_dir, str_lit("/"), make_string_c(crc_str)); + String files_path = concatenate3_strings(permanent_allocator(), cache_dir, str_lit("/"), str_lit("files.manifest")); + String args_path = concatenate3_strings(permanent_allocator(), cache_dir, str_lit("/"), str_lit("args.manifest")); + String env_path = concatenate3_strings(permanent_allocator(), cache_dir, str_lit("/"), str_lit("env.manifest")); + + build_context.build_cache_data.cache_dir = cache_dir; + build_context.build_cache_data.files_path = files_path; + build_context.build_cache_data.args_path = args_path; + build_context.build_cache_data.env_path = env_path; + + auto envs = array_make(heap_allocator()); + defer (array_free(&envs)); + { + #if defined(GB_SYSTEM_WINDOWS) + wchar_t *strings = GetEnvironmentStringsW(); + defer (FreeEnvironmentStringsW(strings)); + + wchar_t *curr_string = strings; + while (curr_string && *curr_string) { + String16 wstr = make_string16_c(curr_string); + curr_string += wstr.len+1; + String str = string16_to_string(temporary_allocator(), wstr); + if (string_starts_with(str, str_lit("CURR_DATE_TIME="))) { + continue; + } + array_add(&envs, str); + } + #else + char **curr_env = environ; + while (curr_env && *curr_env) { + String str = make_string_c(*curr_env++); + if (string_starts_with(str, str_lit("PROMPT="))) { + continue; + } + if (string_starts_with(str, str_lit("RPROMPT="))) { + continue; + } + array_add(&envs, str); + } + #endif + } + array_sort(envs, string_cmp); + + if (check_if_exists_directory_otherwise_create(cache_dir)) { + goto write_cache; + } + + if (check_if_exists_file_otherwise_create(files_path)) { + goto write_cache; + } + if (check_if_exists_file_otherwise_create(args_path)) { + goto write_cache; + } + if (check_if_exists_file_otherwise_create(env_path)) { + goto write_cache; + } + + { + // exists already + LoadedFile loaded_file = {}; + + LoadedFileError file_err = load_file_32( + alloc_cstring(temporary_allocator(), files_path), + &loaded_file, + false + ); + if (file_err > LoadedFile_Empty) { + return false; + } + + String data = {cast(u8 *)loaded_file.data, loaded_file.size}; + String_Iterator it = {data, 0}; + + isize file_count = 0; + + for (; it.pos < data.len; file_count++) { + String line = string_split_iterator(&it, '\n'); + if (line.len == 0) { + break; + } + isize sep = string_index_byte(line, ' '); + if (sep < 0) { + goto write_cache; + } + + String timestamp_str = substring(line, 0, sep); + String path_str = substring(line, sep+1, line.len); + + timestamp_str = string_trim_whitespace(timestamp_str); + path_str = string_trim_whitespace(path_str); + + if (file_count >= files.count) { + goto write_cache; + } + if (files[file_count] != path_str) { + goto write_cache; + } + + u64 timestamp = exact_value_to_u64(exact_value_integer_from_string(timestamp_str)); + gbFileTime last_write_time = gb_file_last_write_time(alloc_cstring(temporary_allocator(), path_str)); + if (last_write_time != timestamp) { + goto write_cache; + } + } + + if (file_count != files.count) { + goto write_cache; + } + } + { + LoadedFile loaded_file = {}; + + LoadedFileError file_err = load_file_32( + alloc_cstring(temporary_allocator(), args_path), + &loaded_file, + false + ); + if (file_err > LoadedFile_Empty) { + return false; + } + + String data = {cast(u8 *)loaded_file.data, loaded_file.size}; + String_Iterator it = {data, 0}; + + isize args_count = 0; + + for (; it.pos < data.len; args_count++) { + String line = string_split_iterator(&it, '\n'); + line = string_trim_whitespace(line); + if (line.len == 0) { + break; + } + if (args_count >= args.count) { + goto write_cache; + } + + if (line != args[args_count]) { + goto write_cache; + } + } + } + { + LoadedFile loaded_file = {}; + + LoadedFileError file_err = load_file_32( + alloc_cstring(temporary_allocator(), env_path), + &loaded_file, + false + ); + if (file_err > LoadedFile_Empty) { + return false; + } + + String data = {cast(u8 *)loaded_file.data, loaded_file.size}; + String_Iterator it = {data, 0}; + + isize env_count = 0; + + for (; it.pos < data.len; env_count++) { + String line = string_split_iterator(&it, '\n'); + line = string_trim_whitespace(line); + if (line.len == 0) { + break; + } + if (env_count >= envs.count) { + goto write_cache; + } + + if (line != envs[env_count]) { + goto write_cache; + } + } + } + + return try_copy_executable_from_cache(); + +write_cache:; + { + char const *path_c = alloc_cstring(temporary_allocator(), files_path); + gb_file_remove(path_c); + + debugf("Cache: updating %s\n", path_c); + + gbFile f = {}; + defer (gb_file_close(&f)); + gb_file_open_mode(&f, gbFileMode_Write, path_c); + + for (String const &path : files) { + gbFileTime ft = gb_file_last_write_time(alloc_cstring(temporary_allocator(), path)); + gb_fprintf(&f, "%llu %.*s\n", cast(unsigned long long)ft, LIT(path)); + } + } + { + char const *path_c = alloc_cstring(temporary_allocator(), args_path); + gb_file_remove(path_c); + + debugf("Cache: updating %s\n", path_c); + + gbFile f = {}; + defer (gb_file_close(&f)); + gb_file_open_mode(&f, gbFileMode_Write, path_c); + + for (String const &arg : args) { + String targ = string_trim_whitespace(arg); + gb_fprintf(&f, "%.*s\n", LIT(targ)); + } + } + { + char const *path_c = alloc_cstring(temporary_allocator(), env_path); + gb_file_remove(path_c); + + debugf("Cache: updating %s\n", path_c); + + gbFile f = {}; + defer (gb_file_close(&f)); + gb_file_open_mode(&f, gbFileMode_Write, path_c); + + for (String const &env : envs) { + gb_fprintf(&f, "%.*s\n", LIT(env)); + } + } + + + return false; +} + diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 44b6e3839..b0df17778 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -1397,8 +1397,8 @@ gb_internal void lb_create_global_procedures_and_types(lbGenerator *gen, Checker for (auto const &entry : gen->modules) { lbModule *m = entry.value; - gb_sort_array(m->global_types_to_create.data, m->global_types_to_create.count, llvm_global_entity_cmp); - gb_sort_array(m->global_procedures_to_create.data, m->global_procedures_to_create.count, llvm_global_entity_cmp); + array_sort(m->global_types_to_create, llvm_global_entity_cmp); + array_sort(m->global_procedures_to_create, llvm_global_entity_cmp); } if (do_threading) { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 77f78a24c..bbeff562c 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -126,16 +126,17 @@ gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) { m->gen = gen; map_set(&gen->modules, cast(void *)pkg, m); lb_init_module(m, c); - if (module_per_file) { - // NOTE(bill): Probably per file is not a good idea, so leave this for later - for (AstFile *file : pkg->files) { - auto m = gb_alloc_item(permanent_allocator(), lbModule); - m->file = file; - m->pkg = pkg; - m->gen = gen; - map_set(&gen->modules, cast(void *)file, m); - lb_init_module(m, c); - } + if (!module_per_file) { + continue; + } + // NOTE(bill): Probably per file is not a good idea, so leave this for later + for (AstFile *file : pkg->files) { + auto m = gb_alloc_item(permanent_allocator(), lbModule); + m->file = file; + m->pkg = pkg; + m->gen = gen; + map_set(&gen->modules, cast(void *)file, m); + lb_init_module(m, c); } } } @@ -144,7 +145,6 @@ gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) { map_set(&gen->modules, cast(void *)1, &gen->default_module); lb_init_module(&gen->default_module, c); - for (auto const &entry : gen->modules) { lbModule *m = entry.value; LLVMContextRef ctx = LLVMGetModuleContext(m->mod); diff --git a/src/main.cpp b/src/main.cpp index bbb326af3..7763ccd23 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -71,6 +71,8 @@ gb_global Timings global_timings = {0}; #include "checker.cpp" #include "docs.cpp" +#include "cached.cpp" + #include "linker.cpp" #if defined(GB_SYSTEM_WINDOWS) && defined(ODIN_TILDE_BACKEND) @@ -391,6 +393,7 @@ enum BuildFlagKind { BuildFlag_InternalIgnoreLLVMBuild, BuildFlag_InternalIgnorePanic, BuildFlag_InternalModulePerFile, + BuildFlag_InternalCached, BuildFlag_Tilde, @@ -594,6 +597,7 @@ gb_internal bool parse_build_flags(Array args) { add_flag(&build_flags, BuildFlag_InternalIgnoreLLVMBuild, str_lit("internal-ignore-llvm-build"),BuildFlagParam_None, Command_all); add_flag(&build_flags, BuildFlag_InternalIgnorePanic, str_lit("internal-ignore-panic"), BuildFlagParam_None, Command_all); add_flag(&build_flags, BuildFlag_InternalModulePerFile, str_lit("internal-module-per-file"), BuildFlagParam_None, Command_all); + add_flag(&build_flags, BuildFlag_InternalCached, str_lit("internal-cached"), BuildFlagParam_None, Command_all); #if ALLOW_TILDE add_flag(&build_flags, BuildFlag_Tilde, str_lit("tilde"), BuildFlagParam_None, Command__does_build); @@ -1413,6 +1417,10 @@ gb_internal bool parse_build_flags(Array args) { case BuildFlag_InternalModulePerFile: build_context.module_per_file = true; break; + case BuildFlag_InternalCached: + build_context.cached = true; + build_context.use_separate_modules = true; + break; case BuildFlag_Tilde: build_context.tilde_backend = true; @@ -1921,9 +1929,6 @@ gb_internal void show_timings(Checker *c, Timings *t) { gb_internal GB_COMPARE_PROC(file_path_cmp) { AstFile *x = *(AstFile **)a; AstFile *y = *(AstFile **)b; - if (x == y) { - return 0; - } return string_compare(x->fullpath, y->fullpath); } @@ -3052,6 +3057,8 @@ int main(int arg_count, char const **arg_ptr) { } else if (command == "root") { gb_printf("%.*s", LIT(odin_root_dir())); return 0; + } else if (command == "clear-cache") { + return try_clear_cache() ? 0 : 1; } else { String argv1 = {}; if (args.count > 1) { @@ -3264,12 +3271,19 @@ int main(int arg_count, char const **arg_ptr) { print_all_errors(); } - MAIN_TIME_SECTION("type check"); checker->parser = parser; init_checker(checker); - defer (destroy_checker(checker)); + defer (destroy_checker(checker)); // this is here because of a `goto` + if (build_context.cached && parser->total_seen_load_directive_count.load() == 0) { + MAIN_TIME_SECTION("check cached build (pre-semantic check)"); + if (try_cached_build(checker, args)) { + goto end_of_code_gen; + } + } + + MAIN_TIME_SECTION("type check"); check_parsed_files(checker); check_defines(&build_context, checker); if (any_errors()) { @@ -3322,6 +3336,13 @@ int main(int arg_count, char const **arg_ptr) { return 0; } + if (build_context.cached) { + MAIN_TIME_SECTION("check cached build"); + if (try_cached_build(checker, args)) { + goto end_of_code_gen; + } + } + #if ALLOW_TILDE if (build_context.tilde_backend) { LinkerData linker_data = {}; @@ -3383,6 +3404,8 @@ int main(int arg_count, char const **arg_ptr) { remove_temp_files(gen); } +end_of_code_gen:; + if (build_context.show_timings) { show_timings(checker, &global_timings); } @@ -3391,6 +3414,12 @@ int main(int arg_count, char const **arg_ptr) { export_dependencies(checker); } + + if (!build_context.build_cache_data.copy_already_done && + build_context.cached) { + try_copy_executable_to_cache(); + } + if (run_output) { String exe_name = path_to_string(heap_allocator(), build_context.build_paths[BuildPath_Output]); defer (gb_free(heap_allocator(), exe_name.text)); diff --git a/src/parser.cpp b/src/parser.cpp index 548e46cbe..eb012e980 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -787,6 +787,9 @@ gb_internal Ast *ast_basic_directive(AstFile *f, Token token, Token name) { Ast *result = alloc_ast_node(f, Ast_BasicDirective); result->BasicDirective.token = token; result->BasicDirective.name = name; + if (string_starts_with(name.string, str_lit("load"))) { + f->seen_load_directive_count++; + } return result; } @@ -6576,6 +6579,13 @@ gb_internal ParseFileError parse_packages(Parser *p, String init_filename) { } } } + + for (AstPackage *pkg : p->packages) { + for (AstFile *file : pkg->files) { + p->total_seen_load_directive_count += file->seen_load_directive_count; + } + } + return ParseFile_None; } diff --git a/src/parser.hpp b/src/parser.hpp index 521fd7a37..86b3393af 100644 --- a/src/parser.hpp +++ b/src/parser.hpp @@ -140,6 +140,8 @@ struct AstFile { // This is effectively a queue but does not require any multi-threading capabilities Array delayed_decls_queues[AstDelayQueue_COUNT]; + std::atomic seen_load_directive_count; + #define PARSER_MAX_FIX_COUNT 6 isize fix_count; TokenPos fix_prev_pos; @@ -210,6 +212,8 @@ struct Parser { std::atomic total_token_count; std::atomic total_line_count; + std::atomic total_seen_load_directive_count; + // TODO(bill): What should this mutex be per? // * Parser // * Package diff --git a/src/string.cpp b/src/string.cpp index 42bf51f80..3c7d96934 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -88,6 +88,13 @@ gb_internal char *alloc_cstring(gbAllocator a, String s) { return c_str; } +gb_internal wchar_t *alloc_wstring(gbAllocator a, String16 s) { + wchar_t *c_str = gb_alloc_array(a, wchar_t, s.len+1); + gb_memmove(c_str, s.text, s.len*2); + c_str[s.len] = '\0'; + return c_str; +} + gb_internal gb_inline bool str_eq_ignore_case(String const &a, String const &b) { if (a.len == b.len) {