From 545fbc54c7b4e59404af144785761a92f1ae702b Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Wed, 24 Jul 2024 00:00:06 +0200 Subject: [PATCH 01/11] testing: add json reporting --- core/testing/runner.odin | 46 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/core/testing/runner.odin b/core/testing/runner.odin index c27c2124a..da0328f91 100644 --- a/core/testing/runner.odin +++ b/core/testing/runner.odin @@ -6,6 +6,7 @@ import "base:runtime" import "core:bytes" import "core:encoding/ansi" @require import "core:encoding/base64" +@require import "core:encoding/json" import "core:fmt" import "core:io" @require import pkg_log "core:log" @@ -44,7 +45,8 @@ SHARED_RANDOM_SEED : u64 : #config(ODIN_TEST_RANDOM_SEED, 0) LOG_LEVEL : string : #config(ODIN_TEST_LOG_LEVEL, "info") // Show only the most necessary logging information. USING_SHORT_LOGS : bool : #config(ODIN_TEST_SHORT_LOGS, false) - +// Output a report of the tests to the given path. +JSON_REPORT : string : #config(ODIN_TEST_JSON_REPORT, "") get_log_level :: #force_inline proc() -> runtime.Logger_Level { when ODIN_DEBUG { @@ -61,6 +63,18 @@ get_log_level :: #force_inline proc() -> runtime.Logger_Level { } } +JSON :: struct { + total: int, + success: int, + duration: time.Duration, + packages: map[string][dynamic]JSON_Test, +} + +JSON_Test :: struct { + success: bool, + name: string, +} + end_t :: proc(t: ^T) { for i := len(t.cleanups)-1; i >= 0; i -= 1 { #no_bounds_check c := t.cleanups[i] @@ -847,5 +861,35 @@ To partly mitigate this, redirect STDERR to a file or use the -define:ODIN_TEST_ fmt.wprintln(stderr, bytes.buffer_to_string(&batch_buffer)) + when JSON_REPORT != "" { + json_report: JSON + + mode: int + when ODIN_OS != .Windows { + mode = os.S_IRUSR|os.S_IWUSR|os.S_IRGRP|os.S_IROTH + } + json_fd, errno := os.open(JSON_REPORT, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + fmt.assertf(errno == os.ERROR_NONE, "unable to open file %q for writing of JSON report, error: %v", JSON_REPORT, errno) + defer os.close(json_fd) + + for test, i in report.all_tests { + #no_bounds_check state := report.all_test_states[i] + + if test.pkg not_in json_report.packages { + json_report.packages[test.pkg] = {} + } + + tests := &json_report.packages[test.pkg] + append(tests, JSON_Test{name = test.name, success = state == .Successful}) + } + + json_report.total = len(internal_tests) + json_report.success = total_success_count + json_report.duration = finished_in + + err := json.marshal_to_writer(os.stream_from_handle(json_fd), json_report, &{ pretty = true }) + fmt.assertf(err == nil, "Error writing JSON report: %v", err) + } + return total_success_count == total_test_count } From f6488383d799241ae31cd85706dd8289b475b24e Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Wed, 24 Jul 2024 02:43:53 +0200 Subject: [PATCH 02/11] fix instrumentation features on LLVM versions with typed pointers Fixes #3970 --- src/llvm_backend_opt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/llvm_backend_opt.cpp b/src/llvm_backend_opt.cpp index e6ccc9a57..7fe1359b4 100644 --- a/src/llvm_backend_opt.cpp +++ b/src/llvm_backend_opt.cpp @@ -396,7 +396,7 @@ gb_internal LLVMValueRef lb_run_instrumentation_pass_insert_call(lbProcedure *p, lbValue cc = lb_find_procedure_value_from_entity(m, entity); LLVMValueRef args[3] = {}; - args[0] = p->value; + args[0] = LLVMConstPointerCast(p->value, lb_type(m, t_rawptr)); if (is_arch_wasm()) { args[1] = LLVMConstPointerNull(lb_type(m, t_rawptr)); From 0e91c8368ced5f7a535baf92e187bd96e22a4e01 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 09:01:41 +0100 Subject: [PATCH 03/11] Add `allocator` parameters to `fmt.caprint*` --- core/fmt/fmt.odin | 45 ++++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index 9aa9c99dc..7b86fd1b7 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -334,6 +334,27 @@ panicf :: proc(fmt: string, args: ..any, loc := #caller_location) -> ! { message := tprintf(fmt, ..args) p("Panic", message, loc) } + +// Creates a formatted C string +// +// *Allocates Using Context's Allocator* +// +// Inputs: +// - args: A variadic list of arguments to be formatted. +// - sep: An optional separator string (default is a single space). +// +// Returns: A formatted C string. +// +@(require_results) +caprint :: proc(args: ..any, sep := " ", allocator := context.allocator) -> cstring { + str: strings.Builder + strings.builder_init(&str, allocator) + sbprint(&str, ..args, sep=sep) + strings.write_byte(&str, 0) + s := strings.to_string(str) + return cstring(raw_data(s)) +} + // Creates a formatted C string // // *Allocates Using Context's Allocator* @@ -346,9 +367,9 @@ panicf :: proc(fmt: string, args: ..any, loc := #caller_location) -> ! { // Returns: A formatted C string // @(require_results) -caprintf :: proc(format: string, args: ..any, newline := false) -> cstring { +caprintf :: proc(format: string, args: ..any, allocator := context.allocator, newline := false) -> cstring { str: strings.Builder - strings.builder_init(&str) + strings.builder_init(&str, allocator) sbprintf(&str, format, ..args, newline=newline) strings.write_byte(&str, 0) s := strings.to_string(str) @@ -365,8 +386,8 @@ caprintf :: proc(format: string, args: ..any, newline := false) -> cstring { // Returns: A formatted C string // @(require_results) -caprintfln :: proc(format: string, args: ..any) -> cstring { - return caprintf(format, ..args, newline=true) +caprintfln :: proc(format: string, args: ..any, allocator := context.allocator) -> cstring { + return caprintf(format, ..args, allocator=allocator, newline=true) } // Creates a formatted C string // @@ -380,12 +401,7 @@ caprintfln :: proc(format: string, args: ..any) -> cstring { // @(require_results) ctprint :: proc(args: ..any, sep := " ") -> cstring { - str: strings.Builder - strings.builder_init(&str, context.temp_allocator) - sbprint(&str, ..args, sep=sep) - strings.write_byte(&str, 0) - s := strings.to_string(str) - return cstring(raw_data(s)) + return caprint(args=args, sep=sep, allocator=context.temp_allocator) } // Creates a formatted C string // @@ -400,12 +416,7 @@ ctprint :: proc(args: ..any, sep := " ") -> cstring { // @(require_results) ctprintf :: proc(format: string, args: ..any, newline := false) -> cstring { - str: strings.Builder - strings.builder_init(&str, context.temp_allocator) - sbprintf(&str, format, ..args, newline=newline) - strings.write_byte(&str, 0) - s := strings.to_string(str) - return cstring(raw_data(s)) + return caprintf(format=format, args=args, allocator=context.temp_allocator, newline=newline) } // Creates a formatted C string, followed by a newline. // @@ -419,7 +430,7 @@ ctprintf :: proc(format: string, args: ..any, newline := false) -> cstring { // @(require_results) ctprintfln :: proc(format: string, args: ..any) -> cstring { - return ctprintf(format, ..args, newline=true) + return caprintf(format=format, args=args, allocator=context.temp_allocator, newline=true) } // Formats using the default print settings and writes to the given strings.Builder // From c407e423d9f1299757583905b9adcda0155e82f5 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 13:37:56 +0100 Subject: [PATCH 04/11] Add `inode` to `os2.Stat` --- core/os/os2/stat.odin | 3 +++ core/os/os2/stat_linux.odin | 2 ++ core/os/os2/stat_windows.odin | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/core/os/os2/stat.odin b/core/os/os2/stat.odin index caf46b661..e67045b38 100644 --- a/core/os/os2/stat.odin +++ b/core/os/os2/stat.odin @@ -8,9 +8,12 @@ Fstat_Callback :: proc(f: ^File, allocator: runtime.Allocator) -> (File_Info, Er File_Info :: struct { fullpath: string, name: string, + + inode: u64, size: i64, mode: int, type: File_Type, + creation_time: time.Time, modification_time: time.Time, access_time: time.Time, diff --git a/core/os/os2/stat_linux.odin b/core/os/os2/stat_linux.odin index c09f9b299..39a364c9a 100644 --- a/core/os/os2/stat_linux.odin +++ b/core/os/os2/stat_linux.odin @@ -28,10 +28,12 @@ _fstat_internal :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> (File_Inf case linux.S_IFSOCK: type = .Socket } mode := int(0o7777 & transmute(u32)s.mode) + // TODO: As of Linux 4.11, the new statx syscall can retrieve creation_time fi := File_Info { fullpath = _get_full_path(fd, allocator), name = "", + inode = u64(s.ino), size = i64(s.size), mode = mode, type = type, diff --git a/core/os/os2/stat_windows.odin b/core/os/os2/stat_windows.odin index dcb37f3ab..a3def0ea7 100644 --- a/core/os/os2/stat_windows.odin +++ b/core/os/os2/stat_windows.odin @@ -262,7 +262,8 @@ _file_info_from_get_file_information_by_handle :: proc(path: string, h: win32.HA fi: File_Info fi.fullpath = path fi.name = basename(path) - fi.size = i64(d.nFileSizeHigh)<<32 + i64(d.nFileSizeLow) + fi.inode = u64(d.nFileIndexHigh)<<32 + u64(d.nFileIndexLow) + fi.size = i64(d.nFileSizeHigh)<<32 + i64(d.nFileSizeLow) type, mode := _file_type_mode_from_file_attributes(d.dwFileAttributes, nil, 0) fi.type = type fi.mode |= mode From 6d2487a69215f8d018e7be830e838c6cb94c15ce Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 13:38:14 +0100 Subject: [PATCH 05/11] Add some more Ntdll calls --- core/sys/windows/ntdll.odin | 108 +++++++++++++++++++++++++++++++++++- core/sys/windows/types.odin | 34 ++++++------ 2 files changed, 124 insertions(+), 18 deletions(-) diff --git a/core/sys/windows/ntdll.odin b/core/sys/windows/ntdll.odin index b75ac695a..20c75c801 100644 --- a/core/sys/windows/ntdll.odin +++ b/core/sys/windows/ntdll.odin @@ -15,10 +15,27 @@ foreign ntdll_lib { ProcessInformationLength: u32, ReturnLength: ^u32, ) -> u32 --- + + NtQueryInformationFile :: proc( + FileHandle: HANDLE, + IoStatusBlock: PIO_STATUS_BLOCK, + FileInformation: rawptr, + Length: ULONG, + FileInformationClass: FILE_INFORMATION_CLASS, + ) -> NTSTATUS --- +} + +PIO_STATUS_BLOCK :: ^IO_STATUS_BLOCK +IO_STATUS_BLOCK :: struct { + using _: struct #raw_union { + Status: NTSTATUS, + Pointer: rawptr, + }, + Information: ULONG_PTR, } -PROCESS_INFO_CLASS :: enum i32 { +PROCESS_INFO_CLASS :: enum c_int { ProcessBasicInformation = 0, ProcessDebugPort = 7, ProcessWow64Information = 26, @@ -29,6 +46,95 @@ PROCESS_INFO_CLASS :: enum i32 { } +PFILE_INFORMATION_CLASS :: ^FILE_INFORMATION_CLASS +FILE_INFORMATION_CLASS :: enum c_int { + FileDirectoryInformation = 1, + FileFullDirectoryInformation = 2, + FileBothDirectoryInformation = 3, + FileBasicInformation = 4, + FileStandardInformation = 5, + FileInternalInformation = 6, + FileEaInformation = 7, + FileAccessInformation = 8, + FileNameInformation = 9, + FileRenameInformation = 10, + FileLinkInformation = 11, + FileNamesInformation = 12, + FileDispositionInformation = 13, + FilePositionInformation = 14, + FileFullEaInformation = 15, + FileModeInformation = 16, + FileAlignmentInformation = 17, + FileAllInformation = 18, + FileAllocationInformation = 19, + FileEndOfFileInformation = 20, + FileAlternateNameInformation = 21, + FileStreamInformation = 22, + FilePipeInformation = 23, + FilePipeLocalInformation = 24, + FilePipeRemoteInformation = 25, + FileMailslotQueryInformation = 26, + FileMailslotSetInformation = 27, + FileCompressionInformation = 28, + FileObjectIdInformation = 29, + FileCompletionInformation = 30, + FileMoveClusterInformation = 31, + FileQuotaInformation = 32, + FileReparsePointInformation = 33, + FileNetworkOpenInformation = 34, + FileAttributeTagInformation = 35, + FileTrackingInformation = 36, + FileIdBothDirectoryInformation = 37, + FileIdFullDirectoryInformation = 38, + FileValidDataLengthInformation = 39, + FileShortNameInformation = 40, + FileIoCompletionNotificationInformation = 41, + FileIoStatusBlockRangeInformation = 42, + FileIoPriorityHintInformation = 43, + FileSfioReserveInformation = 44, + FileSfioVolumeInformation = 45, + FileHardLinkInformation = 46, + FileProcessIdsUsingFileInformation = 47, + FileNormalizedNameInformation = 48, + FileNetworkPhysicalNameInformation = 49, + FileIdGlobalTxDirectoryInformation = 50, + FileIsRemoteDeviceInformation = 51, + FileUnusedInformation = 52, + FileNumaNodeInformation = 53, + FileStandardLinkInformation = 54, + FileRemoteProtocolInformation = 55, + FileRenameInformationBypassAccessCheck = 56, + FileLinkInformationBypassAccessCheck = 57, + FileVolumeNameInformation = 58, + FileIdInformation = 59, + FileIdExtdDirectoryInformation = 60, + FileReplaceCompletionInformation = 61, + FileHardLinkFullIdInformation = 62, + FileIdExtdBothDirectoryInformation = 63, + FileDispositionInformationEx = 64, + FileRenameInformationEx = 65, + FileRenameInformationExBypassAccessCheck = 66, + FileDesiredStorageClassInformation = 67, + FileStatInformation = 68, + FileMemoryPartitionInformation = 69, + FileStatLxInformation = 70, + FileCaseSensitiveInformation = 71, + FileLinkInformationEx = 72, + FileLinkInformationExBypassAccessCheck = 73, + FileStorageReserveIdInformation = 74, + FileCaseSensitiveInformationForceAccessCheck = 75, + FileKnownFolderInformation = 76, + FileStatBasicInformation = 77, + FileId64ExtdDirectoryInformation = 78, + FileId64ExtdBothDirectoryInformation = 79, + FileIdAllExtdDirectoryInformation = 80, + FileIdAllExtdBothDirectoryInformation = 81, + FileStreamReservationInformation, + FileMupProviderInfo, + FileMaximumInformation, +} + + PROCESS_BASIC_INFORMATION :: struct { ExitStatus: NTSTATUS, diff --git a/core/sys/windows/types.odin b/core/sys/windows/types.odin index 675f681e5..ac8b63a63 100644 --- a/core/sys/windows/types.odin +++ b/core/sys/windows/types.odin @@ -2714,41 +2714,41 @@ NEON128 :: struct { EXCEPTION_POINTERS :: struct { ExceptionRecord: ^EXCEPTION_RECORD, - ContextRecord: ^CONTEXT, + ContextRecord: ^CONTEXT, } PVECTORED_EXCEPTION_HANDLER :: #type proc "system" (ExceptionInfo: ^EXCEPTION_POINTERS) -> LONG CONSOLE_READCONSOLE_CONTROL :: struct { - nLength: ULONG, - nInitialChars: ULONG, - dwCtrlWakeupMask: ULONG, + nLength: ULONG, + nInitialChars: ULONG, + dwCtrlWakeupMask: ULONG, dwControlKeyState: ULONG, } PCONSOLE_READCONSOLE_CONTROL :: ^CONSOLE_READCONSOLE_CONTROL BY_HANDLE_FILE_INFORMATION :: struct { - dwFileAttributes: DWORD, - ftCreationTime: FILETIME, - ftLastAccessTime: FILETIME, - ftLastWriteTime: FILETIME, + dwFileAttributes: DWORD, + ftCreationTime: FILETIME, + ftLastAccessTime: FILETIME, + ftLastWriteTime: FILETIME, dwVolumeSerialNumber: DWORD, - nFileSizeHigh: DWORD, - nFileSizeLow: DWORD, - nNumberOfLinks: DWORD, - nFileIndexHigh: DWORD, - nFileIndexLow: DWORD, + nFileSizeHigh: DWORD, + nFileSizeLow: DWORD, + nNumberOfLinks: DWORD, + nFileIndexHigh: DWORD, + nFileIndexLow: DWORD, } LPBY_HANDLE_FILE_INFORMATION :: ^BY_HANDLE_FILE_INFORMATION FILE_STANDARD_INFO :: struct { AllocationSize: LARGE_INTEGER, - EndOfFile: LARGE_INTEGER, - NumberOfLinks: DWORD, - DeletePending: BOOLEAN, - Directory: BOOLEAN, + EndOfFile: LARGE_INTEGER, + NumberOfLinks: DWORD, + DeletePending: BOOLEAN, + Directory: BOOLEAN, } FILE_ATTRIBUTE_TAG_INFO :: struct { From 2ddaae45f39c8d8fc3a0fc2bbe5705b8299edc85 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 13:47:22 +0100 Subject: [PATCH 06/11] Better handling of allocators --- core/os/os2/file.odin | 2 +- core/os/os2/file_linux.odin | 13 ++++++++----- core/os/os2/file_windows.odin | 18 +++++++++++------- core/os/os2/path_linux.odin | 9 ++++----- core/os/os2/pipe_linux.odin | 4 ++-- core/os/os2/stat_linux.odin | 22 +++++++++++----------- 6 files changed, 37 insertions(+), 31 deletions(-) diff --git a/core/os/os2/file.odin b/core/os/os2/file.odin index 8692ecf01..52fd02478 100644 --- a/core/os/os2/file.odin +++ b/core/os/os2/file.odin @@ -106,7 +106,7 @@ open :: proc(name: string, flags := File_Flags{.Read}, perm := 0o777) -> (^File, @(require_results) new_file :: proc(handle: uintptr, name: string) -> ^File { - return _new_file(handle, name) + return _new_file(handle, name) or_else panic("Out of memory") } @(require_results) diff --git a/core/os/os2/file_linux.odin b/core/os/os2/file_linux.odin index cf643b31a..d2a7483ca 100644 --- a/core/os/os2/file_linux.odin +++ b/core/os/os2/file_linux.odin @@ -88,21 +88,24 @@ _open :: proc(name: string, flags: File_Flags, perm: int) -> (f: ^File, err: Err return nil, _get_platform_error(errno) } - return _new_file(uintptr(fd), name), nil + return _new_file(uintptr(fd), name) } -_new_file :: proc(fd: uintptr, _: string = "") -> ^File { - impl := new(File_Impl, file_allocator()) +_new_file :: proc(fd: uintptr, _: string = "") -> (f: ^File, err: Error) { + impl := new(File_Impl, file_allocator()) or_return + defer if err != nil { + free(impl, file_allocator()) + } impl.file.impl = impl impl.fd = linux.Fd(fd) impl.allocator = file_allocator() - impl.name = _get_full_path(impl.fd, impl.allocator) + impl.name = _get_full_path(impl.fd, file_allocator()) or_return impl.file.stream = { data = impl, procedure = _file_stream_proc, } impl.file.fstat = _fstat - return &impl.file + return &impl.file, nil } _destroy :: proc(f: ^File_Impl) -> Error { diff --git a/core/os/os2/file_windows.odin b/core/os/os2/file_windows.odin index 3c18d0546..39a3e7867 100644 --- a/core/os/os2/file_windows.odin +++ b/core/os/os2/file_windows.odin @@ -126,20 +126,24 @@ _open_internal :: proc(name: string, flags: File_Flags, perm: int) -> (handle: u _open :: proc(name: string, flags: File_Flags, perm: int) -> (f: ^File, err: Error) { flags := flags if flags != nil else {.Read} handle := _open_internal(name, flags, perm) or_return - return _new_file(handle, name), nil + return _new_file(handle, name) } -_new_file :: proc(handle: uintptr, name: string) -> ^File { +_new_file :: proc(handle: uintptr, name: string) -> (f: ^File, err: Error) { if handle == INVALID_HANDLE { - return nil + return } - impl := new(File_Impl, file_allocator()) + impl := new(File_Impl, file_allocator()) or_return + defer if err != nil { + free(impl, file_allocator()) + } + impl.file.impl = impl impl.allocator = file_allocator() impl.fd = rawptr(handle) - impl.name, _ = clone_string(name, impl.allocator) - impl.wname, _ = win32_utf8_to_wstring(name, impl.allocator) + impl.name = clone_string(name, impl.allocator) or_return + impl.wname = win32_utf8_to_wstring(name, impl.allocator) or_return handle := _handle(&impl.file) kind := File_Impl_Kind.File @@ -157,7 +161,7 @@ _new_file :: proc(handle: uintptr, name: string) -> ^File { } impl.file.fstat = _fstat - return &impl.file + return &impl.file, nil } _fd :: proc(f: ^File) -> uintptr { diff --git a/core/os/os2/path_linux.odin b/core/os/os2/path_linux.odin index 1634c79c8..be60f9b86 100644 --- a/core/os/os2/path_linux.odin +++ b/core/os/os2/path_linux.odin @@ -188,7 +188,7 @@ _set_working_directory :: proc(dir: string) -> Error { return _get_platform_error(linux.chdir(dir_cstr)) } -_get_full_path :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> string { +_get_full_path :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> (fullpath: string, err: Error) { PROC_FD_PATH :: "/proc/self/fd/" buf: [32]u8 @@ -196,10 +196,9 @@ _get_full_path :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> string { strconv.itoa(buf[len(PROC_FD_PATH):], int(fd)) - fullpath: string - err: Error if fullpath, err = _read_link_cstr(cstring(&buf[0]), allocator); err != nil || fullpath[0] != '/' { - return "" + delete(fullpath, allocator) + fullpath = "" } - return fullpath + return } diff --git a/core/os/os2/pipe_linux.odin b/core/os/os2/pipe_linux.odin index 8835cc30f..c3fecfb9e 100644 --- a/core/os/os2/pipe_linux.odin +++ b/core/os/os2/pipe_linux.odin @@ -10,8 +10,8 @@ _pipe :: proc() -> (r, w: ^File, err: Error) { return nil, nil,_get_platform_error(errno) } - r = _new_file(uintptr(fds[0])) - w = _new_file(uintptr(fds[1])) + r = _new_file(uintptr(fds[0])) or_return + w = _new_file(uintptr(fds[1])) or_return return } diff --git a/core/os/os2/stat_linux.odin b/core/os/os2/stat_linux.odin index 39a364c9a..eb31e2200 100644 --- a/core/os/os2/stat_linux.odin +++ b/core/os/os2/stat_linux.odin @@ -11,7 +11,7 @@ _fstat :: proc(f: ^File, allocator: runtime.Allocator) -> (File_Info, Error) { return _fstat_internal(impl.fd, allocator) } -_fstat_internal :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> (File_Info, Error) { +_fstat_internal :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> (fi: File_Info, err: Error) { s: linux.Stat errno := linux.fstat(fd, &s) if errno != .NONE { @@ -30,20 +30,20 @@ _fstat_internal :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> (File_Inf mode := int(0o7777 & transmute(u32)s.mode) // TODO: As of Linux 4.11, the new statx syscall can retrieve creation_time - fi := File_Info { - fullpath = _get_full_path(fd, allocator), - name = "", - inode = u64(s.ino), - size = i64(s.size), - mode = mode, - type = type, + fi = File_Info { + fullpath = _get_full_path(fd, allocator) or_return, + name = "", + inode = u64(s.ino), + size = i64(s.size), + mode = mode, + type = type, modification_time = time.Time {i64(s.mtime.time_sec) * i64(time.Second) + i64(s.mtime.time_nsec)}, - access_time = time.Time {i64(s.atime.time_sec) * i64(time.Second) + i64(s.atime.time_nsec)}, - creation_time = time.Time{i64(s.ctime.time_sec) * i64(time.Second) + i64(s.ctime.time_nsec)}, // regular stat does not provide this + access_time = time.Time {i64(s.atime.time_sec) * i64(time.Second) + i64(s.atime.time_nsec)}, + creation_time = time.Time{i64(s.ctime.time_sec) * i64(time.Second) + i64(s.ctime.time_nsec)}, // regular stat does not provide this } fi.creation_time = fi.modification_time fi.name = filepath.base(fi.fullpath) - return fi, nil + return } // NOTE: _stat and _lstat are using _fstat to avoid a race condition when populating fullpath From 38e983cac63a171fb2c96328db644fed8b35d0aa Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 13:47:57 +0100 Subject: [PATCH 07/11] Remove dead code --- core/os/os2/heap.odin | 4 ---- 1 file changed, 4 deletions(-) diff --git a/core/os/os2/heap.odin b/core/os/os2/heap.odin index e0cffaf0d..8f9c7680a 100644 --- a/core/os/os2/heap.odin +++ b/core/os/os2/heap.odin @@ -17,7 +17,3 @@ heap_allocator_proc :: proc(allocator_data: rawptr, mode: runtime.Allocator_Mode old_memory: rawptr, old_size: int, loc := #caller_location) -> ([]byte, runtime.Allocator_Error) { return _heap_allocator_proc(allocator_data, mode, size, alignment, old_memory, old_size, loc) } - - -@(private) -error_allocator := heap_allocator From d4af7b86a7a41d7a59c111d2eb22fdd0e8686e78 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 14:19:30 +0100 Subject: [PATCH 08/11] Begin cleaning up `os2.read_directory` --- core/os/os2/dir.odin | 65 ++++++++++++- core/os/os2/dir_linux.odin | 18 +++- core/os/os2/dir_windows.odin | 174 ++++++++++++++++++++--------------- core/os/os2/stat.odin | 12 ++- 4 files changed, 186 insertions(+), 83 deletions(-) diff --git a/core/os/os2/dir.odin b/core/os/os2/dir.odin index 5f9deb08a..6334ee7b8 100644 --- a/core/os/os2/dir.odin +++ b/core/os/os2/dir.odin @@ -1,21 +1,80 @@ package os2 import "base:runtime" +import "core:slice" -read_directory :: proc(f: ^File, n: int, allocator: runtime.Allocator) -> (fi: []File_Info, err: Error) { - return _read_directory(f, n, allocator) +@(require_results) +read_directory :: proc(f: ^File, n: int, allocator: runtime.Allocator) -> (files: []File_Info, err: Error) { + if f == nil { + return nil, .Invalid_File + } + + n := n + size := n + if n <= 0 { + n = -1 + size = 100 + } + + TEMP_ALLOCATOR_GUARD() + + it := read_directory_iterator_create(f) or_return + defer _read_directory_iterator_destroy(&it) + + dfi := make([dynamic]File_Info, 0, size, temp_allocator()) + defer if err != nil { + for fi in dfi { + file_info_delete(fi, allocator) + } + } + + for fi, index in read_directory_iterator(&it) { + if n > 0 && index == n { + break + } + append(&dfi, file_info_clone(fi, allocator) or_return) + } + + return slice.clone(dfi[:], allocator) } + +@(require_results) read_all_directory :: proc(f: ^File, allocator: runtime.Allocator) -> (fi: []File_Info, err: Error) { return read_directory(f, -1, allocator) } +@(require_results) read_directory_by_path :: proc(path: string, n: int, allocator: runtime.Allocator) -> (fi: []File_Info, err: Error) { f := open(path) or_return defer close(f) return read_directory(f, n, allocator) } +@(require_results) read_all_directory_by_path :: proc(path: string, allocator: runtime.Allocator) -> (fi: []File_Info, err: Error) { return read_directory_by_path(path, -1, allocator) -} \ No newline at end of file +} + + +Read_Directory_Iterator :: struct { + f: ^File, + impl: Read_Directory_Iterator_Impl, +} + + +@(require_results) +read_directory_iterator_create :: proc(f: ^File) -> (Read_Directory_Iterator, Error) { + return _read_directory_iterator_create(f) +} + +read_directory_iterator_destroy :: proc(it: ^Read_Directory_Iterator) { + _read_directory_iterator_destroy(it) +} + + +// NOTE(bill): `File_Info` does not need to deleted on each iteration. Any copies must be manually copied with `file_info_clone` +@(require_results) +read_directory_iterator :: proc(it: ^Read_Directory_Iterator) -> (fi: File_Info, index: int, ok: bool) { + return _read_directory_iterator(it) +} diff --git a/core/os/os2/dir_linux.odin b/core/os/os2/dir_linux.odin index b3d4aed0b..d4f62e213 100644 --- a/core/os/os2/dir_linux.odin +++ b/core/os/os2/dir_linux.odin @@ -1,8 +1,20 @@ +//+private package os2 -import "base:runtime" +Read_Directory_Iterator_Impl :: struct { -@(private) -_read_directory :: proc(f: ^File, n: int, allocator: runtime.Allocator) -> (files: []File_Info, err: Error) { +} + + +@(require_results) +_read_directory_iterator :: proc(it: ^Read_Directory_Iterator) -> (fi: File_Info, index: int, ok: bool) { return } + +@(require_results) +_read_directory_iterator_create :: proc(f: ^File) -> (Read_Directory_Iterator, Error) { + return {}, nil +} + +_read_directory_iterator_destroy :: proc(it: ^Read_Directory_Iterator) { +} diff --git a/core/os/os2/dir_windows.odin b/core/os/os2/dir_windows.odin index 36246267d..99dbe3c17 100644 --- a/core/os/os2/dir_windows.odin +++ b/core/os/os2/dir_windows.odin @@ -1,67 +1,104 @@ +//+private package os2 import "base:runtime" import "core:time" import win32 "core:sys/windows" -@(private) -_read_directory :: proc(f: ^File, n: int, allocator: runtime.Allocator) -> (files: []File_Info, err: Error) { - find_data_to_file_info :: proc(base_path: string, d: ^win32.WIN32_FIND_DATAW, allocator: runtime.Allocator) -> (fi: File_Info, err: Error) { - // Ignore "." and ".." - if d.cFileName[0] == '.' && d.cFileName[1] == 0 { - return - } - if d.cFileName[0] == '.' && d.cFileName[1] == '.' && d.cFileName[2] == 0 { - return - } - path := concatenate({base_path, `\`, win32_utf16_to_utf8(d.cFileName[:], temp_allocator()) or_else ""}, allocator) or_return - fi.fullpath = path - fi.name = basename(path) - fi.size = i64(d.nFileSizeHigh)<<32 + i64(d.nFileSizeLow) +@(private="file") +find_data_to_file_info :: proc(base_path: string, d: ^win32.WIN32_FIND_DATAW, allocator: runtime.Allocator) -> (fi: File_Info, err: Error) { + // Ignore "." and ".." + if d.cFileName[0] == '.' && d.cFileName[1] == 0 { + return + } + if d.cFileName[0] == '.' && d.cFileName[1] == '.' && d.cFileName[2] == 0 { + return + } + path := concatenate({base_path, `\`, win32_utf16_to_utf8(d.cFileName[:], temp_allocator()) or_else ""}, allocator) or_return + fi.fullpath = path + fi.name = basename(path) + fi.size = i64(d.nFileSizeHigh)<<32 + i64(d.nFileSizeLow) - if d.dwFileAttributes & win32.FILE_ATTRIBUTE_READONLY != 0 { - fi.mode |= 0o444 - } else { - fi.mode |= 0o666 - } + if d.dwFileAttributes & win32.FILE_ATTRIBUTE_READONLY != 0 { + fi.mode |= 0o444 + } else { + fi.mode |= 0o666 + } - is_sym := false - if d.dwFileAttributes & win32.FILE_ATTRIBUTE_REPARSE_Point == 0 { - is_sym = false - } else { - is_sym = d.dwReserved0 == win32.IO_REPARSE_TAG_SYMLINK || d.dwReserved0 == win32.IO_REPARSE_TAG_MOUNT_POINT - } + is_sym := false + if d.dwFileAttributes & win32.FILE_ATTRIBUTE_REPARSE_Point == 0 { + is_sym = false + } else { + is_sym = d.dwReserved0 == win32.IO_REPARSE_TAG_SYMLINK || d.dwReserved0 == win32.IO_REPARSE_TAG_MOUNT_POINT + } - if is_sym { - fi.type = .Symlink - } else if d.dwFileAttributes & win32.FILE_ATTRIBUTE_DIRECTORY != 0 { - fi.type = .Directory - fi.mode |= 0o111 - } + if is_sym { + fi.type = .Symlink + } else if d.dwFileAttributes & win32.FILE_ATTRIBUTE_DIRECTORY != 0 { + fi.type = .Directory + fi.mode |= 0o111 + } - fi.creation_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftCreationTime)) - fi.modification_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftLastWriteTime)) - fi.access_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftLastAccessTime)) + fi.creation_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftCreationTime)) + fi.modification_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftLastWriteTime)) + fi.access_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftLastAccessTime)) + return +} + +Read_Directory_Iterator_Impl :: struct { + find_data: win32.WIN32_FIND_DATAW, + find_handle: win32.HANDLE, + path: string, + prev_fi: File_Info, + no_more_files: bool, + index: int, +} + + +@(require_results) +_read_directory_iterator :: proc(it: ^Read_Directory_Iterator) -> (fi: File_Info, index: int, ok: bool) { + if it.f == nil { + return + } + if it.impl.no_more_files { return } - if f == nil { - return nil, .Invalid_File + + err: Error + fi, err = find_data_to_file_info(it.impl.path, &it.impl.find_data, file_allocator()) + if err != nil { + return + } + if fi.name != "" { + file_info_delete(it.impl.prev_fi, file_allocator()) + it.impl.prev_fi = fi + ok = true + index = it.impl.index + it.impl.index += 1 } - TEMP_ALLOCATOR_GUARD() + if !win32.FindNextFileW(it.impl.find_handle, &it.impl.find_data) { + e := _get_platform_error() + if pe, _ := is_platform_error(e); pe == i32(win32.ERROR_NO_MORE_FILES) { + it.impl.no_more_files = true + } + it.impl.no_more_files = true + return + } + return +} +@(require_results) +_read_directory_iterator_create :: proc(f: ^File) -> (it: Read_Directory_Iterator, err: Error) { + if f == nil { + return + } impl := (^File_Impl)(f.impl) if !is_directory(impl.name) { - return nil, .Invalid_Dir - } - - n := n - size := n - if n <= 0 { - n = -1 - size = 100 + err = .Invalid_Dir + return } wpath: []u16 @@ -73,46 +110,31 @@ _read_directory :: proc(f: ^File, n: int, allocator: runtime.Allocator) -> (file wpath = impl.wname[:i] } + TEMP_ALLOCATOR_GUARD() - wpath_search := make([]u16, len(wpath)+3, context.temp_allocator) + wpath_search := make([]u16, len(wpath)+3, temp_allocator()) copy(wpath_search, wpath) wpath_search[len(wpath)+0] = '\\' wpath_search[len(wpath)+1] = '*' wpath_search[len(wpath)+2] = 0 - find_data := &win32.WIN32_FIND_DATAW{} - find_handle := win32.FindFirstFileW(raw_data(wpath_search), find_data) - if find_handle == win32.INVALID_HANDLE_VALUE { - return nil, _get_platform_error() + it.impl.find_handle = win32.FindFirstFileW(raw_data(wpath_search), &it.impl.find_data) + if it.impl.find_handle == win32.INVALID_HANDLE_VALUE { + err = _get_platform_error() + return } - defer win32.FindClose(find_handle) - - path := _cleanpath_from_buf(wpath, temp_allocator()) or_return - - dfi := make([dynamic]File_Info, 0, size, allocator) defer if err != nil { - for fi in dfi { - file_info_delete(fi, allocator) - } - delete(dfi) - } - for n != 0 { - fi: File_Info - fi = find_data_to_file_info(path, find_data, allocator) or_return - if fi.name != "" { - append(&dfi, fi) - n -= 1 - } - - if !win32.FindNextFileW(find_handle, find_data) { - e := _get_platform_error() - if pe, _ := is_platform_error(e); pe == i32(win32.ERROR_NO_MORE_FILES) { - break - } - return dfi[:], e - } + win32.FindClose(it.impl.find_handle) } - return dfi[:], nil + it.impl.path = _cleanpath_from_buf(wpath, file_allocator()) or_return + return } +_read_directory_iterator_destroy :: proc(it: ^Read_Directory_Iterator) { + if it.f == nil { + return + } + file_info_delete(it.impl.prev_fi, file_allocator()) + win32.FindClose(it.impl.find_handle) +} \ No newline at end of file diff --git a/core/os/os2/stat.odin b/core/os/os2/stat.odin index e67045b38..a324e7189 100644 --- a/core/os/os2/stat.odin +++ b/core/os/os2/stat.odin @@ -1,7 +1,9 @@ package os2 -import "core:time" import "base:runtime" +import "core:path/filepath" +import "core:strings" +import "core:time" Fstat_Callback :: proc(f: ^File, allocator: runtime.Allocator) -> (File_Info, Error) @@ -19,6 +21,14 @@ File_Info :: struct { access_time: time.Time, } +@(require_results) +file_info_clone :: proc(fi: File_Info, allocator: runtime.Allocator) -> (cloned: File_Info, err: runtime.Allocator_Error) { + cloned = fi + cloned.fullpath = strings.clone(fi.fullpath) or_return + cloned.name = filepath.base(cloned.fullpath) + return +} + file_info_slice_delete :: proc(infos: []File_Info, allocator: runtime.Allocator) { for i := len(infos)-1; i >= 0; i -= 1 { file_info_delete(infos[i], allocator) From 9d8953538bb28db656479ead989c14b9e3e08d6b Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 14:25:42 +0100 Subject: [PATCH 09/11] Add missing attribute --- core/os/os2/file_windows.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/os/os2/file_windows.odin b/core/os/os2/file_windows.odin index 39a3e7867..55365c1bd 100644 --- a/core/os/os2/file_windows.odin +++ b/core/os/os2/file_windows.odin @@ -94,7 +94,7 @@ _open_internal :: proc(name: string, flags: File_Flags, perm: int) -> (handle: u create_mode = win32.TRUNCATE_EXISTING } - attrs: u32 = win32.FILE_ATTRIBUTE_NORMAL + attrs: u32 = win32.FILE_ATTRIBUTE_NORMAL|win32.FILE_FLAG_BACKUP_SEMANTICS if perm & S_IWRITE == 0 { attrs = win32.FILE_ATTRIBUTE_READONLY if create_mode == win32.CREATE_ALWAYS { From 07b1819dc8154e1786ebb8ce942e45200b0fc0d8 Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 14:53:33 +0100 Subject: [PATCH 10/11] Improve `os2.read_directory` --- core/os/os2/dir_windows.odin | 66 ++++++++++++++--------------------- core/os/os2/stat.odin | 2 +- core/os/os2/stat_linux.odin | 2 +- core/os/os2/stat_windows.odin | 2 +- 4 files changed, 30 insertions(+), 42 deletions(-) diff --git a/core/os/os2/dir_windows.odin b/core/os/os2/dir_windows.odin index 99dbe3c17..c767bd3b9 100644 --- a/core/os/os2/dir_windows.odin +++ b/core/os/os2/dir_windows.odin @@ -15,30 +15,15 @@ find_data_to_file_info :: proc(base_path: string, d: ^win32.WIN32_FIND_DATAW, al return } path := concatenate({base_path, `\`, win32_utf16_to_utf8(d.cFileName[:], temp_allocator()) or_else ""}, allocator) or_return + + fi.fullpath = path fi.name = basename(path) fi.size = i64(d.nFileSizeHigh)<<32 + i64(d.nFileSizeLow) - if d.dwFileAttributes & win32.FILE_ATTRIBUTE_READONLY != 0 { - fi.mode |= 0o444 - } else { - fi.mode |= 0o666 - } - - is_sym := false - if d.dwFileAttributes & win32.FILE_ATTRIBUTE_REPARSE_Point == 0 { - is_sym = false - } else { - is_sym = d.dwReserved0 == win32.IO_REPARSE_TAG_SYMLINK || d.dwReserved0 == win32.IO_REPARSE_TAG_MOUNT_POINT - } - - if is_sym { - fi.type = .Symlink - } else if d.dwFileAttributes & win32.FILE_ATTRIBUTE_DIRECTORY != 0 { - fi.type = .Directory - fi.mode |= 0o111 - } + fi.type, fi.mode = _file_type_mode_from_file_attributes(d.dwFileAttributes, nil, d.dwReserved0) + // fi.inode = u128(u64(d.nFileIndexHigh)<<32 + u64(d.nFileIndexLow)) fi.creation_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftCreationTime)) fi.modification_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftLastWriteTime)) fi.access_time = time.unix(0, win32.FILETIME_as_unix_nanoseconds(d.ftLastAccessTime)) @@ -60,31 +45,33 @@ _read_directory_iterator :: proc(it: ^Read_Directory_Iterator) -> (fi: File_Info if it.f == nil { return } - if it.impl.no_more_files { - return - } - - err: Error - fi, err = find_data_to_file_info(it.impl.path, &it.impl.find_data, file_allocator()) - if err != nil { - return - } - if fi.name != "" { + for !it.impl.no_more_files { + err: Error file_info_delete(it.impl.prev_fi, file_allocator()) - it.impl.prev_fi = fi - ok = true - index = it.impl.index - it.impl.index += 1 - } + it.impl.prev_fi = {} - if !win32.FindNextFileW(it.impl.find_handle, &it.impl.find_data) { - e := _get_platform_error() - if pe, _ := is_platform_error(e); pe == i32(win32.ERROR_NO_MORE_FILES) { + fi, err = find_data_to_file_info(it.impl.path, &it.impl.find_data, file_allocator()) + if err != nil { + return + } + if fi.name != "" { + it.impl.prev_fi = fi + ok = true + index = it.impl.index + it.impl.index += 1 + } + + if !win32.FindNextFileW(it.impl.find_handle, &it.impl.find_data) { + e := _get_platform_error() + if pe, _ := is_platform_error(e); pe == i32(win32.ERROR_NO_MORE_FILES) { + it.impl.no_more_files = true + } it.impl.no_more_files = true } - it.impl.no_more_files = true - return + if ok { + return + } } return } @@ -94,6 +81,7 @@ _read_directory_iterator_create :: proc(f: ^File) -> (it: Read_Directory_Iterato if f == nil { return } + it.f = f impl := (^File_Impl)(f.impl) if !is_directory(impl.name) { diff --git a/core/os/os2/stat.odin b/core/os/os2/stat.odin index a324e7189..5c063e771 100644 --- a/core/os/os2/stat.odin +++ b/core/os/os2/stat.odin @@ -11,7 +11,7 @@ File_Info :: struct { fullpath: string, name: string, - inode: u64, + inode: u128, // might be zero if cannot be determined size: i64, mode: int, type: File_Type, diff --git a/core/os/os2/stat_linux.odin b/core/os/os2/stat_linux.odin index eb31e2200..6ccac1be0 100644 --- a/core/os/os2/stat_linux.odin +++ b/core/os/os2/stat_linux.odin @@ -33,7 +33,7 @@ _fstat_internal :: proc(fd: linux.Fd, allocator: runtime.Allocator) -> (fi: File fi = File_Info { fullpath = _get_full_path(fd, allocator) or_return, name = "", - inode = u64(s.ino), + inode = u128(u64(s.ino)), size = i64(s.size), mode = mode, type = type, diff --git a/core/os/os2/stat_windows.odin b/core/os/os2/stat_windows.odin index a3def0ea7..4d67ddb58 100644 --- a/core/os/os2/stat_windows.odin +++ b/core/os/os2/stat_windows.odin @@ -262,7 +262,7 @@ _file_info_from_get_file_information_by_handle :: proc(path: string, h: win32.HA fi: File_Info fi.fullpath = path fi.name = basename(path) - fi.inode = u64(d.nFileIndexHigh)<<32 + u64(d.nFileIndexLow) + fi.inode = u128(u64(d.nFileIndexHigh)<<32 + u64(d.nFileIndexLow)) fi.size = i64(d.nFileSizeHigh)<<32 + i64(d.nFileSizeLow) type, mode := _file_type_mode_from_file_attributes(d.dwFileAttributes, nil, 0) fi.type = type From 2dbccbde54694e959d50a72fbc133b7391d1644c Mon Sep 17 00:00:00 2001 From: gingerBill Date: Wed, 24 Jul 2024 14:53:48 +0100 Subject: [PATCH 11/11] Improve win32 types --- core/sys/windows/types.odin | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/core/sys/windows/types.odin b/core/sys/windows/types.odin index ac8b63a63..557fb5a58 100644 --- a/core/sys/windows/types.odin +++ b/core/sys/windows/types.odin @@ -1027,16 +1027,28 @@ TRACKMOUSEEVENT :: struct { } WIN32_FIND_DATAW :: struct { - dwFileAttributes: DWORD, - ftCreationTime: FILETIME, - ftLastAccessTime: FILETIME, - ftLastWriteTime: FILETIME, - nFileSizeHigh: DWORD, - nFileSizeLow: DWORD, - dwReserved0: DWORD, - dwReserved1: DWORD, - cFileName: [260]wchar_t, // #define MAX_PATH 260 - cAlternateFileName: [14]wchar_t, + dwFileAttributes: DWORD, + ftCreationTime: FILETIME, + ftLastAccessTime: FILETIME, + ftLastWriteTime: FILETIME, + nFileSizeHigh: DWORD, + nFileSizeLow: DWORD, + dwReserved0: DWORD, + dwReserved1: DWORD, + cFileName: [MAX_PATH]WCHAR, + cAlternateFileName: [14]WCHAR, + _OBSOLETE_dwFileType: DWORD, // Obsolete. Do not use. + _OBSOLETE_dwCreatorType: DWORD, // Obsolete. Do not use + _OBSOLETE_wFinderFlags: WORD, // Obsolete. Do not use +} + +FILE_ID_128 :: struct { + Identifier: [16]BYTE, +} + +FILE_ID_INFO :: struct { + VolumeSerialNumber: ULONGLONG, + FileId: FILE_ID_128, } CREATESTRUCTA :: struct {