From 7eabeda870c977927ae0060ccdc36a923a6000e5 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Sat, 2 Dec 2023 21:10:44 +0100 Subject: [PATCH 1/3] make sure an eof is also returned from the file streams on non-windows --- core/os/stream.odin | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/os/stream.odin b/core/os/stream.odin index a5132239f..4c0b35600 100644 --- a/core/os/stream.odin +++ b/core/os/stream.odin @@ -54,6 +54,7 @@ _file_stream_proc :: proc(stream_data: rawptr, mode: io.Stream_Mode, p: []byte, return io.query_utility({.Close, .Flush, .Read, .Read_At, .Write, .Write_At, .Seek, .Size, .Query}) } } + if err == nil && os_err != 0 { when ODIN_OS == .Windows { if os_err == ERROR_HANDLE_EOF { @@ -62,5 +63,12 @@ _file_stream_proc :: proc(stream_data: rawptr, mode: io.Stream_Mode, p: []byte, } err = .Unknown } + + when ODIN_OS != .Windows { + if err == nil && os_err == 0 && n == 0 { + err = .EOF + } + } + return } From 9a1c4dc56d101f53ac916b83bfedadaa840b31d0 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Sat, 2 Dec 2023 21:11:11 +0100 Subject: [PATCH 2/3] implement a max read/write at a time for non-windows (windows already has this) --- core/os/os_darwin.odin | 26 +++++++++++++++++++++----- core/os/os_freebsd.odin | 15 +++++++++++++-- core/os/os_linux.odin | 26 ++++++++++++++++++++++---- core/os/os_openbsd.odin | 15 +++++++++++++-- 4 files changed, 69 insertions(+), 13 deletions(-) diff --git a/core/os/os_darwin.odin b/core/os/os_darwin.odin index 82cf5e1f3..7aeb40c4f 100644 --- a/core/os/os_darwin.odin +++ b/core/os/os_darwin.odin @@ -568,15 +568,24 @@ close :: proc(fd: Handle) -> bool { return _unix_close(fd) == 0 } +// If you read or write more than `SSIZE_MAX` bytes, most darwin implementations will return `EINVAL` +// but it is really implementation defined. `SSIZE_MAX` is also implementation defined but usually +// the max of an i32 on Darwin. +// In practice a read/write call would probably never read/write these big buffers all at once, +// which is why the number of bytes is returned and why there are procs that will call this in a +// loop for you. +// We set a max of 1GB to keep alignment and to be safe. @(private) -MAX_RW :: 0x7fffffff // The limit on Darwin is max(i32), trying to read/write more than that fails. +MAX_RW :: 1 << 30 write :: proc(fd: Handle, data: []byte) -> (int, Errno) { if len(data) == 0 { return 0, ERROR_NONE } - bytes_written := _unix_write(fd, raw_data(data), c.size_t(len(data))) + to_write := min(c.size_t(len(data)), MAX_RW) + + bytes_written := _unix_write(fd, raw_data(data), to_write) if bytes_written < 0 { return -1, Errno(get_last_error()) } @@ -588,18 +597,23 @@ read :: proc(fd: Handle, data: []u8) -> (int, Errno) { return 0, ERROR_NONE } - bytes_read := _unix_read(fd, raw_data(data), c.size_t(len(data))) + to_read := min(c.size_t(len(data)), MAX_RW) + + bytes_read := _unix_read(fd, raw_data(data), to_read) if bytes_read < 0 { return -1, Errno(get_last_error()) } return bytes_read, ERROR_NONE } + read_at :: proc(fd: Handle, data: []byte, offset: i64) -> (int, Errno) { if len(data) == 0 { return 0, ERROR_NONE } - bytes_read := _unix_pread(fd, raw_data(data), c.size_t(len(data)), offset) + to_read := min(c.size_t(len(data)), MAX_RW) + + bytes_read := _unix_pread(fd, raw_data(data), to_read, offset) if bytes_read < 0 { return -1, Errno(get_last_error()) } @@ -611,7 +625,9 @@ write_at :: proc(fd: Handle, data: []byte, offset: i64) -> (int, Errno) { return 0, ERROR_NONE } - bytes_written := _unix_pwrite(fd, raw_data(data), c.size_t(len(data)), offset) + to_write := min(c.size_t(len(data)), MAX_RW) + + bytes_written := _unix_pwrite(fd, raw_data(data), to_write, offset) if bytes_written < 0 { return -1, Errno(get_last_error()) } diff --git a/core/os/os_freebsd.odin b/core/os/os_freebsd.odin index c2ea82bf5..9e22b7e41 100644 --- a/core/os/os_freebsd.odin +++ b/core/os/os_freebsd.odin @@ -326,8 +326,17 @@ close :: proc(fd: Handle) -> Errno { return ERROR_NONE } +// If you read or write more than `INT_MAX` bytes, FreeBSD returns `EINVAL`. +// In practice a read/write call would probably never read/write these big buffers all at once, +// which is why the number of bytes is returned and why there are procs that will call this in a +// loop for you. +// We set a max of 1GB to keep alignment and to be safe. +@(private) +MAX_RW :: 1 << 30 + read :: proc(fd: Handle, data: []byte) -> (int, Errno) { - bytes_read := _unix_read(fd, &data[0], c.size_t(len(data))) + to_read := min(c.size_t(len(data)), MAX_RW) + bytes_read := _unix_read(fd, &data[0], to_read) if bytes_read == -1 { return -1, Errno(get_last_error()) } @@ -338,7 +347,9 @@ write :: proc(fd: Handle, data: []byte) -> (int, Errno) { if len(data) == 0 { return 0, ERROR_NONE } - bytes_written := _unix_write(fd, &data[0], c.size_t(len(data))) + + to_write := min(c.size_t(len(data)), MAX_RW) + bytes_written := _unix_write(fd, &data[0], to_write) if bytes_written == -1 { return -1, Errno(get_last_error()) } diff --git a/core/os/os_linux.odin b/core/os/os_linux.odin index aabf42574..2261e17a4 100644 --- a/core/os/os_linux.odin +++ b/core/os/os_linux.odin @@ -547,12 +547,23 @@ close :: proc(fd: Handle) -> Errno { return _get_errno(unix.sys_close(int(fd))) } +// If you read or write more than `SSIZE_MAX` bytes, result is implementation defined (probably an error). +// `SSIZE_MAX` is also implementation defined but usually the max of a `ssize_t` which is `max(int)` in Odin. +// In practice a read/write call would probably never read/write these big buffers all at once, +// which is why the number of bytes is returned and why there are procs that will call this in a +// loop for you. +// We set a max of 1GB to keep alignment and to be safe. +@(private) +MAX_RW :: 1 << 30 + read :: proc(fd: Handle, data: []byte) -> (int, Errno) { if len(data) == 0 { return 0, ERROR_NONE } - bytes_read := unix.sys_read(int(fd), raw_data(data), len(data)) + to_read := min(uint(len(data)), MAX_RW) + + bytes_read := unix.sys_read(int(fd), raw_data(data), to_read) if bytes_read < 0 { return -1, _get_errno(bytes_read) } @@ -564,18 +575,23 @@ write :: proc(fd: Handle, data: []byte) -> (int, Errno) { return 0, ERROR_NONE } - bytes_written := unix.sys_write(int(fd), raw_data(data), len(data)) + to_write := min(uint(len(data)), MAX_RW) + + bytes_written := unix.sys_write(int(fd), raw_data(data), to_write) if bytes_written < 0 { return -1, _get_errno(bytes_written) } return bytes_written, ERROR_NONE } + read_at :: proc(fd: Handle, data: []byte, offset: i64) -> (int, Errno) { if len(data) == 0 { return 0, ERROR_NONE } - bytes_read := unix.sys_pread(int(fd), raw_data(data), len(data), offset) + to_read := min(uint(len(data)), MAX_RW) + + bytes_read := unix.sys_pread(int(fd), raw_data(data), to_read, offset) if bytes_read < 0 { return -1, _get_errno(bytes_read) } @@ -587,7 +603,9 @@ write_at :: proc(fd: Handle, data: []byte, offset: i64) -> (int, Errno) { return 0, ERROR_NONE } - bytes_written := unix.sys_pwrite(int(fd), raw_data(data), uint(len(data)), offset) + to_write := min(uint(len(data)), MAX_RW) + + bytes_written := unix.sys_pwrite(int(fd), raw_data(data), to_write, offset) if bytes_written < 0 { return -1, _get_errno(bytes_written) } diff --git a/core/os/os_openbsd.odin b/core/os/os_openbsd.odin index 957873a0b..e76901f0b 100644 --- a/core/os/os_openbsd.odin +++ b/core/os/os_openbsd.odin @@ -325,8 +325,17 @@ close :: proc(fd: Handle) -> Errno { return ERROR_NONE } +// If you read or write more than `SSIZE_MAX` bytes, OpenBSD returns `EINVAL`. +// In practice a read/write call would probably never read/write these big buffers all at once, +// which is why the number of bytes is returned and why there are procs that will call this in a +// loop for you. +// We set a max of 1GB to keep alignment and to be safe. +@(private) +MAX_RW :: 1 << 30 + read :: proc(fd: Handle, data: []byte) -> (int, Errno) { - bytes_read := _unix_read(fd, &data[0], c.size_t(len(data))) + to_read := min(c.size_t(len(data)), MAX_RW) + bytes_read := _unix_read(fd, &data[0], to_read) if bytes_read == -1 { return -1, Errno(get_last_error()) } @@ -337,7 +346,9 @@ write :: proc(fd: Handle, data: []byte) -> (int, Errno) { if len(data) == 0 { return 0, ERROR_NONE } - bytes_written := _unix_write(fd, &data[0], c.size_t(len(data))) + + to_write := min(c.size_t(len(data)), MAX_RW) + bytes_written := _unix_write(fd, &data[0], to_write) if bytes_written == -1 { return -1, Errno(get_last_error()) } From 0b2f357bbedda1e3c21bc61a46fd34827091d815 Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Wed, 13 Dec 2023 02:33:05 +0100 Subject: [PATCH 3/3] fix eof logic --- core/os/stream.odin | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/core/os/stream.odin b/core/os/stream.odin index 4c0b35600..d7ce11d26 100644 --- a/core/os/stream.odin +++ b/core/os/stream.odin @@ -27,19 +27,31 @@ _file_stream_proc :: proc(stream_data: rawptr, mode: io.Stream_Mode, p: []byte, case .Read: n_int, os_err = read(fd, p) n = i64(n_int) + if n == 0 && os_err == 0 { + err = .EOF + } case .Read_At: when !(ODIN_OS == .FreeBSD || ODIN_OS == .OpenBSD) { n_int, os_err = read_at(fd, p, offset) n = i64(n_int) + if n == 0 && os_err == 0 { + err = .EOF + } } case .Write: n_int, os_err = write(fd, p) n = i64(n_int) + if n == 0 && os_err == 0 { + err = .EOF + } case .Write_At: when !(ODIN_OS == .FreeBSD || ODIN_OS == .OpenBSD) { n_int, os_err = write_at(fd, p, offset) n = i64(n_int) + if n == 0 && os_err == 0 { + err = .EOF + } } case .Seek: n, os_err = seek(fd, offset, int(whence)) @@ -63,12 +75,5 @@ _file_stream_proc :: proc(stream_data: rawptr, mode: io.Stream_Mode, p: []byte, } err = .Unknown } - - when ODIN_OS != .Windows { - if err == nil && os_err == 0 && n == 0 { - err = .EOF - } - } - return }