diff --git a/.gitattributes b/.gitattributes index 0178b8a8f..e58996311 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,6 @@ *.odin linguist-language=Odin -* text=auto \ No newline at end of file +* text=auto + +# These files must always have *nix line-endings +Makefile text eol=lf +*.sh text eol=lf \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c43fca2c..07e89f3fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: timeout-minutes: 10 build_macOS: name: MacOS Build, Check, and Test - runs-on: macos-latest + runs-on: macos-13 steps: - uses: actions/checkout@v1 - name: Download LLVM, and setup PATH diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ff90ab57e..c9a2c821b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -77,7 +77,7 @@ jobs: build_macos: name: MacOS Build if: github.repository == 'odin-lang/Odin' - runs-on: macos-latest + runs-on: macos-13 steps: - uses: actions/checkout@v1 - name: Download LLVM and setup PATH diff --git a/.gitignore b/.gitignore index 228f006a3..f6c3927a2 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,8 @@ tests/documentation/all.odin-doc tests/internal/test_map tests/internal/test_pow tests/internal/test_rtti +tests/core/test_base64 +tests/core/test_cbor tests/core/test_core_compress tests/core/test_core_container tests/core/test_core_filepath @@ -40,8 +42,10 @@ tests/core/test_core_net tests/core/test_core_os_exit tests/core/test_core_reflect tests/core/test_core_strings +tests/core/test_core_time tests/core/test_crypto tests/core/test_hash +tests/core/test_hex tests/core/test_hxa tests/core/test_json tests/core/test_linalg_glsl_math diff --git a/base/intrinsics/intrinsics.odin b/base/intrinsics/intrinsics.odin index 458596adf..dca33bfd9 100644 --- a/base/intrinsics/intrinsics.odin +++ b/base/intrinsics/intrinsics.odin @@ -38,9 +38,9 @@ count_leading_zeros :: proc(x: $T) -> T where type_is_integer(T) || type_is_sim reverse_bits :: proc(x: $T) -> T where type_is_integer(T) || type_is_simd_vector(T) --- byte_swap :: proc(x: $T) -> T where type_is_integer(T) || type_is_float(T) --- -overflow_add :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- -overflow_sub :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- -overflow_mul :: proc(lhs, rhs: $T) -> (T, bool) #optional_ok --- +overflow_add :: proc(lhs, rhs: $T) -> (T, bool) --- +overflow_sub :: proc(lhs, rhs: $T) -> (T, bool) --- +overflow_mul :: proc(lhs, rhs: $T) -> (T, bool) --- sqrt :: proc(x: $T) -> T where type_is_float(T) || (type_is_simd_vector(T) && type_is_float(type_elem_type(T))) --- diff --git a/base/runtime/core_builtin.odin b/base/runtime/core_builtin.odin index c5c419de3..00c30d3fd 100644 --- a/base/runtime/core_builtin.odin +++ b/base/runtime/core_builtin.odin @@ -40,7 +40,7 @@ copy_slice :: proc "contextless" (dst, src: $T/[]$E) -> int { } return n } -// `copy_from_string` is a built-in procedure that copies elements from a source slice `src` to a destination string `dst`. +// `copy_from_string` is a built-in procedure that copies elements from a source string `src` to a destination slice `dst`. // The source and destination may overlap. Copy returns the number of elements copied, which will be the minimum // of len(src) and len(dst). // @@ -53,7 +53,7 @@ copy_from_string :: proc "contextless" (dst: $T/[]$E/u8, src: $S/string) -> int } return n } -// `copy` is a built-in procedure that copies elements from a source slice `src` to a destination slice/string `dst`. +// `copy` is a built-in procedure that copies elements from a source slice/string `src` to a destination slice `dst`. // The source and destination may overlap. Copy returns the number of elements copied, which will be the minimum // of len(src) and len(dst). @builtin diff --git a/base/runtime/error_checks.odin b/base/runtime/error_checks.odin index ea6333c29..742e06a71 100644 --- a/base/runtime/error_checks.odin +++ b/base/runtime/error_checks.odin @@ -19,6 +19,7 @@ type_assertion_trap :: proc "contextless" () -> ! { } +@(disabled=ODIN_NO_BOUNDS_CHECK) bounds_check_error :: proc "contextless" (file: string, line, column: i32, index, count: int) { if uint(index) < uint(count) { return @@ -61,6 +62,7 @@ multi_pointer_slice_handle_error :: proc "contextless" (file: string, line, colu } +@(disabled=ODIN_NO_BOUNDS_CHECK) multi_pointer_slice_expr_error :: proc "contextless" (file: string, line, column: i32, lo, hi: int) { if lo <= hi { return @@ -68,6 +70,7 @@ multi_pointer_slice_expr_error :: proc "contextless" (file: string, line, column multi_pointer_slice_handle_error(file, line, column, lo, hi) } +@(disabled=ODIN_NO_BOUNDS_CHECK) slice_expr_error_hi :: proc "contextless" (file: string, line, column: i32, hi: int, len: int) { if 0 <= hi && hi <= len { return @@ -75,6 +78,7 @@ slice_expr_error_hi :: proc "contextless" (file: string, line, column: i32, hi: slice_handle_error(file, line, column, 0, hi, len) } +@(disabled=ODIN_NO_BOUNDS_CHECK) slice_expr_error_lo_hi :: proc "contextless" (file: string, line, column: i32, lo, hi: int, len: int) { if 0 <= lo && lo <= len && lo <= hi && hi <= len { return @@ -82,6 +86,7 @@ slice_expr_error_lo_hi :: proc "contextless" (file: string, line, column: i32, l slice_handle_error(file, line, column, lo, hi, len) } +@(disabled=ODIN_NO_BOUNDS_CHECK) dynamic_array_expr_error :: proc "contextless" (file: string, line, column: i32, low, high, max: int) { if 0 <= low && low <= high && high <= max { return @@ -102,6 +107,7 @@ dynamic_array_expr_error :: proc "contextless" (file: string, line, column: i32, } +@(disabled=ODIN_NO_BOUNDS_CHECK) matrix_bounds_check_error :: proc "contextless" (file: string, line, column: i32, row_index, column_index, row_count, column_count: int) { if uint(row_index) < uint(row_count) && uint(column_index) < uint(column_count) { @@ -224,6 +230,7 @@ when ODIN_NO_RTTI { } +@(disabled=ODIN_NO_BOUNDS_CHECK) make_slice_error_loc :: #force_inline proc "contextless" (loc := #caller_location, len: int) { if 0 <= len { return @@ -239,6 +246,7 @@ make_slice_error_loc :: #force_inline proc "contextless" (loc := #caller_locatio handle_error(loc, len) } +@(disabled=ODIN_NO_BOUNDS_CHECK) make_dynamic_array_error_loc :: #force_inline proc "contextless" (loc := #caller_location, len, cap: int) { if 0 <= len && len <= cap { return @@ -256,6 +264,7 @@ make_dynamic_array_error_loc :: #force_inline proc "contextless" (loc := #caller handle_error(loc, len, cap) } +@(disabled=ODIN_NO_BOUNDS_CHECK) make_map_expr_error_loc :: #force_inline proc "contextless" (loc := #caller_location, cap: int) { if 0 <= cap { return @@ -274,19 +283,22 @@ make_map_expr_error_loc :: #force_inline proc "contextless" (loc := #caller_loca - +@(disabled=ODIN_NO_BOUNDS_CHECK) bounds_check_error_loc :: #force_inline proc "contextless" (loc := #caller_location, index, count: int) { bounds_check_error(loc.file_path, loc.line, loc.column, index, count) } +@(disabled=ODIN_NO_BOUNDS_CHECK) slice_expr_error_hi_loc :: #force_inline proc "contextless" (loc := #caller_location, hi: int, len: int) { slice_expr_error_hi(loc.file_path, loc.line, loc.column, hi, len) } +@(disabled=ODIN_NO_BOUNDS_CHECK) slice_expr_error_lo_hi_loc :: #force_inline proc "contextless" (loc := #caller_location, lo, hi: int, len: int) { slice_expr_error_lo_hi(loc.file_path, loc.line, loc.column, lo, hi, len) } +@(disabled=ODIN_NO_BOUNDS_CHECK) dynamic_array_expr_error_loc :: #force_inline proc "contextless" (loc := #caller_location, low, high, max: int) { dynamic_array_expr_error(loc.file_path, loc.line, loc.column, low, high, max) } diff --git a/base/runtime/internal.odin b/base/runtime/internal.odin index 6ca61c721..3e9c524bd 100644 --- a/base/runtime/internal.odin +++ b/base/runtime/internal.odin @@ -1042,19 +1042,17 @@ fixdfti :: proc(a: u64) -> i128 { __write_bits :: proc "contextless" (dst, src: [^]byte, offset: uintptr, size: uintptr) { for i in 0..>3]) & (1<<(i&7)) != 0) + dst[j>>3] &~= 1<<(j&7) + dst[j>>3] |= the_bit<<(j&7) } } __read_bits :: proc "contextless" (dst, src: [^]byte, offset: uintptr, size: uintptr) { for j in 0..>3]) & (1<<(i&7)) != 0) + dst[j>>3] &~= 1<<(j&7) + dst[j>>3] |= the_bit<<(j&7) } } \ No newline at end of file diff --git a/base/runtime/print.odin b/base/runtime/print.odin index c93c2ab49..ed5893e15 100644 --- a/base/runtime/print.odin +++ b/base/runtime/print.odin @@ -6,7 +6,7 @@ _INTEGER_DIGITS :: "0123456789abcdefghijklmnopqrstuvwxyz" _INTEGER_DIGITS_VAR := _INTEGER_DIGITS when !ODIN_NO_RTTI { - print_any_single :: proc "contextless" (arg: any) { + print_any_single :: #force_no_inline proc "contextless" (arg: any) { x := arg if x.data == nil { print_string("nil") @@ -72,7 +72,7 @@ when !ODIN_NO_RTTI { print_string("") } } - println_any :: proc "contextless" (args: ..any) { + println_any :: #force_no_inline proc "contextless" (args: ..any) { context = default_context() loop: for arg, i in args { assert(arg.id != nil) @@ -122,12 +122,12 @@ encode_rune :: proc "contextless" (c: rune) -> ([4]u8, int) { return buf, 4 } -print_string :: proc "contextless" (str: string) -> (n: int) { +print_string :: #force_no_inline proc "contextless" (str: string) -> (n: int) { n, _ = stderr_write(transmute([]byte)str) return } -print_strings :: proc "contextless" (args: ..string) -> (n: int) { +print_strings :: #force_no_inline proc "contextless" (args: ..string) -> (n: int) { for str in args { m, err := stderr_write(transmute([]byte)str) n += m @@ -138,12 +138,12 @@ print_strings :: proc "contextless" (args: ..string) -> (n: int) { return } -print_byte :: proc "contextless" (b: byte) -> (n: int) { +print_byte :: #force_no_inline proc "contextless" (b: byte) -> (n: int) { n, _ = stderr_write([]byte{b}) return } -print_encoded_rune :: proc "contextless" (r: rune) { +print_encoded_rune :: #force_no_inline proc "contextless" (r: rune) { print_byte('\'') switch r { @@ -170,7 +170,7 @@ print_encoded_rune :: proc "contextless" (r: rune) { print_byte('\'') } -print_rune :: proc "contextless" (r: rune) -> int #no_bounds_check { +print_rune :: #force_no_inline proc "contextless" (r: rune) -> int #no_bounds_check { RUNE_SELF :: 0x80 if r < RUNE_SELF { @@ -183,7 +183,7 @@ print_rune :: proc "contextless" (r: rune) -> int #no_bounds_check { } -print_u64 :: proc "contextless" (x: u64) #no_bounds_check { +print_u64 :: #force_no_inline proc "contextless" (x: u64) #no_bounds_check { a: [129]byte i := len(a) b := u64(10) @@ -198,7 +198,7 @@ print_u64 :: proc "contextless" (x: u64) #no_bounds_check { } -print_i64 :: proc "contextless" (x: i64) #no_bounds_check { +print_i64 :: #force_no_inline proc "contextless" (x: i64) #no_bounds_check { b :: i64(10) u := x @@ -223,25 +223,29 @@ print_uint :: proc "contextless" (x: uint) { print_u64(u64(x)) } print_uintptr :: proc "contextless" (x: uintptr) { print_u64(u64(x)) } print_int :: proc "contextless" (x: int) { print_i64(i64(x)) } -print_caller_location :: proc "contextless" (loc: Source_Code_Location) { +print_caller_location :: #force_no_inline proc "contextless" (loc: Source_Code_Location) { print_string(loc.file_path) when ODIN_ERROR_POS_STYLE == .Default { print_byte('(') print_u64(u64(loc.line)) - print_byte(':') - print_u64(u64(loc.column)) + if loc.column != 0 { + print_byte(':') + print_u64(u64(loc.column)) + } print_byte(')') } else when ODIN_ERROR_POS_STYLE == .Unix { print_byte(':') print_u64(u64(loc.line)) - print_byte(':') - print_u64(u64(loc.column)) + if loc.column != 0 { + print_byte(':') + print_u64(u64(loc.column)) + } print_byte(':') } else { #panic("unhandled ODIN_ERROR_POS_STYLE") } } -print_typeid :: proc "contextless" (id: typeid) { +print_typeid :: #force_no_inline proc "contextless" (id: typeid) { when ODIN_NO_RTTI { if id == nil { print_string("nil") @@ -257,7 +261,9 @@ print_typeid :: proc "contextless" (id: typeid) { } } } -print_type :: proc "contextless" (ti: ^Type_Info) { + +@(optimization_mode="size") +print_type :: #force_no_inline proc "contextless" (ti: ^Type_Info) { if ti == nil { print_string("nil") return diff --git a/build_odin.sh b/build_odin.sh index df4451060..cacd1a7ed 100755 --- a/build_odin.sh +++ b/build_odin.sh @@ -32,11 +32,11 @@ if [ -z "$LLVM_CONFIG" ]; then elif [ -n "$(command -v llvm-config-12)" ]; then LLVM_CONFIG="llvm-config-12" elif [ -n "$(command -v llvm-config-11)" ]; then LLVM_CONFIG="llvm-config-11" # freebsd - elif [ -n "$(command -v llvm-config17)" ]; then LLVM_CONFIG="llvm-config-17" - elif [ -n "$(command -v llvm-config14)" ]; then LLVM_CONFIG="llvm-config-14" - elif [ -n "$(command -v llvm-config13)" ]; then LLVM_CONFIG="llvm-config-13" - elif [ -n "$(command -v llvm-config12)" ]; then LLVM_CONFIG="llvm-config-12" - elif [ -n "$(command -v llvm-config11)" ]; then LLVM_CONFIG="llvm-config-11" + elif [ -n "$(command -v llvm-config17)" ]; then LLVM_CONFIG="llvm-config17" + elif [ -n "$(command -v llvm-config14)" ]; then LLVM_CONFIG="llvm-config14" + elif [ -n "$(command -v llvm-config13)" ]; then LLVM_CONFIG="llvm-config13" + elif [ -n "$(command -v llvm-config12)" ]; then LLVM_CONFIG="llvm-config12" + elif [ -n "$(command -v llvm-config11)" ]; then LLVM_CONFIG="llvm-config11" # fallback elif [ -n "$(command -v llvm-config)" ]; then LLVM_CONFIG="llvm-config" else diff --git a/core/bufio/reader.odin b/core/bufio/reader.odin index e5307f105..8ec736a66 100644 --- a/core/bufio/reader.odin +++ b/core/bufio/reader.odin @@ -81,7 +81,7 @@ _reader_read_new_chunk :: proc(b: ^Reader) -> io.Error { for i := b.max_consecutive_empty_reads; i > 0; i -= 1 { n, err := io.read(b.rd, b.buf[b.w:]) if n < 0 { - return .Negative_Read + return err if err != nil else .Negative_Read } b.w += n if err != nil { @@ -189,7 +189,7 @@ reader_read :: proc(b: ^Reader, p: []byte) -> (n: int, err: io.Error) { if len(p) >= len(b.buf) { n, b.err = io.read(b.rd, p) if n < 0 { - return 0, .Negative_Read + return 0, b.err if b.err != nil else .Negative_Read } if n > 0 { @@ -202,7 +202,7 @@ reader_read :: proc(b: ^Reader, p: []byte) -> (n: int, err: io.Error) { b.r, b.w = 0, 0 n, b.err = io.read(b.rd, b.buf) if n < 0 { - return 0, .Negative_Read + return 0, b.err if b.err != nil else .Negative_Read } if n == 0 { return 0, _reader_consume_err(b) @@ -290,7 +290,7 @@ reader_write_to :: proc(b: ^Reader, w: io.Writer) -> (n: i64, err: io.Error) { write_buf :: proc(b: ^Reader, w: io.Writer) -> (i64, io.Error) { n, err := io.write(w, b.buf[b.r:b.w]) if n < 0 { - return 0, .Negative_Write + return 0, err if err != nil else .Negative_Write } b.r += n return i64(n), err diff --git a/core/bufio/writer.odin b/core/bufio/writer.odin index 3c7fd30c5..5edd3dd6b 100644 --- a/core/bufio/writer.odin +++ b/core/bufio/writer.odin @@ -95,6 +95,10 @@ writer_write :: proc(b: ^Writer, p: []byte) -> (n: int, err: io.Error) { m: int if writer_buffered(b) == 0 { m, b.err = io.write(b.wr, p) + if m < 0 && b.err == nil { + b.err = .Negative_Write + break + } } else { m = copy(b.buf[b.n:], p) b.n += m diff --git a/core/bytes/buffer.odin b/core/bytes/buffer.odin index abfee6f2f..cb2ef9c62 100644 --- a/core/bytes/buffer.odin +++ b/core/bytes/buffer.odin @@ -359,7 +359,7 @@ buffer_read_from :: proc(b: ^Buffer, r: io.Reader) -> (n: i64, err: io.Error) #n resize(&b.buf, i) m, e := io.read(r, b.buf[i:cap(b.buf)]) if m < 0 { - err = .Negative_Read + err = e if e != nil else .Negative_Read return } diff --git a/core/c/frontend/tokenizer/doc.odin b/core/c/frontend/tokenizer/doc.odin index 9b1734fc4..43747dfe8 100644 --- a/core/c/frontend/tokenizer/doc.odin +++ b/core/c/frontend/tokenizer/doc.odin @@ -1,34 +1,31 @@ /* -package demo +Example: + package demo -import tokenizer "core:c/frontend/tokenizer" -import preprocessor "core:c/frontend/preprocessor" -import "core:fmt" + import tokenizer "core:c/frontend/tokenizer" + import preprocessor "core:c/frontend/preprocessor" + import "core:fmt" -main :: proc() { - t := &tokenizer.Tokenizer{}; - tokenizer.init_defaults(t); + main :: proc() { + t := &tokenizer.Tokenizer{}; + tokenizer.init_defaults(t); - cpp := &preprocessor.Preprocessor{}; - cpp.warn, cpp.err = t.warn, t.err; - preprocessor.init_lookup_tables(cpp); - preprocessor.init_default_macros(cpp); - cpp.include_paths = {"my/path/to/include"}; + cpp := &preprocessor.Preprocessor{}; + cpp.warn, cpp.err = t.warn, t.err; + preprocessor.init_lookup_tables(cpp); + preprocessor.init_default_macros(cpp); + cpp.include_paths = {"my/path/to/include"}; - tok := tokenizer.tokenize_file(t, "the/source/file.c", 1); + tok := tokenizer.tokenize_file(t, "the/source/file.c", 1); - tok = preprocessor.preprocess(cpp, tok); - if tok != nil { - for t := tok; t.kind != .EOF; t = t.next { - fmt.println(t.lit); + tok = preprocessor.preprocess(cpp, tok); + if tok != nil { + for t := tok; t.kind != .EOF; t = t.next { + fmt.println(t.lit); + } } + + fmt.println("[Done]"); } - - fmt.println("[Done]"); -} */ - - package c_frontend_tokenizer - - diff --git a/core/compress/gzip/example.odin b/core/compress/gzip/example.odin index 635134e40..09540aafc 100644 --- a/core/compress/gzip/example.odin +++ b/core/compress/gzip/example.odin @@ -1,5 +1,5 @@ //+build ignore -package gzip +package compress_gzip /* Copyright 2021 Jeroen van Rijn . diff --git a/core/compress/gzip/gzip.odin b/core/compress/gzip/gzip.odin index 50945fc77..57ed3c3c5 100644 --- a/core/compress/gzip/gzip.odin +++ b/core/compress/gzip/gzip.odin @@ -1,4 +1,4 @@ -package gzip +package compress_gzip /* Copyright 2021 Jeroen van Rijn . diff --git a/core/compress/shoco/model.odin b/core/compress/shoco/model.odin index bbc38903d..f62236c00 100644 --- a/core/compress/shoco/model.odin +++ b/core/compress/shoco/model.odin @@ -5,7 +5,7 @@ */ // package shoco is an implementation of the shoco short string compressor -package shoco +package compress_shoco DEFAULT_MODEL :: Shoco_Model { min_char = 39, diff --git a/core/compress/shoco/shoco.odin b/core/compress/shoco/shoco.odin index e65acb0bc..3c1f412ba 100644 --- a/core/compress/shoco/shoco.odin +++ b/core/compress/shoco/shoco.odin @@ -9,7 +9,7 @@ */ // package shoco is an implementation of the shoco short string compressor -package shoco +package compress_shoco import "base:intrinsics" import "core:compress" diff --git a/core/compress/zlib/example.odin b/core/compress/zlib/example.odin index 19017b279..fedd6671d 100644 --- a/core/compress/zlib/example.odin +++ b/core/compress/zlib/example.odin @@ -1,5 +1,5 @@ //+build ignore -package zlib +package compress_zlib /* Copyright 2021 Jeroen van Rijn . diff --git a/core/compress/zlib/zlib.odin b/core/compress/zlib/zlib.odin index d4dc6e3d7..b7f381f2b 100644 --- a/core/compress/zlib/zlib.odin +++ b/core/compress/zlib/zlib.odin @@ -1,5 +1,5 @@ //+vet !using-param -package zlib +package compress_zlib /* Copyright 2021 Jeroen van Rijn . diff --git a/core/container/bit_array/bit_array.odin b/core/container/bit_array/bit_array.odin index dbd2e0d3a..a8720715c 100644 --- a/core/container/bit_array/bit_array.odin +++ b/core/container/bit_array/bit_array.odin @@ -1,4 +1,4 @@ -package dynamic_bit_array +package container_dynamic_bit_array import "base:intrinsics" import "core:mem" diff --git a/core/container/bit_array/doc.odin b/core/container/bit_array/doc.odin index 52e252d8a..77e1904a8 100644 --- a/core/container/bit_array/doc.odin +++ b/core/container/bit_array/doc.odin @@ -1,53 +1,52 @@ -package dynamic_bit_array - /* - The Bit Array can be used in several ways: +The Bit Array can be used in several ways: - -- By default you don't need to instantiate a Bit Array: +- By default you don't need to instantiate a Bit Array: - package test + package test - import "core:fmt" - import "core:container/bit_array" + import "core:fmt" + import "core:container/bit_array" - main :: proc() { - using bit_array + main :: proc() { + using bit_array - bits: Bit_Array + bits: Bit_Array - // returns `true` - fmt.println(set(&bits, 42)) + // returns `true` + fmt.println(set(&bits, 42)) - // returns `false`, `false`, because this Bit Array wasn't created to allow negative indices. - was_set, was_retrieved := get(&bits, -1) - fmt.println(was_set, was_retrieved) - destroy(&bits) + // returns `false`, `false`, because this Bit Array wasn't created to allow negative indices. + was_set, was_retrieved := get(&bits, -1) + fmt.println(was_set, was_retrieved) + destroy(&bits) + } + +- A Bit Array can optionally allow for negative indices, if the minimum value was given during creation: + + package test + + import "core:fmt" + import "core:container/bit_array" + + main :: proc() { + Foo :: enum int { + Negative_Test = -42, + Bar = 420, + Leaves = 69105, } - -- A Bit Array can optionally allow for negative indices, if the mininum value was given during creation: + using bit_array - package test + bits := create(int(max(Foo)), int(min(Foo))) + defer destroy(bits) - import "core:fmt" - import "core:container/bit_array" - - main :: proc() { - Foo :: enum int { - Negative_Test = -42, - Bar = 420, - Leaves = 69105, - } - - using bit_array - - bits := create(int(max(Foo)), int(min(Foo))) - defer destroy(bits) - - fmt.printf("Set(Bar): %v\n", set(bits, Foo.Bar)) - fmt.printf("Get(Bar): %v, %v\n", get(bits, Foo.Bar)) - fmt.printf("Set(Negative_Test): %v\n", set(bits, Foo.Negative_Test)) - fmt.printf("Get(Leaves): %v, %v\n", get(bits, Foo.Leaves)) - fmt.printf("Get(Negative_Test): %v, %v\n", get(bits, Foo.Negative_Test)) - fmt.printf("Freed.\n") - } -*/ \ No newline at end of file + fmt.printf("Set(Bar): %v\n", set(bits, Foo.Bar)) + fmt.printf("Get(Bar): %v, %v\n", get(bits, Foo.Bar)) + fmt.printf("Set(Negative_Test): %v\n", set(bits, Foo.Negative_Test)) + fmt.printf("Get(Leaves): %v, %v\n", get(bits, Foo.Leaves)) + fmt.printf("Get(Negative_Test): %v, %v\n", get(bits, Foo.Negative_Test)) + fmt.printf("Freed.\n") + } +*/ +package container_dynamic_bit_array diff --git a/core/crypto/crypto.odin b/core/crypto/crypto.odin index 05f25111a..f0874cc6d 100644 --- a/core/crypto/crypto.odin +++ b/core/crypto/crypto.odin @@ -49,15 +49,12 @@ compare_byte_ptrs_constant_time :: proc "contextless" (a, b: ^byte, n: int) -> i // the system entropy source. This routine will block if the system entropy // source is not ready yet. All system entropy source failures are treated // as catastrophic, resulting in a panic. +// +// Support for the system entropy source can be checked with the +// `HAS_RAND_BYTES` boolean constant. rand_bytes :: proc (dst: []byte) { // zero-fill the buffer first mem.zero_explicit(raw_data(dst), len(dst)) _rand_bytes(dst) } - -// has_rand_bytes returns true iff the target has support for accessing the -// system entropty source. -has_rand_bytes :: proc () -> bool { - return _has_rand_bytes() -} diff --git a/core/crypto/rand_bsd.odin b/core/crypto/rand_bsd.odin index 658221769..641b72933 100644 --- a/core/crypto/rand_bsd.odin +++ b/core/crypto/rand_bsd.odin @@ -3,14 +3,13 @@ package crypto foreign import libc "system:c" +HAS_RAND_BYTES :: true + foreign libc { arc4random_buf :: proc(buf: [^]byte, nbytes: uint) --- } +@(private) _rand_bytes :: proc(dst: []byte) { arc4random_buf(raw_data(dst), len(dst)) } - -_has_rand_bytes :: proc () -> bool { - return true -} diff --git a/core/crypto/rand_darwin.odin b/core/crypto/rand_darwin.odin index 2864b46dd..5355f31c5 100644 --- a/core/crypto/rand_darwin.odin +++ b/core/crypto/rand_darwin.odin @@ -1,16 +1,17 @@ package crypto import "core:fmt" -import "core:sys/darwin" +import CF "core:sys/darwin/CoreFoundation" +import Sec "core:sys/darwin/Security" + +HAS_RAND_BYTES :: true + +@(private) _rand_bytes :: proc(dst: []byte) { - res := darwin.SecRandomCopyBytes(count=len(dst), bytes=raw_data(dst)) - if res != .Success { - msg := darwin.CFStringCopyToOdinString(darwin.SecCopyErrorMessageString(res)) - panic(fmt.tprintf("crypto/rand_bytes: SecRandomCopyBytes returned non-zero result: %v %s", res, msg)) + err := Sec.RandomCopyBytes(count=len(dst), bytes=raw_data(dst)) + if err != .Success { + msg := CF.StringCopyToOdinString(Sec.CopyErrorMessageString(err)) + panic(fmt.tprintf("crypto/rand_bytes: SecRandomCopyBytes returned non-zero result: %v %s", err, msg)) } } - -_has_rand_bytes :: proc () -> bool { - return true -} diff --git a/core/crypto/rand_generic.odin b/core/crypto/rand_generic.odin index cab76418b..46fb881b3 100644 --- a/core/crypto/rand_generic.odin +++ b/core/crypto/rand_generic.odin @@ -7,10 +7,9 @@ //+build !js package crypto +HAS_RAND_BYTES :: false + +@(private) _rand_bytes :: proc(dst: []byte) { unimplemented("crypto: rand_bytes not supported on this OS") } - -_has_rand_bytes :: proc () -> bool { - return false -} diff --git a/core/crypto/rand_js.odin b/core/crypto/rand_js.odin index cb2711404..72093810e 100644 --- a/core/crypto/rand_js.odin +++ b/core/crypto/rand_js.odin @@ -6,8 +6,12 @@ foreign odin_env { env_rand_bytes :: proc "contextless" (buf: []byte) --- } +HAS_RAND_BYTES :: true + +@(private) _MAX_PER_CALL_BYTES :: 65536 // 64kiB +@(private) _rand_bytes :: proc(dst: []byte) { dst := dst @@ -18,7 +22,3 @@ _rand_bytes :: proc(dst: []byte) { dst = dst[to_read:] } } - -_has_rand_bytes :: proc () -> bool { - return true -} diff --git a/core/crypto/rand_linux.odin b/core/crypto/rand_linux.odin index 05c05597d..43b3b3075 100644 --- a/core/crypto/rand_linux.odin +++ b/core/crypto/rand_linux.odin @@ -4,8 +4,12 @@ import "core:fmt" import "core:sys/linux" +HAS_RAND_BYTES :: true + +@(private) _MAX_PER_CALL_BYTES :: 33554431 // 2^25 - 1 +@(private) _rand_bytes :: proc (dst: []byte) { dst := dst l := len(dst) @@ -34,7 +38,3 @@ _rand_bytes :: proc (dst: []byte) { dst = dst[n_read:] } } - -_has_rand_bytes :: proc () -> bool { - return true -} diff --git a/core/crypto/rand_windows.odin b/core/crypto/rand_windows.odin index e1d9f6118..a92d376cb 100644 --- a/core/crypto/rand_windows.odin +++ b/core/crypto/rand_windows.odin @@ -4,6 +4,9 @@ import win32 "core:sys/windows" import "core:os" import "core:fmt" +HAS_RAND_BYTES :: true + +@(private) _rand_bytes :: proc(dst: []byte) { ret := (os.Errno)(win32.BCryptGenRandom(nil, raw_data(dst), u32(len(dst)), win32.BCRYPT_USE_SYSTEM_PREFERRED_RNG)) if ret != os.ERROR_NONE { @@ -21,7 +24,3 @@ _rand_bytes :: proc(dst: []byte) { } } } - -_has_rand_bytes :: proc () -> bool { - return true -} diff --git a/core/debug/trace/doc.odin b/core/debug/trace/doc.odin new file mode 100644 index 000000000..e65548769 --- /dev/null +++ b/core/debug/trace/doc.odin @@ -0,0 +1,51 @@ +/* +A debug stack trace library. Only works when debug symbols are enabled `-debug`. + +Example: + import "base:runtime" + import "core:debug/trace" + + import "core:fmt" + + global_trace_ctx: trace.Context + + debug_trace_assertion_failure_proc :: proc(prefix, message: string, loc := #caller_location) -> ! { + runtime.print_caller_location(loc) + runtime.print_string(" ") + runtime.print_string(prefix) + if len(message) > 0 { + runtime.print_string(": ") + runtime.print_string(message) + } + runtime.print_byte('\n') + + ctx := &trace_ctx + if !trace.in_resolve(ctx) { + buf: [64]trace.Frame + runtime.print_string("Debug Trace:\n") + frames := trace.frames(ctx, 1, buf[:]) + for f, i in frames { + fl := trace.resolve(ctx, f, context.temp_allocator) + if fl.loc.file_path == "" && fl.loc.line == 0 { + continue + } + runtime.print_caller_location(fl.loc) + runtime.print_string(" - frame ") + runtime.print_int(i) + runtime.print_byte('\n') + } + } + runtime.trap() + } + + main :: proc() { + trace.init(&global_trace_ctx) + defer trace.destroy(&global_trace_ctx) + + context.assertion_failure_proc = debug_trace_assertion_failure_proc + + ... + } + +*/ +package debug_trace \ No newline at end of file diff --git a/core/debug/trace/trace.odin b/core/debug/trace/trace.odin new file mode 100644 index 000000000..134609b05 --- /dev/null +++ b/core/debug/trace/trace.odin @@ -0,0 +1,47 @@ +package debug_trace + +import "base:intrinsics" +import "base:runtime" + +Frame :: distinct uintptr + +Frame_Location :: struct { + using loc: runtime.Source_Code_Location, + allocator: runtime.Allocator, +} + +delete_frame_location :: proc(fl: Frame_Location) -> runtime.Allocator_Error { + allocator := fl.allocator + delete(fl.loc.procedure, allocator) or_return + delete(fl.loc.file_path, allocator) or_return + return nil +} + +Context :: struct { + in_resolve: bool, // atomic + impl: _Context, +} + +init :: proc(ctx: ^Context) -> bool { + return _init(ctx) +} + +destroy :: proc(ctx: ^Context) -> bool { + return _destroy(ctx) +} + +@(require_results) +frames :: proc(ctx: ^Context, skip: uint, frames_buffer: []Frame) -> []Frame { + return _frames(ctx, skip, frames_buffer) +} + +@(require_results) +resolve :: proc(ctx: ^Context, frame: Frame, allocator: runtime.Allocator) -> (result: Frame_Location) { + return _resolve(ctx, frame, allocator) +} + + +@(require_results) +in_resolve :: proc "contextless" (ctx: ^Context) -> bool { + return intrinsics.atomic_load(&ctx.in_resolve) +} \ No newline at end of file diff --git a/core/debug/trace/trace_cpp.odin b/core/debug/trace/trace_cpp.odin new file mode 100644 index 000000000..894046c45 --- /dev/null +++ b/core/debug/trace/trace_cpp.odin @@ -0,0 +1,195 @@ +//+private file +//+build linux, darwin +package debug_trace + +import "base:intrinsics" +import "base:runtime" +import "core:strings" +import "core:fmt" +import "core:c" + +// NOTE: Relies on C++23 which adds and becomes ABI and that can be used +foreign import stdcpplibbacktrace "system:stdc++_libbacktrace" + +foreign import libdl "system:dl" + +backtrace_state :: struct {} +backtrace_error_callback :: proc "c" (data: rawptr, msg: cstring, errnum: c.int) +backtrace_simple_callback :: proc "c" (data: rawptr, pc: uintptr) -> c.int +backtrace_full_callback :: proc "c" (data: rawptr, pc: uintptr, filename: cstring, lineno: c.int, function: cstring) -> c.int +backtrace_syminfo_callback :: proc "c" (data: rawptr, pc: uintptr, symname: cstring, symval: uintptr, symsize: uintptr) + +@(default_calling_convention="c", link_prefix="__glibcxx_") +foreign stdcpplibbacktrace { + backtrace_create_state :: proc( + filename: cstring, + threaded: c.int, + error_callback: backtrace_error_callback, + data: rawptr, + ) -> ^backtrace_state --- + backtrace_simple :: proc( + state: ^backtrace_state, + skip: c.int, + callback: backtrace_simple_callback, + error_callback: backtrace_error_callback, + data: rawptr, + ) -> c.int --- + backtrace_pcinfo :: proc( + state: ^backtrace_state, + pc: uintptr, + callback: backtrace_full_callback, + error_callback: backtrace_error_callback, + data: rawptr, + ) -> c.int --- + backtrace_syminfo :: proc( + state: ^backtrace_state, + addr: uintptr, + callback: backtrace_syminfo_callback, + error_callback: backtrace_error_callback, + data: rawptr, + ) -> c.int --- + + // NOTE(bill): this is technically an internal procedure, but it is exposed + backtrace_free :: proc( + state: ^backtrace_state, + p: rawptr, + size: c.size_t, // unused + error_callback: backtrace_error_callback, // unused + data: rawptr, // unused + ) --- +} + +Dl_info :: struct { + dli_fname: cstring, + dli_fbase: rawptr, + dli_sname: cstring, + dli_saddr: rawptr, +} + +@(default_calling_convention="c") +foreign libdl { + dladdr :: proc(addr: rawptr, info: ^Dl_info) -> c.int --- +} + +@(private="package") +_Context :: struct { + state: ^backtrace_state, +} + +@(private="package") +_init :: proc(ctx: ^Context) -> (ok: bool) { + defer if !ok do destroy(ctx) + + ctx.impl.state = backtrace_create_state("odin-debug-trace", 1, nil, ctx) + return ctx.impl.state != nil +} + +@(private="package") +_destroy :: proc(ctx: ^Context) -> bool { + if ctx != nil { + backtrace_free(ctx.impl.state, nil, 0, nil, nil) + } + return true +} + +@(private="package") +_frames :: proc "contextless" (ctx: ^Context, skip: uint, frames_buffer: []Frame) -> (frames: []Frame) { + Backtrace_Context :: struct { + ctx: ^Context, + frames: []Frame, + frame_count: int, + } + + btc := &Backtrace_Context{ + ctx = ctx, + frames = frames_buffer, + } + backtrace_simple( + ctx.impl.state, + c.int(skip + 2), + proc "c" (user: rawptr, address: uintptr) -> c.int { + btc := (^Backtrace_Context)(user) + address := Frame(address) + if address == 0 { + return 1 + } + if btc.frame_count == len(btc.frames) { + return 1 + } + btc.frames[btc.frame_count] = address + btc.frame_count += 1 + return 0 + }, + nil, + btc, + ) + + if btc.frame_count > 0 { + frames = btc.frames[:btc.frame_count] + } + return +} + +@(private="package") +_resolve :: proc(ctx: ^Context, frame: Frame, allocator: runtime.Allocator) -> Frame_Location { + intrinsics.atomic_store(&ctx.in_resolve, true) + defer intrinsics.atomic_store(&ctx.in_resolve, false) + + Backtrace_Context :: struct { + rt_ctx: runtime.Context, + allocator: runtime.Allocator, + frame: Frame_Location, + } + + btc := &Backtrace_Context{ + rt_ctx = context, + allocator = allocator, + } + done := backtrace_pcinfo( + ctx.impl.state, + uintptr(frame), + proc "c" (data: rawptr, address: uintptr, file: cstring, line: c.int, symbol: cstring) -> c.int { + btc := (^Backtrace_Context)(data) + context = btc.rt_ctx + + frame := &btc.frame + + if file != nil { + frame.file_path = strings.clone_from_cstring(file, btc.allocator) + } else if info: Dl_info; dladdr(rawptr(address), &info) != 0 && info.dli_fname != "" { + frame.file_path = strings.clone_from_cstring(info.dli_fname, btc.allocator) + } + if symbol != nil { + frame.procedure = strings.clone_from_cstring(symbol, btc.allocator) + } else if info: Dl_info; dladdr(rawptr(address), &info) != 0 && info.dli_sname != "" { + frame.procedure = strings.clone_from_cstring(info.dli_sname, btc.allocator) + } else { + frame.procedure = fmt.aprintf("(procedure: 0x%x)", allocator=btc.allocator) + } + frame.line = i32(line) + return 0 + }, + nil, + btc, + ) + if done != 0 { + return btc.frame + } + + // NOTE(bill): pcinfo cannot resolve, but it might be possible to get the procedure name at least + backtrace_syminfo( + ctx.impl.state, + uintptr(frame), + proc "c" (data: rawptr, address: uintptr, symbol: cstring, _ignore0, _ignore1: uintptr) { + if symbol != nil { + btc := (^Backtrace_Context)(data) + context = btc.rt_ctx + btc.frame.procedure = strings.clone_from_cstring(symbol, btc.allocator) + } + }, + nil, + btc, + ) + + return btc.frame +} \ No newline at end of file diff --git a/core/debug/trace/trace_nil.odin b/core/debug/trace/trace_nil.odin new file mode 100644 index 000000000..40478898b --- /dev/null +++ b/core/debug/trace/trace_nil.odin @@ -0,0 +1,18 @@ +//+build !windows !linux !darwin +package debug_trace + +_Context :: struct { +} + +_init :: proc(ctx: ^Context) -> (ok: bool) { + return true +} +_destroy :: proc(ctx: ^Context) -> bool { + return true +} +_frames :: proc(ctx: ^Context, skip: uint, allocator: runtime.Allocator) -> []Frame { + return nil +} +_resolve :: proc(ctx: ^Context, frame: Frame, allocator: runtime.Allocator) -> (result: runtime.Source_Code_Location) { + return +} diff --git a/core/debug/trace/trace_windows.odin b/core/debug/trace/trace_windows.odin new file mode 100644 index 000000000..de1461e96 --- /dev/null +++ b/core/debug/trace/trace_windows.odin @@ -0,0 +1,68 @@ +//+private +//+build windows +package debug_trace + +import "base:intrinsics" +import "base:runtime" + +import win32 "core:sys/windows" +import "core:fmt" + +_Context :: struct { + hProcess: win32.HANDLE, + lock: win32.SRWLOCK, +} + +_init :: proc "contextless" (ctx: ^Context) -> (ok: bool) { + defer if !ok { _destroy(ctx) } + ctx.impl.hProcess = win32.GetCurrentProcess() + win32.SymInitialize(ctx.impl.hProcess, nil, true) or_return + win32.SymSetOptions(win32.SYMOPT_LOAD_LINES) + return true +} + +_destroy :: proc "contextless" (ctx: ^Context) -> bool { + if ctx != nil { + win32.SymCleanup(ctx.impl.hProcess) + } + return true +} + +_frames :: proc "contextless" (ctx: ^Context, skip: uint, frames_buffer: []Frame) -> []Frame { + frame_count := win32.RtlCaptureStackBackTrace(u32(skip) + 2, u32(len(frames_buffer)), ([^]rawptr)(&frames_buffer[0]), nil) + for i in 0.. (fl: Frame_Location) { + intrinsics.atomic_store(&ctx.in_resolve, true) + defer intrinsics.atomic_store(&ctx.in_resolve, false) + + // NOTE(bill): Dbghelp is not thread-safe + win32.AcquireSRWLockExclusive(&ctx.impl.lock) + defer win32.ReleaseSRWLockExclusive(&ctx.impl.lock) + + data: [size_of(win32.SYMBOL_INFOW) + size_of([256]win32.WCHAR)]byte + symbol := (^win32.SYMBOL_INFOW)(&data[0]) + symbol.SizeOfStruct = size_of(symbol) + symbol.MaxNameLen = 255 + if win32.SymFromAddrW(ctx.impl.hProcess, win32.DWORD64(frame), &{}, symbol) { + fl.procedure, _ = win32.wstring_to_utf8(&symbol.Name[0], -1, allocator) + } else { + fl.procedure = fmt.aprintf("(procedure: 0x%x)", frame, allocator=allocator) + } + + line: win32.IMAGEHLP_LINE64 + line.SizeOfStruct = size_of(line) + if win32.SymGetLineFromAddrW64(ctx.impl.hProcess, win32.DWORD64(frame), &{}, &line) { + fl.file_path, _ = win32.wstring_to_utf8(line.FileName, -1, allocator) + fl.line = i32(line.LineNumber) + } + + return +} \ No newline at end of file diff --git a/core/dynlib/doc.odin b/core/dynlib/doc.odin index 849e03a71..f5c91c54e 100644 --- a/core/dynlib/doc.odin +++ b/core/dynlib/doc.odin @@ -1,6 +1,5 @@ -//+build ignore /* -Package core:dynlib implements loading of shared libraries/DLLs and their symbols. +Package `core:dynlib` implements loading of shared libraries/DLLs and their symbols. The behaviour of dynamically loaded libraries is specific to the target platform of the program. For in depth detail on the underlying behaviour please refer to your target platform's documentation. @@ -8,4 +7,4 @@ For in depth detail on the underlying behaviour please refer to your target plat See `example` directory for an example library exporting 3 symbols and a host program loading them automatically by defining a symbol table struct. */ -package dynlib \ No newline at end of file +package dynlib diff --git a/core/dynlib/lib.odin b/core/dynlib/lib.odin index 30d55edae..3d41cbe2e 100644 --- a/core/dynlib/lib.odin +++ b/core/dynlib/lib.odin @@ -135,7 +135,7 @@ initialize_symbols :: proc( prefixed_symbol_buf: [2048]u8 = --- count = 0 - for field, i in reflect.struct_fields_zipped(T) { + for field in reflect.struct_fields_zipped(T) { // Calculate address of struct member field_ptr := rawptr(uintptr(symbol_table) + field.offset) diff --git a/core/encoding/base32/base32.odin b/core/encoding/base32/base32.odin index 7ab35afd0..962a3ead4 100644 --- a/core/encoding/base32/base32.odin +++ b/core/encoding/base32/base32.odin @@ -1,4 +1,4 @@ -package base32 +package encoding_base32 // @note(zh): Encoding utility for Base32 // A secondary param can be used to supply a custom alphabet to diff --git a/core/encoding/base64/base64.odin b/core/encoding/base64/base64.odin index cf2ea1c12..1013a7d0b 100644 --- a/core/encoding/base64/base64.odin +++ b/core/encoding/base64/base64.odin @@ -1,4 +1,8 @@ -package base64 +package encoding_base64 + +import "core:io" +import "core:mem" +import "core:strings" // @note(zh): Encoding utility for Base64 // A secondary param can be used to supply a custom alphabet to @@ -39,59 +43,132 @@ DEC_TABLE := [128]int { 49, 50, 51, -1, -1, -1, -1, -1, } -encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> string #no_bounds_check { - length := len(data) - if length == 0 { - return "" - } +encode :: proc(data: []byte, ENC_TBL := ENC_TABLE, allocator := context.allocator) -> (encoded: string, err: mem.Allocator_Error) #optional_allocator_error { + out_length := encoded_len(data) + if out_length == 0 { + return + } - out_length := ((4 * length / 3) + 3) &~ 3 - out := make([]byte, out_length, allocator) + out := strings.builder_make(0, out_length, allocator) or_return + ioerr := encode_into(strings.to_stream(&out), data, ENC_TBL) - c0, c1, c2, block: int + assert(ioerr == nil, "string builder should not IO error") + assert(strings.builder_cap(out) == out_length, "buffer resized, `encoded_len` was wrong") - for i, d := 0, 0; i < length; i, d = i + 3, d + 4 { - c0, c1, c2 = int(data[i]), -1, -1 - - if i + 1 < length { c1 = int(data[i + 1]) } - if i + 2 < length { c2 = int(data[i + 2]) } - - block = (c0 << 16) | (max(c1, 0) << 8) | max(c2, 0) - - out[d] = ENC_TBL[block >> 18 & 63] - out[d + 1] = ENC_TBL[block >> 12 & 63] - out[d + 2] = c1 == -1 ? PADDING : ENC_TBL[block >> 6 & 63] - out[d + 3] = c2 == -1 ? PADDING : ENC_TBL[block & 63] - } - return string(out) + return strings.to_string(out), nil } -decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> []byte #no_bounds_check { - length := len(data) - if length == 0 { - return nil - } +encode_into :: proc(w: io.Writer, data: []byte, ENC_TBL := ENC_TABLE) -> io.Error { + length := len(data) + if length == 0 { + return nil + } - pad_count := data[length - 1] == PADDING ? (data[length - 2] == PADDING ? 2 : 1) : 0 - out_length := ((length * 6) >> 3) - pad_count - out := make([]byte, out_length, allocator) + c0, c1, c2, block: int + out: [4]byte + for i := 0; i < length; i += 3 { + #no_bounds_check { + c0, c1, c2 = int(data[i]), -1, -1 - c0, c1, c2, c3: int - b0, b1, b2: int + if i + 1 < length { c1 = int(data[i + 1]) } + if i + 2 < length { c2 = int(data[i + 2]) } - for i, j := 0, 0; i < length; i, j = i + 4, j + 3 { - c0 = DEC_TBL[data[i]] - c1 = DEC_TBL[data[i + 1]] - c2 = DEC_TBL[data[i + 2]] - c3 = DEC_TBL[data[i + 3]] - - b0 = (c0 << 2) | (c1 >> 4) - b1 = (c1 << 4) | (c2 >> 2) - b2 = (c2 << 6) | c3 - - out[j] = byte(b0) - out[j + 1] = byte(b1) - out[j + 2] = byte(b2) - } - return out + block = (c0 << 16) | (max(c1, 0) << 8) | max(c2, 0) + + out[0] = ENC_TBL[block >> 18 & 63] + out[1] = ENC_TBL[block >> 12 & 63] + out[2] = c1 == -1 ? PADDING : ENC_TBL[block >> 6 & 63] + out[3] = c2 == -1 ? PADDING : ENC_TBL[block & 63] + } + io.write_full(w, out[:]) or_return + } + return nil +} + +encoded_len :: proc(data: []byte) -> int { + length := len(data) + if length == 0 { + return 0 + } + + return ((4 * length / 3) + 3) &~ 3 +} + +decode :: proc(data: string, DEC_TBL := DEC_TABLE, allocator := context.allocator) -> (decoded: []byte, err: mem.Allocator_Error) #optional_allocator_error { + out_length := decoded_len(data) + + out := strings.builder_make(0, out_length, allocator) or_return + ioerr := decode_into(strings.to_stream(&out), data, DEC_TBL) + + assert(ioerr == nil, "string builder should not IO error") + assert(strings.builder_cap(out) == out_length, "buffer resized, `decoded_len` was wrong") + + return out.buf[:], nil +} + +decode_into :: proc(w: io.Writer, data: string, DEC_TBL := DEC_TABLE) -> io.Error { + length := decoded_len(data) + if length == 0 { + return nil + } + + c0, c1, c2, c3: int + b0, b1, b2: int + buf: [3]byte + i, j: int + for ; j + 3 <= length; i, j = i + 4, j + 3 { + #no_bounds_check { + c0 = DEC_TBL[data[i]] + c1 = DEC_TBL[data[i + 1]] + c2 = DEC_TBL[data[i + 2]] + c3 = DEC_TBL[data[i + 3]] + + b0 = (c0 << 2) | (c1 >> 4) + b1 = (c1 << 4) | (c2 >> 2) + b2 = (c2 << 6) | c3 + + buf[0] = byte(b0) + buf[1] = byte(b1) + buf[2] = byte(b2) + } + + io.write_full(w, buf[:]) or_return + } + + rest := length - j + if rest > 0 { + #no_bounds_check { + c0 = DEC_TBL[data[i]] + c1 = DEC_TBL[data[i + 1]] + c2 = DEC_TBL[data[i + 2]] + + b0 = (c0 << 2) | (c1 >> 4) + b1 = (c1 << 4) | (c2 >> 2) + } + + switch rest { + case 1: io.write_byte(w, byte(b0)) or_return + case 2: io.write_full(w, {byte(b0), byte(b1)}) or_return + } + } + + return nil +} + +decoded_len :: proc(data: string) -> int { + length := len(data) + if length == 0 { + return 0 + } + + padding: int + if data[length - 1] == PADDING { + if length > 1 && data[length - 2] == PADDING { + padding = 2 + } else { + padding = 1 + } + } + + return ((length * 6) >> 3) - padding } diff --git a/core/encoding/cbor/cbor.odin b/core/encoding/cbor/cbor.odin new file mode 100644 index 000000000..d0e406ab1 --- /dev/null +++ b/core/encoding/cbor/cbor.odin @@ -0,0 +1,673 @@ +package encoding_cbor + +import "base:intrinsics" + +import "core:encoding/json" +import "core:io" +import "core:mem" +import "core:strconv" +import "core:strings" + +// If we are decoding a stream of either a map or list, the initial capacity will be this value. +INITIAL_STREAMED_CONTAINER_CAPACITY :: 8 + +// If we are decoding a stream of either text or bytes, the initial capacity will be this value. +INITIAL_STREAMED_BYTES_CAPACITY :: 16 + +// The default maximum amount of bytes to allocate on a buffer/container at once to prevent +// malicious input from causing massive allocations. +DEFAULT_MAX_PRE_ALLOC :: mem.Kilobyte + +// Known/common headers are defined, undefined headers can still be valid. +// Higher 3 bits is for the major type and lower 5 bits for the additional information. +Header :: enum u8 { + U8 = (u8(Major.Unsigned) << 5) | u8(Add.One_Byte), + U16 = (u8(Major.Unsigned) << 5) | u8(Add.Two_Bytes), + U32 = (u8(Major.Unsigned) << 5) | u8(Add.Four_Bytes), + U64 = (u8(Major.Unsigned) << 5) | u8(Add.Eight_Bytes), + + Neg_U8 = (u8(Major.Negative) << 5) | u8(Add.One_Byte), + Neg_U16 = (u8(Major.Negative) << 5) | u8(Add.Two_Bytes), + Neg_U32 = (u8(Major.Negative) << 5) | u8(Add.Four_Bytes), + Neg_U64 = (u8(Major.Negative) << 5) | u8(Add.Eight_Bytes), + + False = (u8(Major.Other) << 5) | u8(Add.False), + True = (u8(Major.Other) << 5) | u8(Add.True), + + Nil = (u8(Major.Other) << 5) | u8(Add.Nil), + Undefined = (u8(Major.Other) << 5) | u8(Add.Undefined), + + Simple = (u8(Major.Other) << 5) | u8(Add.One_Byte), + + F16 = (u8(Major.Other) << 5) | u8(Add.Two_Bytes), + F32 = (u8(Major.Other) << 5) | u8(Add.Four_Bytes), + F64 = (u8(Major.Other) << 5) | u8(Add.Eight_Bytes), + + Break = (u8(Major.Other) << 5) | u8(Add.Break), +} + +// The higher 3 bits of the header which denotes what type of value it is. +Major :: enum u8 { + Unsigned, + Negative, + Bytes, + Text, + Array, + Map, + Tag, + Other, +} + +// The lower 3 bits of the header which denotes additional information for the type of value. +Add :: enum u8 { + False = 20, + True = 21, + Nil = 22, + Undefined = 23, + + One_Byte = 24, + Two_Bytes = 25, + Four_Bytes = 26, + Eight_Bytes = 27, + + Length_Unknown = 31, + Break = Length_Unknown, +} + +Value :: union { + u8, + u16, + u32, + u64, + + Negative_U8, + Negative_U16, + Negative_U32, + Negative_U64, + + // Pointers so the size of the Value union stays small. + ^Bytes, + ^Text, + ^Array, + ^Map, + ^Tag, + + Simple, + f16, + f32, + f64, + bool, + Undefined, + Nil, +} + +Bytes :: []byte +Text :: string + +Array :: []Value + +Map :: []Map_Entry +Map_Entry :: struct { + key: Value, // Can be any unsigned, negative, float, Simple, bool, Text. + value: Value, +} + +Tag :: struct { + number: Tag_Number, + value: Value, // Value based on the number. +} + +Tag_Number :: u64 + +Nil :: distinct rawptr +Undefined :: distinct rawptr + +// A distinct atom-like number, range from `0..=19` and `32..=max(u8)`. +Simple :: distinct u8 +Atom :: Simple + +Unmarshal_Error :: union #shared_nil { + io.Error, + mem.Allocator_Error, + Decode_Data_Error, + Unmarshal_Data_Error, + Maybe(Unsupported_Type_Error), +} + +Marshal_Error :: union #shared_nil { + io.Error, + mem.Allocator_Error, + Encode_Data_Error, + Marshal_Data_Error, + Maybe(Unsupported_Type_Error), +} + +Decode_Error :: union #shared_nil { + io.Error, + mem.Allocator_Error, + Decode_Data_Error, +} + +Encode_Error :: union #shared_nil { + io.Error, + mem.Allocator_Error, + Encode_Data_Error, +} + +Decode_Data_Error :: enum { + None, + Bad_Major, // An invalid major type was encountered. + Bad_Argument, // A general unexpected value (most likely invalid additional info in header). + Bad_Tag_Value, // When the type of value for the given tag is not valid. + Nested_Indefinite_Length, // When an streamed/indefinite length container nests another, this is not allowed. + Nested_Tag, // When a tag's value is another tag, this is not allowed. + Length_Too_Big, // When the length of a container (map, array, bytes, string) is more than `max(int)`. + Disallowed_Streaming, // When the `.Disallow_Streaming` flag is set and a streaming header is encountered. + Break, // When the `break` header was found without any stream to break off. +} + +Encode_Data_Error :: enum { + None, + Invalid_Simple, // When a simple is being encoded that is out of the range `0..=19` and `32..=max(u8)`. + Int_Too_Big, // When an int is being encoded that is larger than `max(u64)` or smaller than `min(u64)`. + Bad_Tag_Value, // When the type of value is not supported by the tag implementation. +} + +Unmarshal_Data_Error :: enum { + None, + Invalid_Parameter, // When the given `any` can not be unmarshalled into. + Non_Pointer_Parameter, // When the given `any` is not a pointer. +} + +Marshal_Data_Error :: enum { + None, + Invalid_CBOR_Tag, // When the struct tag `cbor_tag:""` is not a registered name or number. +} + +// Error that is returned when a type couldn't be marshalled into or out of, as much information +// as possible/available is added. +Unsupported_Type_Error :: struct { + id: typeid, + hdr: Header, + add: Add, +} + +_unsupported :: proc(v: any, hdr: Header, add: Add = nil) -> Maybe(Unsupported_Type_Error) { + return Unsupported_Type_Error{ + id = v.id, + hdr = hdr, + add = add, + } +} + +// Actual value is `-1 - x` (be careful of overflows). + +Negative_U8 :: distinct u8 +Negative_U16 :: distinct u16 +Negative_U32 :: distinct u32 +Negative_U64 :: distinct u64 + +// Turns the CBOR negative unsigned int type into a signed integer type. +negative_to_int :: proc { + negative_u8_to_int, + negative_u16_to_int, + negative_u32_to_int, + negative_u64_to_int, +} + +negative_u8_to_int :: #force_inline proc(u: Negative_U8) -> i16 { + return -1 - i16(u) +} + +negative_u16_to_int :: #force_inline proc(u: Negative_U16) -> i32 { + return -1 - i32(u) +} + +negative_u32_to_int :: #force_inline proc(u: Negative_U32) -> i64 { + return -1 - i64(u) +} + +negative_u64_to_int :: #force_inline proc(u: Negative_U64) -> i128 { + return -1 - i128(u) +} + +// Utility for converting between the different errors when they are subsets of the other. +err_conv :: proc { + encode_to_marshal_err, + encode_to_marshal_err_p2, + decode_to_unmarshal_err, + decode_to_unmarshal_err_p, + decode_to_unmarshal_err_p2, +} + +encode_to_marshal_err :: #force_inline proc(err: Encode_Error) -> Marshal_Error { + switch e in err { + case nil: return nil + case io.Error: return e + case mem.Allocator_Error: return e + case Encode_Data_Error: return e + case: return nil + } +} + +encode_to_marshal_err_p2 :: #force_inline proc(v: $T, v2: $T2, err: Encode_Error) -> (T, T2, Marshal_Error) { + return v, v2, err_conv(err) +} + +decode_to_unmarshal_err :: #force_inline proc(err: Decode_Error) -> Unmarshal_Error { + switch e in err { + case nil: return nil + case io.Error: return e + case mem.Allocator_Error: return e + case Decode_Data_Error: return e + case: return nil + } +} + +decode_to_unmarshal_err_p :: #force_inline proc(v: $T, err: Decode_Error) -> (T, Unmarshal_Error) { + return v, err_conv(err) +} + +decode_to_unmarshal_err_p2 :: #force_inline proc(v: $T, v2: $T2, err: Decode_Error) -> (T, T2, Unmarshal_Error) { + return v, v2, err_conv(err) +} + +// Recursively frees all memory allocated when decoding the passed value. +destroy :: proc(val: Value, allocator := context.allocator) { + context.allocator = allocator + #partial switch v in val { + case ^Map: + if v == nil { return } + for entry in v { + destroy(entry.key) + destroy(entry.value) + } + delete(v^) + free(v) + case ^Array: + if v == nil { return } + for entry in v { + destroy(entry) + } + delete(v^) + free(v) + case ^Text: + if v == nil { return } + delete(v^) + free(v) + case ^Bytes: + if v == nil { return } + delete(v^) + free(v) + case ^Tag: + if v == nil { return } + destroy(v.value) + free(v) + } +} + +/* +to_diagnostic_format either writes or returns a human-readable representation of the value, +optionally formatted, defined as the diagnostic format in [[RFC 8949 Section 8;https://www.rfc-editor.org/rfc/rfc8949.html#name-diagnostic-notation]]. + +Incidentally, if the CBOR does not contain any of the additional types defined on top of JSON +this will also be valid JSON. +*/ +to_diagnostic_format :: proc { + to_diagnostic_format_string, + to_diagnostic_format_writer, +} + +// Turns the given CBOR value into a human-readable string. +// See docs on the proc group `diagnose` for more info. +to_diagnostic_format_string :: proc(val: Value, padding := 0, allocator := context.allocator) -> (string, mem.Allocator_Error) #optional_allocator_error { + b := strings.builder_make(allocator) + w := strings.to_stream(&b) + err := to_diagnostic_format_writer(w, val, padding) + if err == .EOF { + // The string builder stream only returns .EOF, and only if it can't write (out of memory). + return "", .Out_Of_Memory + } + assert(err == nil) + + return strings.to_string(b), nil +} + +// Writes the given CBOR value into the writer as human-readable text. +// See docs on the proc group `diagnose` for more info. +to_diagnostic_format_writer :: proc(w: io.Writer, val: Value, padding := 0) -> io.Error { + @(require_results) + indent :: proc(padding: int) -> int { + padding := padding + if padding != -1 { + padding += 1 + } + return padding + } + + @(require_results) + dedent :: proc(padding: int) -> int { + padding := padding + if padding != -1 { + padding -= 1 + } + return padding + } + + comma :: proc(w: io.Writer, padding: int) -> io.Error { + _ = io.write_string(w, ", " if padding == -1 else ",") or_return + return nil + } + + newline :: proc(w: io.Writer, padding: int) -> io.Error { + if padding != -1 { + io.write_string(w, "\n") or_return + for _ in 0.. (Value, mem.Allocator_Error) #optional_allocator_error { + internal :: proc(val: json.Value) -> (ret: Value, err: mem.Allocator_Error) { + switch v in val { + case json.Null: return Nil{}, nil + case json.Integer: + i, major := _int_to_uint(v) + #partial switch major { + case .Unsigned: return i, nil + case .Negative: return Negative_U64(i), nil + case: unreachable() + } + case json.Float: return v, nil + case json.Boolean: return v, nil + case json.String: + container := new(Text) or_return + + // We need the string to have a nil byte at the end so we clone to cstring. + container^ = string(strings.clone_to_cstring(v) or_return) + return container, nil + case json.Array: + arr := new(Array) or_return + arr^ = make([]Value, len(v)) or_return + for _, i in arr { + arr[i] = internal(v[i]) or_return + } + return arr, nil + case json.Object: + m := new(Map) or_return + dm := make([dynamic]Map_Entry, 0, len(v)) or_return + for mkey, mval in v { + append(&dm, Map_Entry{from_json(mkey) or_return, from_json(mval) or_return}) + } + m^ = dm[:] + return m, nil + } + return nil, nil + } + + context.allocator = allocator + return internal(val) +} + +/* +Converts from CBOR to JSON. + +NOTE: overflow on integers or floats is not handled. + +Everything is copied to the given allocator, the passed in CBOR value can be `destroy`'ed after. + +If a CBOR map with non-string keys is encountered it is turned into an array of tuples. +*/ +to_json :: proc(val: Value, allocator := context.allocator) -> (json.Value, mem.Allocator_Error) #optional_allocator_error { + internal :: proc(val: Value) -> (ret: json.Value, err: mem.Allocator_Error) { + switch v in val { + case Simple: return json.Integer(v), nil + + case u8: return json.Integer(v), nil + case u16: return json.Integer(v), nil + case u32: return json.Integer(v), nil + case u64: return json.Integer(v), nil + + case Negative_U8: return json.Integer(negative_to_int(v)), nil + case Negative_U16: return json.Integer(negative_to_int(v)), nil + case Negative_U32: return json.Integer(negative_to_int(v)), nil + case Negative_U64: return json.Integer(negative_to_int(v)), nil + + case f16: return json.Float(v), nil + case f32: return json.Float(v), nil + case f64: return json.Float(v), nil + + case bool: return json.Boolean(v), nil + + case Undefined: return json.Null{}, nil + case Nil: return json.Null{}, nil + + case ^Bytes: return json.String(strings.clone(string(v^)) or_return), nil + case ^Text: return json.String(strings.clone(v^) or_return), nil + + case ^Map: + keys_all_strings :: proc(m: ^Map) -> bool { + for entry in m { + #partial switch kv in entry.key { + case ^Bytes: + case ^Text: + case: return false + } + } + return false + } + + if keys_all_strings(v) { + obj := make(json.Object, len(v)) or_return + for entry in v { + k: string + #partial switch kv in entry.key { + case ^Bytes: k = string(kv^) + case ^Text: k = kv^ + case: unreachable() + } + + v := internal(entry.value) or_return + obj[k] = v + } + return obj, nil + } else { + // Resort to an array of tuples if keys aren't all strings. + arr := make(json.Array, 0, len(v)) or_return + for entry in v { + entry_arr := make(json.Array, 0, 2) or_return + append(&entry_arr, internal(entry.key) or_return) or_return + append(&entry_arr, internal(entry.value) or_return) or_return + append(&arr, entry_arr) or_return + } + return arr, nil + } + + case ^Array: + arr := make(json.Array, 0, len(v)) or_return + for entry in v { + append(&arr, internal(entry) or_return) or_return + } + return arr, nil + + case ^Tag: + obj := make(json.Object, 2) or_return + obj[strings.clone("number") or_return] = internal(v.number) or_return + obj[strings.clone("value") or_return] = internal(v.value) or_return + return obj, nil + + case: return json.Null{}, nil + } + } + + context.allocator = allocator + return internal(val) +} + +_int_to_uint :: proc { + _i8_to_uint, + _i16_to_uint, + _i32_to_uint, + _i64_to_uint, + _i128_to_uint, +} + +_u128_to_u64 :: #force_inline proc(v: u128) -> (u64, Encode_Data_Error) { + if v > u128(max(u64)) { + return 0, .Int_Too_Big + } + + return u64(v), nil +} + +_i8_to_uint :: #force_inline proc(v: i8) -> (u: u8, m: Major) { + if v < 0 { + return u8(abs(v)-1), .Negative + } + + return u8(v), .Unsigned +} + +_i16_to_uint :: #force_inline proc(v: i16) -> (u: u16, m: Major) { + if v < 0 { + return u16(abs(v)-1), .Negative + } + + return u16(v), .Unsigned +} + +_i32_to_uint :: #force_inline proc(v: i32) -> (u: u32, m: Major) { + if v < 0 { + return u32(abs(v)-1), .Negative + } + + return u32(v), .Unsigned +} + +_i64_to_uint :: #force_inline proc(v: i64) -> (u: u64, m: Major) { + if v < 0 { + return u64(abs(v)-1), .Negative + } + + return u64(v), .Unsigned +} + +_i128_to_uint :: proc(v: i128) -> (u: u64, m: Major, err: Encode_Data_Error) { + if v < 0 { + m = .Negative + u, err = _u128_to_u64(u128(abs(v) - 1)) + return + } + + m = .Unsigned + u, err = _u128_to_u64(u128(v)) + return +} diff --git a/core/encoding/cbor/coding.odin b/core/encoding/cbor/coding.odin new file mode 100644 index 000000000..0d276a7a1 --- /dev/null +++ b/core/encoding/cbor/coding.odin @@ -0,0 +1,886 @@ +package encoding_cbor + +import "base:intrinsics" +import "base:runtime" + +import "core:bytes" +import "core:encoding/endian" +import "core:io" +import "core:slice" +import "core:strings" + +Encoder_Flag :: enum { + // CBOR defines a tag header that also acts as a file/binary header, + // this way decoders can check the first header of the binary and see if it is CBOR. + Self_Described_CBOR, + + // Integers are stored in the smallest integer type it fits. + // This involves checking each int against the max of all its smaller types. + Deterministic_Int_Size, + + // Floats are stored in the smallest size float type without losing precision. + // This involves casting each float down to its smaller types and checking if it changed. + Deterministic_Float_Size, + + // Sort maps by their keys in bytewise lexicographic order of their deterministic encoding. + // NOTE: In order to do this, all keys of a map have to be pre-computed, sorted, and + // then written, this involves temporary allocations for the keys and a copy of the map itself. + Deterministic_Map_Sorting, +} + +Encoder_Flags :: bit_set[Encoder_Flag] + +// Flags for fully deterministic output (if you are not using streaming/indeterminate length). +ENCODE_FULLY_DETERMINISTIC :: Encoder_Flags{.Deterministic_Int_Size, .Deterministic_Float_Size, .Deterministic_Map_Sorting} + +// Flags for the smallest encoding output. +ENCODE_SMALL :: Encoder_Flags{.Deterministic_Int_Size, .Deterministic_Float_Size} + +Encoder :: struct { + flags: Encoder_Flags, + writer: io.Writer, + temp_allocator: runtime.Allocator, +} + +Decoder_Flag :: enum { + // Rejects (with an error `.Disallowed_Streaming`) when a streaming CBOR header is encountered. + Disallow_Streaming, + + // Pre-allocates buffers and containers with the size that was set in the CBOR header. + // This should only be enabled when you control both ends of the encoding, if you don't, + // attackers can craft input that causes massive (`max(u64)`) byte allocations for a few bytes of + // CBOR. + Trusted_Input, + + // Makes the decoder shrink of excess capacity from allocated buffers/containers before returning. + Shrink_Excess, +} + +Decoder_Flags :: bit_set[Decoder_Flag] + +Decoder :: struct { + // The max amount of bytes allowed to pre-allocate when `.Trusted_Input` is not set on the + // flags. + max_pre_alloc: int, + + flags: Decoder_Flags, + reader: io.Reader, +} + +/* +Decodes both deterministic and non-deterministic CBOR into a `Value` variant. + +`Text` and `Bytes` can safely be cast to cstrings because of an added 0 byte. + +Allocations are done using the given allocator, +*no* allocations are done on the `context.temp_allocator`. + +A value can be (fully and recursively) deallocated using the `destroy` proc in this package. + +Disable streaming/indeterminate lengths with the `.Disallow_Streaming` flag. + +Shrink excess bytes in buffers and containers with the `.Shrink_Excess` flag. + +Mark the input as trusted input with the `.Trusted_Input` flag, this turns off the safety feature +of not pre-allocating more than `max_pre_alloc` bytes before reading into the bytes. You should only +do this when you own both sides of the encoding and are sure there can't be malicious bytes used as +an input. +*/ +decode_from :: proc { + decode_from_string, + decode_from_reader, + decode_from_decoder, +} +decode :: decode_from + +// Decodes the given string as CBOR. +// See docs on the proc group `decode` for more information. +decode_from_string :: proc(s: string, flags: Decoder_Flags = {}, allocator := context.allocator) -> (v: Value, err: Decode_Error) { + r: strings.Reader + strings.reader_init(&r, s) + return decode_from_reader(strings.reader_to_stream(&r), flags, allocator) +} + +// Reads a CBOR value from the given reader. +// See docs on the proc group `decode` for more information. +decode_from_reader :: proc(r: io.Reader, flags: Decoder_Flags = {}, allocator := context.allocator) -> (v: Value, err: Decode_Error) { + return decode_from_decoder( + Decoder{ DEFAULT_MAX_PRE_ALLOC, flags, r }, + allocator=allocator, + ) +} + +// Reads a CBOR value from the given decoder. +// See docs on the proc group `decode` for more information. +decode_from_decoder :: proc(d: Decoder, allocator := context.allocator) -> (v: Value, err: Decode_Error) { + context.allocator = allocator + + d := d + + if d.max_pre_alloc <= 0 { + d.max_pre_alloc = DEFAULT_MAX_PRE_ALLOC + } + + v, err = _decode_from_decoder(d) + // Normal EOF does not exist here, we try to read the exact amount that is said to be provided. + if err == .EOF { err = .Unexpected_EOF } + return +} + +_decode_from_decoder :: proc(d: Decoder, hdr: Header = Header(0)) -> (v: Value, err: Decode_Error) { + hdr := hdr + r := d.reader + if hdr == Header(0) { hdr = _decode_header(r) or_return } + switch hdr { + case .U8: return _decode_u8 (r) + case .U16: return _decode_u16(r) + case .U32: return _decode_u32(r) + case .U64: return _decode_u64(r) + + case .Neg_U8: return Negative_U8 (_decode_u8 (r) or_return), nil + case .Neg_U16: return Negative_U16(_decode_u16(r) or_return), nil + case .Neg_U32: return Negative_U32(_decode_u32(r) or_return), nil + case .Neg_U64: return Negative_U64(_decode_u64(r) or_return), nil + + case .Simple: return _decode_simple(r) + + case .F16: return _decode_f16(r) + case .F32: return _decode_f32(r) + case .F64: return _decode_f64(r) + + case .True: return true, nil + case .False: return false, nil + + case .Nil: return Nil{}, nil + case .Undefined: return Undefined{}, nil + + case .Break: return nil, .Break + } + + maj, add := _header_split(hdr) + switch maj { + case .Unsigned: return _decode_tiny_u8(add) + case .Negative: return Negative_U8(_decode_tiny_u8(add) or_return), nil + case .Bytes: return _decode_bytes_ptr(d, add) + case .Text: return _decode_text_ptr(d, add) + case .Array: return _decode_array_ptr(d, add) + case .Map: return _decode_map_ptr(d, add) + case .Tag: return _decode_tag_ptr(d, add) + case .Other: return _decode_tiny_simple(add) + case: return nil, .Bad_Major + } +} + +/* +Encodes the CBOR value into a binary CBOR. + +Flags can be used to control the output (mainly determinism, which coincidently affects size). + +The default flags `ENCODE_SMALL` (`.Deterministic_Int_Size`, `.Deterministic_Float_Size`) will try +to put ints and floats into their smallest possible byte size without losing equality. + +Adding the `.Self_Described_CBOR` flag will wrap the value in a tag that lets generic decoders know +the contents are CBOR from just reading the first byte. + +Adding the `.Deterministic_Map_Sorting` flag will sort the encoded maps by the byte content of the +encoded key. This flag has a cost on performance and memory efficiency because all keys in a map +have to be precomputed, sorted and only then written to the output. + +Empty flags will do nothing extra to the value. + +The allocations for the `.Deterministic_Map_Sorting` flag are done using the given temp_allocator. +but are followed by the necessary `delete` and `free` calls if the allocator supports them. +This is helpful when the CBOR size is so big that you don't want to collect all the temporary +allocations until the end. +*/ +encode_into :: proc { + encode_into_bytes, + encode_into_builder, + encode_into_writer, + encode_into_encoder, +} +encode :: encode_into + +// Encodes the CBOR value into binary CBOR allocated on the given allocator. +// See the docs on the proc group `encode_into` for more info. +encode_into_bytes :: proc(v: Value, flags := ENCODE_SMALL, allocator := context.allocator, temp_allocator := context.temp_allocator) -> (data: []byte, err: Encode_Error) { + b := strings.builder_make(allocator) or_return + encode_into_builder(&b, v, flags, temp_allocator) or_return + return b.buf[:], nil +} + +// Encodes the CBOR value into binary CBOR written to the given builder. +// See the docs on the proc group `encode_into` for more info. +encode_into_builder :: proc(b: ^strings.Builder, v: Value, flags := ENCODE_SMALL, temp_allocator := context.temp_allocator) -> Encode_Error { + return encode_into_writer(strings.to_stream(b), v, flags, temp_allocator) +} + +// Encodes the CBOR value into binary CBOR written to the given writer. +// See the docs on the proc group `encode_into` for more info. +encode_into_writer :: proc(w: io.Writer, v: Value, flags := ENCODE_SMALL, temp_allocator := context.temp_allocator) -> Encode_Error { + return encode_into_encoder(Encoder{flags, w, temp_allocator}, v) +} + +// Encodes the CBOR value into binary CBOR written to the given encoder. +// See the docs on the proc group `encode_into` for more info. +encode_into_encoder :: proc(e: Encoder, v: Value) -> Encode_Error { + e := e + + if e.temp_allocator.procedure == nil { + e.temp_allocator = context.temp_allocator + } + + if .Self_Described_CBOR in e.flags { + _encode_u64(e, TAG_SELF_DESCRIBED_CBOR, .Tag) or_return + e.flags &~= { .Self_Described_CBOR } + } + + switch v_spec in v { + case u8: return _encode_u8(e.writer, v_spec, .Unsigned) + case u16: return _encode_u16(e, v_spec, .Unsigned) + case u32: return _encode_u32(e, v_spec, .Unsigned) + case u64: return _encode_u64(e, v_spec, .Unsigned) + case Negative_U8: return _encode_u8(e.writer, u8(v_spec), .Negative) + case Negative_U16: return _encode_u16(e, u16(v_spec), .Negative) + case Negative_U32: return _encode_u32(e, u32(v_spec), .Negative) + case Negative_U64: return _encode_u64(e, u64(v_spec), .Negative) + case ^Bytes: return _encode_bytes(e, v_spec^) + case ^Text: return _encode_text(e, v_spec^) + case ^Array: return _encode_array(e, v_spec^) + case ^Map: return _encode_map(e, v_spec^) + case ^Tag: return _encode_tag(e, v_spec^) + case Simple: return _encode_simple(e.writer, v_spec) + case f16: return _encode_f16(e.writer, v_spec) + case f32: return _encode_f32(e, v_spec) + case f64: return _encode_f64(e, v_spec) + case bool: return _encode_bool(e.writer, v_spec) + case Nil: return _encode_nil(e.writer) + case Undefined: return _encode_undefined(e.writer) + case: return nil + } +} + +_decode_header :: proc(r: io.Reader) -> (hdr: Header, err: io.Error) { + hdr = Header(_decode_u8(r) or_return) + return +} + +_header_split :: proc(hdr: Header) -> (Major, Add) { + return Major(u8(hdr) >> 5), Add(u8(hdr) & 0x1f) +} + +_decode_u8 :: proc(r: io.Reader) -> (v: u8, err: io.Error) { + byte: [1]byte = --- + io.read_full(r, byte[:]) or_return + return byte[0], nil +} + +_encode_uint :: proc { + _encode_u8, + _encode_u16, + _encode_u32, + _encode_u64, +} + +_encode_u8 :: proc(w: io.Writer, v: u8, major: Major = .Unsigned) -> (err: io.Error) { + header := u8(major) << 5 + if v < u8(Add.One_Byte) { + header |= v + _, err = io.write_full(w, {header}) + return + } + + header |= u8(Add.One_Byte) + _, err = io.write_full(w, {header, v}) + return +} + +_decode_tiny_u8 :: proc(additional: Add) -> (u8, Decode_Data_Error) { + if additional < .One_Byte { + return u8(additional), nil + } + + return 0, .Bad_Argument +} + +_decode_u16 :: proc(r: io.Reader) -> (v: u16, err: io.Error) { + bytes: [2]byte = --- + io.read_full(r, bytes[:]) or_return + return endian.unchecked_get_u16be(bytes[:]), nil +} + +_encode_u16 :: proc(e: Encoder, v: u16, major: Major = .Unsigned) -> Encode_Error { + if .Deterministic_Int_Size in e.flags { + return _encode_deterministic_uint(e.writer, v, major) + } + return _encode_u16_exact(e.writer, v, major) +} + +_encode_u16_exact :: proc(w: io.Writer, v: u16, major: Major = .Unsigned) -> (err: io.Error) { + bytes: [3]byte = --- + bytes[0] = (u8(major) << 5) | u8(Add.Two_Bytes) + endian.unchecked_put_u16be(bytes[1:], v) + _, err = io.write_full(w, bytes[:]) + return +} + +_decode_u32 :: proc(r: io.Reader) -> (v: u32, err: io.Error) { + bytes: [4]byte = --- + io.read_full(r, bytes[:]) or_return + return endian.unchecked_get_u32be(bytes[:]), nil +} + +_encode_u32 :: proc(e: Encoder, v: u32, major: Major = .Unsigned) -> Encode_Error { + if .Deterministic_Int_Size in e.flags { + return _encode_deterministic_uint(e.writer, v, major) + } + return _encode_u32_exact(e.writer, v, major) +} + +_encode_u32_exact :: proc(w: io.Writer, v: u32, major: Major = .Unsigned) -> (err: io.Error) { + bytes: [5]byte = --- + bytes[0] = (u8(major) << 5) | u8(Add.Four_Bytes) + endian.unchecked_put_u32be(bytes[1:], v) + _, err = io.write_full(w, bytes[:]) + return +} + +_decode_u64 :: proc(r: io.Reader) -> (v: u64, err: io.Error) { + bytes: [8]byte = --- + io.read_full(r, bytes[:]) or_return + return endian.unchecked_get_u64be(bytes[:]), nil +} + +_encode_u64 :: proc(e: Encoder, v: u64, major: Major = .Unsigned) -> Encode_Error { + if .Deterministic_Int_Size in e.flags { + return _encode_deterministic_uint(e.writer, v, major) + } + return _encode_u64_exact(e.writer, v, major) +} + +_encode_u64_exact :: proc(w: io.Writer, v: u64, major: Major = .Unsigned) -> (err: io.Error) { + bytes: [9]byte = --- + bytes[0] = (u8(major) << 5) | u8(Add.Eight_Bytes) + endian.unchecked_put_u64be(bytes[1:], v) + _, err = io.write_full(w, bytes[:]) + return +} + +_decode_bytes_ptr :: proc(d: Decoder, add: Add, type: Major = .Bytes) -> (v: ^Bytes, err: Decode_Error) { + v = new(Bytes) or_return + defer if err != nil { free(v) } + + v^ = _decode_bytes(d, add, type) or_return + return +} + +_decode_bytes :: proc(d: Decoder, add: Add, type: Major = .Bytes, allocator := context.allocator) -> (v: Bytes, err: Decode_Error) { + context.allocator = allocator + + add := add + n, scap := _decode_len_str(d, add) or_return + + buf := strings.builder_make(0, scap) or_return + defer if err != nil { strings.builder_destroy(&buf) } + buf_stream := strings.to_stream(&buf) + + if n == -1 { + indefinite_loop: for { + header := _decode_header(d.reader) or_return + maj: Major + maj, add = _header_split(header) + #partial switch maj { + case type: + iter_n, iter_cap := _decode_len_str(d, add) or_return + if iter_n == -1 { + return nil, .Nested_Indefinite_Length + } + reserve(&buf.buf, len(buf.buf) + iter_cap) or_return + io.copy_n(buf_stream, d.reader, i64(iter_n)) or_return + + case .Other: + if add != .Break { return nil, .Bad_Argument } + break indefinite_loop + + case: + return nil, .Bad_Major + } + } + } else { + io.copy_n(buf_stream, d.reader, i64(n)) or_return + } + + v = buf.buf[:] + + // Write zero byte so this can be converted to cstring. + strings.write_byte(&buf, 0) + + if .Shrink_Excess in d.flags { shrink(&buf.buf) } + return +} + +_encode_bytes :: proc(e: Encoder, val: Bytes, major: Major = .Bytes) -> (err: Encode_Error) { + assert(len(val) >= 0) + _encode_u64(e, u64(len(val)), major) or_return + _, err = io.write_full(e.writer, val[:]) + return +} + +_decode_text_ptr :: proc(d: Decoder, add: Add) -> (v: ^Text, err: Decode_Error) { + v = new(Text) or_return + defer if err != nil { free(v) } + + v^ = _decode_text(d, add) or_return + return +} + +_decode_text :: proc(d: Decoder, add: Add, allocator := context.allocator) -> (v: Text, err: Decode_Error) { + return (Text)(_decode_bytes(d, add, .Text, allocator) or_return), nil +} + +_encode_text :: proc(e: Encoder, val: Text) -> Encode_Error { + return _encode_bytes(e, transmute([]byte)val, .Text) +} + +_decode_array_ptr :: proc(d: Decoder, add: Add) -> (v: ^Array, err: Decode_Error) { + v = new(Array) or_return + defer if err != nil { free(v) } + + v^ = _decode_array(d, add) or_return + return +} + +_decode_array :: proc(d: Decoder, add: Add) -> (v: Array, err: Decode_Error) { + n, scap := _decode_len_container(d, add) or_return + array := make([dynamic]Value, 0, scap) or_return + defer if err != nil { + for entry in array { destroy(entry) } + delete(array) + } + + for i := 0; n == -1 || i < n; i += 1 { + val, verr := _decode_from_decoder(d) + if n == -1 && verr == .Break { + break + } else if verr != nil { + err = verr + return + } + + append(&array, val) or_return + } + + if .Shrink_Excess in d.flags { shrink(&array) } + + v = array[:] + return +} + +_encode_array :: proc(e: Encoder, arr: Array) -> Encode_Error { + assert(len(arr) >= 0) + _encode_u64(e, u64(len(arr)), .Array) + for val in arr { + encode(e, val) or_return + } + return nil +} + +_decode_map_ptr :: proc(d: Decoder, add: Add) -> (v: ^Map, err: Decode_Error) { + v = new(Map) or_return + defer if err != nil { free(v) } + + v^ = _decode_map(d, add) or_return + return +} + +_decode_map :: proc(d: Decoder, add: Add) -> (v: Map, err: Decode_Error) { + n, scap := _decode_len_container(d, add) or_return + items := make([dynamic]Map_Entry, 0, scap) or_return + defer if err != nil { + for entry in items { + destroy(entry.key) + destroy(entry.value) + } + delete(items) + } + + for i := 0; n == -1 || i < n; i += 1 { + key, kerr := _decode_from_decoder(d) + if n == -1 && kerr == .Break { + break + } else if kerr != nil { + return nil, kerr + } + + value := _decode_from_decoder(d) or_return + + append(&items, Map_Entry{ + key = key, + value = value, + }) or_return + } + + if .Shrink_Excess in d.flags { shrink(&items) } + + v = items[:] + return +} + +_encode_map :: proc(e: Encoder, m: Map) -> (err: Encode_Error) { + assert(len(m) >= 0) + _encode_u64(e, u64(len(m)), .Map) or_return + + if .Deterministic_Map_Sorting not_in e.flags { + for entry in m { + encode(e, entry.key) or_return + encode(e, entry.value) or_return + } + return + } + + // Deterministic_Map_Sorting needs us to sort the entries by the byte contents of the + // encoded key. + // + // This means we have to store and sort them before writing incurring extra (temporary) allocations. + + Map_Entry_With_Key :: struct { + encoded_key: []byte, + entry: Map_Entry, + } + + entries := make([]Map_Entry_With_Key, len(m), e.temp_allocator) or_return + defer delete(entries, e.temp_allocator) + + for &entry, i in entries { + entry.entry = m[i] + + buf := strings.builder_make(e.temp_allocator) or_return + + ke := e + ke.writer = strings.to_stream(&buf) + + encode(ke, entry.entry.key) or_return + entry.encoded_key = buf.buf[:] + } + + // Sort lexicographic on the bytes of the key. + slice.sort_by_cmp(entries, proc(a, b: Map_Entry_With_Key) -> slice.Ordering { + return slice.Ordering(bytes.compare(a.encoded_key, b.encoded_key)) + }) + + for entry in entries { + io.write_full(e.writer, entry.encoded_key) or_return + delete(entry.encoded_key, e.temp_allocator) + + encode(e, entry.entry.value) or_return + } + + return nil +} + +_decode_tag_ptr :: proc(d: Decoder, add: Add) -> (v: Value, err: Decode_Error) { + tag := _decode_tag(d, add) or_return + if t, ok := tag.?; ok { + defer if err != nil { destroy(t.value) } + tp := new(Tag) or_return + tp^ = t + return tp, nil + } + + // no error, no tag, this was the self described CBOR tag, skip it. + return _decode_from_decoder(d) +} + +_decode_tag :: proc(d: Decoder, add: Add) -> (v: Maybe(Tag), err: Decode_Error) { + num := _decode_uint_as_u64(d.reader, add) or_return + + // CBOR can be wrapped in a tag that decoders can use to see/check if the binary data is CBOR. + // We can ignore it here. + if num == TAG_SELF_DESCRIBED_CBOR { + return + } + + t := Tag{ + number = num, + value = _decode_from_decoder(d) or_return, + } + + if nested, ok := t.value.(^Tag); ok { + destroy(nested) + return nil, .Nested_Tag + } + + return t, nil +} + +_decode_uint_as_u64 :: proc(r: io.Reader, add: Add) -> (nr: u64, err: Decode_Error) { + #partial switch add { + case .One_Byte: return u64(_decode_u8(r) or_return), nil + case .Two_Bytes: return u64(_decode_u16(r) or_return), nil + case .Four_Bytes: return u64(_decode_u32(r) or_return), nil + case .Eight_Bytes: return u64(_decode_u64(r) or_return), nil + case: return u64(_decode_tiny_u8(add) or_return), nil + } +} + +_encode_tag :: proc(e: Encoder, val: Tag) -> Encode_Error { + _encode_u64(e, val.number, .Tag) or_return + return encode(e, val.value) +} + +_decode_simple :: proc(r: io.Reader) -> (v: Simple, err: io.Error) { + buf: [1]byte = --- + io.read_full(r, buf[:]) or_return + return Simple(buf[0]), nil +} + +_encode_simple :: proc(w: io.Writer, v: Simple) -> (err: Encode_Error) { + header := u8(Major.Other) << 5 + + if v < Simple(Add.False) { + header |= u8(v) + _, err = io.write_full(w, {header}) + return + } else if v <= Simple(Add.Break) { + return .Invalid_Simple + } + + header |= u8(Add.One_Byte) + _, err = io.write_full(w, {header, u8(v)}) + return +} + +_decode_tiny_simple :: proc(add: Add) -> (Simple, Decode_Data_Error) { + if add < Add.False { + return Simple(add), nil + } + + return 0, .Bad_Argument +} + +_decode_f16 :: proc(r: io.Reader) -> (v: f16, err: io.Error) { + bytes: [2]byte = --- + io.read_full(r, bytes[:]) or_return + n := endian.unchecked_get_u16be(bytes[:]) + return transmute(f16)n, nil +} + +_encode_f16 :: proc(w: io.Writer, v: f16) -> (err: io.Error) { + bytes: [3]byte = --- + bytes[0] = u8(Header.F16) + endian.unchecked_put_u16be(bytes[1:], transmute(u16)v) + _, err = io.write_full(w, bytes[:]) + return +} + +_decode_f32 :: proc(r: io.Reader) -> (v: f32, err: io.Error) { + bytes: [4]byte = --- + io.read_full(r, bytes[:]) or_return + n := endian.unchecked_get_u32be(bytes[:]) + return transmute(f32)n, nil +} + +_encode_f32 :: proc(e: Encoder, v: f32) -> io.Error { + if .Deterministic_Float_Size in e.flags { + return _encode_deterministic_float(e.writer, v) + } + return _encode_f32_exact(e.writer, v) +} + +_encode_f32_exact :: proc(w: io.Writer, v: f32) -> (err: io.Error) { + bytes: [5]byte = --- + bytes[0] = u8(Header.F32) + endian.unchecked_put_u32be(bytes[1:], transmute(u32)v) + _, err = io.write_full(w, bytes[:]) + return +} + +_decode_f64 :: proc(r: io.Reader) -> (v: f64, err: io.Error) { + bytes: [8]byte = --- + io.read_full(r, bytes[:]) or_return + n := endian.unchecked_get_u64be(bytes[:]) + return transmute(f64)n, nil +} + +_encode_f64 :: proc(e: Encoder, v: f64) -> io.Error { + if .Deterministic_Float_Size in e.flags { + return _encode_deterministic_float(e.writer, v) + } + return _encode_f64_exact(e.writer, v) +} + +_encode_f64_exact :: proc(w: io.Writer, v: f64) -> (err: io.Error) { + bytes: [9]byte = --- + bytes[0] = u8(Header.F64) + endian.unchecked_put_u64be(bytes[1:], transmute(u64)v) + _, err = io.write_full(w, bytes[:]) + return +} + +_encode_bool :: proc(w: io.Writer, v: bool) -> (err: io.Error) { + switch v { + case true: _, err = io.write_full(w, {u8(Header.True )}); return + case false: _, err = io.write_full(w, {u8(Header.False)}); return + case: unreachable() + } +} + +_encode_undefined :: proc(w: io.Writer) -> io.Error { + _, err := io.write_full(w, {u8(Header.Undefined)}) + return err +} + +_encode_nil :: proc(w: io.Writer) -> io.Error { + _, err := io.write_full(w, {u8(Header.Nil)}) + return err +} + +// Streaming + +encode_stream_begin :: proc(w: io.Writer, major: Major) -> (err: io.Error) { + assert(major >= Major(.Bytes) && major <= Major(.Map), "illegal stream type") + + header := (u8(major) << 5) | u8(Add.Length_Unknown) + _, err = io.write_full(w, {header}) + return +} + +encode_stream_end :: proc(w: io.Writer) -> io.Error { + header := (u8(Major.Other) << 5) | u8(Add.Break) + _, err := io.write_full(w, {header}) + return err +} + +encode_stream_bytes :: _encode_bytes +encode_stream_text :: _encode_text +encode_stream_array_item :: encode + +encode_stream_map_entry :: proc(e: Encoder, key: Value, val: Value) -> Encode_Error { + encode(e, key) or_return + return encode(e, val) +} + +// For `Bytes` and `Text` strings: Decodes the number of items the header says follows. +// If the number is not specified -1 is returned and streaming should be initiated. +// A suitable starting capacity is also returned for a buffer that is allocated up the stack. +_decode_len_str :: proc(d: Decoder, add: Add) -> (n: int, scap: int, err: Decode_Error) { + if add == .Length_Unknown { + if .Disallow_Streaming in d.flags { + return -1, -1, .Disallowed_Streaming + } + return -1, INITIAL_STREAMED_BYTES_CAPACITY, nil + } + + _n := _decode_uint_as_u64(d.reader, add) or_return + if _n > u64(max(int)) { return -1, -1, .Length_Too_Big } + n = int(_n) + + scap = n + 1 // Space for zero byte. + if .Trusted_Input not_in d.flags { + scap = min(d.max_pre_alloc, scap) + } + + return +} + +// For `Array` and `Map` types: Decodes the number of items the header says follows. +// If the number is not specified -1 is returned and streaming should be initiated. +// A suitable starting capacity is also returned for a buffer that is allocated up the stack. +_decode_len_container :: proc(d: Decoder, add: Add) -> (n: int, scap: int, err: Decode_Error) { + if add == .Length_Unknown { + if .Disallow_Streaming in d.flags { + return -1, -1, .Disallowed_Streaming + } + return -1, INITIAL_STREAMED_CONTAINER_CAPACITY, nil + } + + _n := _decode_uint_as_u64(d.reader, add) or_return + if _n > u64(max(int)) { return -1, -1, .Length_Too_Big } + n = int(_n) + + scap = n + if .Trusted_Input not_in d.flags { + // NOTE: if this is a map it will be twice this. + scap = min(d.max_pre_alloc / size_of(Value), scap) + } + + return +} + +// Deterministic encoding is (among other things) encoding all values into their smallest +// possible representation. +// See section 4 of RFC 8949. + +_encode_deterministic_uint :: proc { + _encode_u8, + _encode_deterministic_u16, + _encode_deterministic_u32, + _encode_deterministic_u64, + _encode_deterministic_u128, +} + +_encode_deterministic_u16 :: proc(w: io.Writer, v: u16, major: Major = .Unsigned) -> Encode_Error { + switch { + case v <= u16(max(u8)): return _encode_u8(w, u8(v), major) + case: return _encode_u16_exact(w, v, major) + } +} + +_encode_deterministic_u32 :: proc(w: io.Writer, v: u32, major: Major = .Unsigned) -> Encode_Error { + switch { + case v <= u32(max(u8)): return _encode_u8(w, u8(v), major) + case v <= u32(max(u16)): return _encode_u16_exact(w, u16(v), major) + case: return _encode_u32_exact(w, u32(v), major) + } +} + +_encode_deterministic_u64 :: proc(w: io.Writer, v: u64, major: Major = .Unsigned) -> Encode_Error { + switch { + case v <= u64(max(u8)): return _encode_u8(w, u8(v), major) + case v <= u64(max(u16)): return _encode_u16_exact(w, u16(v), major) + case v <= u64(max(u32)): return _encode_u32_exact(w, u32(v), major) + case: return _encode_u64_exact(w, u64(v), major) + } +} + +_encode_deterministic_u128 :: proc(w: io.Writer, v: u128, major: Major = .Unsigned) -> Encode_Error { + switch { + case v <= u128(max(u8)): return _encode_u8(w, u8(v), major) + case v <= u128(max(u16)): return _encode_u16_exact(w, u16(v), major) + case v <= u128(max(u32)): return _encode_u32_exact(w, u32(v), major) + case v <= u128(max(u64)): return _encode_u64_exact(w, u64(v), major) + case: return .Int_Too_Big + } +} + +_encode_deterministic_negative :: #force_inline proc(w: io.Writer, v: $T) -> Encode_Error + where T == Negative_U8 || T == Negative_U16 || T == Negative_U32 || T == Negative_U64 { + return _encode_deterministic_uint(w, v, .Negative) +} + +// A Deterministic float is a float in the smallest type that stays the same after down casting. +_encode_deterministic_float :: proc { + _encode_f16, + _encode_deterministic_f32, + _encode_deterministic_f64, +} + +_encode_deterministic_f32 :: proc(w: io.Writer, v: f32) -> io.Error { + if (f32(f16(v)) == v) { + return _encode_f16(w, f16(v)) + } + + return _encode_f32_exact(w, v) +} + +_encode_deterministic_f64 :: proc(w: io.Writer, v: f64) -> io.Error { + if (f64(f16(v)) == v) { + return _encode_f16(w, f16(v)) + } + + if (f64(f32(v)) == v) { + return _encode_f32_exact(w, f32(v)) + } + + return _encode_f64_exact(w, v) +} diff --git a/core/encoding/cbor/doc.odin b/core/encoding/cbor/doc.odin new file mode 100644 index 000000000..937b1b61b --- /dev/null +++ b/core/encoding/cbor/doc.odin @@ -0,0 +1,141 @@ +/* +Package cbor encodes, decodes, marshals and unmarshals types from/into RCF 8949 compatible CBOR binary. +Also provided are conversion to and from JSON and the CBOR diagnostic format. + +**Allocations:** + +In general, when in the following table it says allocations are done on the `temp_allocator`, these allocations +are still attempted to be deallocated. +This allows you to use an allocator with freeing implemented as the `temp_allocator` which is handy with big CBOR. + +- *Encoding*: If the `.Deterministic_Map_Sorting` flag is set on the encoder, this allocates on the given `temp_allocator` + some space for the keys of maps in order to sort them and then write them. + Other than that there are no allocations (only for the final bytes if you use `cbor.encode_into_bytes`. + +- *Decoding*: Allocates everything on the given allocator and input given can be deleted after decoding. + *No* temporary allocations are done. + +- *Marshal*: Same allocation strategy as encoding. + +- *Unmarshal*: Allocates everything on the given allocator and input given can be deleted after unmarshalling. + Some temporary allocations are done on the given `temp_allocator`. + +**Determinism:** + +CBOR defines a deterministic en/decoder, which among other things uses the smallest type possible for integers and floats, +and sorts map keys by their (encoded) lexical bytewise order. + +You can enable this behaviour using a combination of flags, also available as the `cbor.ENCODE_FULLY_DETERMINISTIC` constant. +If you just want the small size that comes with this, but not the map sorting (which has a performance cost) you can use the +`cbor.ENCODE_SMALL` constant for the flags. + +A deterministic float is a float in the smallest type (f16, f32, f64) that hasn't changed after conversion. +A deterministic integer is an integer in the smallest representation (u8, u16, u32, u64) it fits in. + +**Untrusted Input:** + +By default input is treated as untrusted, this means the sizes that are encoded in the CBOR are not blindly trusted. +If you were to trust these sizes, and allocate space for them an attacker would be able to cause massive allocations with small payloads. + +The decoder has a `max_pre_alloc` field that specifies the maximum amount of bytes (roughly) to pre allocate, a KiB by default. + +This does mean reallocations are more common though, you can, if you know the input is trusted, add the `.Trusted_Input` flag to the decoder. + +**Tags:** + +CBOR describes tags that you can wrap values with to assign a number to describe what type of data will follow. + +More information and a list of default tags can be found here: [[RFC 8949 Section 3.4;https://www.rfc-editor.org/rfc/rfc8949.html#name-tagging-of-items]]. + +A list of registered extension types can be found here: [[IANA CBOR assignments;https://www.iana.org/assignments/cbor-tags/cbor-tags.xhtml]]. + +Tags can either be assigned to a distinct Odin type (used by default), +or be used with struct tags (`cbor_tag:"base64"`, or `cbor_tag:"1"` for example). + +By default, the following tags are supported/provided by this implementation: + +- *1/epoch*: Assign this tag to `time.Time` or integer fields to use the defined seconds since epoch format. + +- *24/cbor*: Assign this tag to string or byte fields to store encoded CBOR (not decoding it). + +- *34/base64*: Assign this tag to string or byte fields to store and decode the contents in base64. + +- *2 & 3*: Used automatically by the implementation to encode and decode big numbers into/from `core:math/big`. + +- *55799*: Self described CBOR, used when `.Self_Described_CBOR` flag is used to wrap the entire binary. + This shows other implementations that we are dealing with CBOR by just looking at the first byte of input. + +- *1010*: An extension tag that defines a string type followed by its value, this is used by this implementation to support Odin's unions. + +Users can provide their own tag implementations using the `cbor.tag_register_type(...)` to register a tag for a distinct Odin type +used automatically when it is encountered during marshal and unmarshal. +Or with `cbor.tag_register_number(...)` to register a tag number along with an identifier for convenience that can be used with struct tags, +e.g. `cbor_tag:"69"` or `cbor_tag:"my_tag"`. + +You can look at the default tags provided for pointers on how these implementations work. + +Example: + package main + + import "core:encoding/cbor" + import "core:fmt" + import "core:time" + + Possibilities :: union { + string, + int, + } + + Data :: struct { + str: string, + neg: cbor.Negative_U16, // Store a CBOR value directly. + now: time.Time `cbor_tag:"epoch"`, // Wrapped in the epoch tag. + ignore_this: ^Data `cbor:"-"`, // Ignored by implementation. + renamed: f32 `cbor:"renamed :)"`, // Renamed when encoded. + my_union: Possibilities, // Union support. + } + + main :: proc() { + now := time.Time{_nsec = 1701117968 * 1e9} + + data := Data{ + str = "Hello, World!", + neg = 300, + now = now, + ignore_this = &Data{}, + renamed = 123123.125, + my_union = 3, + } + + // Marshal the struct into binary CBOR. + binary, err := cbor.marshal(data, cbor.ENCODE_FULLY_DETERMINISTIC) + assert(err == nil) + defer delete(binary) + + // Decode the binary data into a `cbor.Value`. + decoded, derr := cbor.decode(string(binary)) + assert(derr == nil) + defer cbor.destroy(decoded) + + // Turn the CBOR into a human readable representation defined as the diagnostic format in [[RFC 8949 Section 8;https://www.rfc-editor.org/rfc/rfc8949.html#name-diagnostic-notation]]. + diagnosis, eerr := cbor.to_diagnostic_format(decoded) + assert(eerr == nil) + defer delete(diagnosis) + + fmt.println(diagnosis) + } + +Output: + { + "my_union": 1010([ + "int", + 3 + ]), + "neg": -301, + "now": 1(1701117968), + "renamed :)": 123123.12500000, + "str": "Hello, World!" + } +*/ +package encoding_cbor + diff --git a/core/encoding/cbor/marshal.odin b/core/encoding/cbor/marshal.odin new file mode 100644 index 000000000..37c9dd180 --- /dev/null +++ b/core/encoding/cbor/marshal.odin @@ -0,0 +1,575 @@ +package encoding_cbor + +import "base:intrinsics" +import "base:runtime" + +import "core:bytes" +import "core:io" +import "core:mem" +import "core:reflect" +import "core:slice" +import "core:strconv" +import "core:strings" +import "core:unicode/utf8" + +/* +Marshal a value into binary CBOR. + +Flags can be used to control the output (mainly determinism, which coincidently affects size). + +The default flags `ENCODE_SMALL` (`.Deterministic_Int_Size`, `.Deterministic_Float_Size`) will try +to put ints and floats into their smallest possible byte size without losing equality. + +Adding the `.Self_Described_CBOR` flag will wrap the value in a tag that lets generic decoders know +the contents are CBOR from just reading the first byte. + +Adding the `.Deterministic_Map_Sorting` flag will sort the encoded maps by the byte content of the +encoded key. This flag has a cost on performance and memory efficiency because all keys in a map +have to be precomputed, sorted and only then written to the output. + +Empty flags will do nothing extra to the value. + +The allocations for the `.Deterministic_Map_Sorting` flag are done using the given `temp_allocator`. +but are followed by the necessary `delete` and `free` calls if the allocator supports them. +This is helpful when the CBOR size is so big that you don't want to collect all the temporary +allocations until the end. +*/ +marshal_into :: proc { + marshal_into_bytes, + marshal_into_builder, + marshal_into_writer, + marshal_into_encoder, +} + +marshal :: marshal_into + +// Marshals the given value into a CBOR byte stream (allocated using the given allocator). +// See docs on the `marshal_into` proc group for more info. +marshal_into_bytes :: proc(v: any, flags := ENCODE_SMALL, allocator := context.allocator, temp_allocator := context.temp_allocator) -> (bytes: []byte, err: Marshal_Error) { + b, alloc_err := strings.builder_make(allocator) + // The builder as a stream also returns .EOF if it ran out of memory so this is consistent. + if alloc_err != nil { + return nil, .EOF + } + + defer if err != nil { strings.builder_destroy(&b) } + + if err = marshal_into_builder(&b, v, flags, temp_allocator); err != nil { + return + } + + return b.buf[:], nil +} + +// Marshals the given value into a CBOR byte stream written to the given builder. +// See docs on the `marshal_into` proc group for more info. +marshal_into_builder :: proc(b: ^strings.Builder, v: any, flags := ENCODE_SMALL, temp_allocator := context.temp_allocator) -> Marshal_Error { + return marshal_into_writer(strings.to_writer(b), v, flags, temp_allocator) +} + +// Marshals the given value into a CBOR byte stream written to the given writer. +// See docs on the `marshal_into` proc group for more info. +marshal_into_writer :: proc(w: io.Writer, v: any, flags := ENCODE_SMALL, temp_allocator := context.temp_allocator) -> Marshal_Error { + encoder := Encoder{flags, w, temp_allocator} + return marshal_into_encoder(encoder, v) +} + +// Marshals the given value into a CBOR byte stream written to the given encoder. +// See docs on the `marshal_into` proc group for more info. +marshal_into_encoder :: proc(e: Encoder, v: any) -> (err: Marshal_Error) { + e := e + + if e.temp_allocator.procedure == nil { + e.temp_allocator = context.temp_allocator + } + + if .Self_Described_CBOR in e.flags { + err_conv(_encode_u64(e, TAG_SELF_DESCRIBED_CBOR, .Tag)) or_return + e.flags &~= { .Self_Described_CBOR } + } + + if v == nil { + return _encode_nil(e.writer) + } + + // Check if type has a tag implementation to use. + if impl, ok := _tag_implementations_type[v.id]; ok { + return impl->marshal(e, v) + } + + ti := runtime.type_info_base(type_info_of(v.id)) + a := any{v.data, ti.id} + + #partial switch info in ti.variant { + case runtime.Type_Info_Named: + unreachable() + + case runtime.Type_Info_Pointer: + switch vv in v { + case Undefined: return _encode_undefined(e.writer) + case Nil: return _encode_nil(e.writer) + } + + case runtime.Type_Info_Integer: + switch vv in v { + case Simple: return err_conv(_encode_simple(e.writer, vv)) + case Negative_U8: return _encode_u8(e.writer, u8(vv), .Negative) + case Negative_U16: return err_conv(_encode_u16(e, u16(vv), .Negative)) + case Negative_U32: return err_conv(_encode_u32(e, u32(vv), .Negative)) + case Negative_U64: return err_conv(_encode_u64(e, u64(vv), .Negative)) + } + + switch i in a { + case i8: return _encode_uint(e.writer, _int_to_uint(i)) + case i16: return err_conv(_encode_uint(e, _int_to_uint(i))) + case i32: return err_conv(_encode_uint(e, _int_to_uint(i))) + case i64: return err_conv(_encode_uint(e, _int_to_uint(i))) + case i128: return err_conv(_encode_uint(e, _int_to_uint(i128(i)) or_return)) + case int: return err_conv(_encode_uint(e, _int_to_uint(i64(i)))) + + case u8: return _encode_uint(e.writer, i) + case u16: return err_conv(_encode_uint(e, i)) + case u32: return err_conv(_encode_uint(e, i)) + case u64: return err_conv(_encode_uint(e, i)) + case u128: return err_conv(_encode_uint(e, _u128_to_u64(u128(i)) or_return)) + case uint: return err_conv(_encode_uint(e, u64(i))) + case uintptr: return err_conv(_encode_uint(e, u64(i))) + + case i16le: return err_conv(_encode_uint(e, _int_to_uint(i16(i)))) + case i32le: return err_conv(_encode_uint(e, _int_to_uint(i32(i)))) + case i64le: return err_conv(_encode_uint(e, _int_to_uint(i64(i)))) + case i128le: return err_conv(_encode_uint(e, _int_to_uint(i128(i)) or_return)) + + case u16le: return err_conv(_encode_uint(e, u16(i))) + case u32le: return err_conv(_encode_uint(e, u32(i))) + case u64le: return err_conv(_encode_uint(e, u64(i))) + case u128le: return err_conv(_encode_uint(e, _u128_to_u64(u128(i)) or_return)) + + case i16be: return err_conv(_encode_uint(e, _int_to_uint(i16(i)))) + case i32be: return err_conv(_encode_uint(e, _int_to_uint(i32(i)))) + case i64be: return err_conv(_encode_uint(e, _int_to_uint(i64(i)))) + case i128be: return err_conv(_encode_uint(e, _int_to_uint(i128(i)) or_return)) + + case u16be: return err_conv(_encode_uint(e, u16(i))) + case u32be: return err_conv(_encode_uint(e, u32(i))) + case u64be: return err_conv(_encode_uint(e, u64(i))) + case u128be: return err_conv(_encode_uint(e, _u128_to_u64(u128(i)) or_return)) + } + + case runtime.Type_Info_Rune: + buf, w := utf8.encode_rune(a.(rune)) + return err_conv(_encode_text(e, string(buf[:w]))) + + case runtime.Type_Info_Float: + switch f in a { + case f16: return _encode_f16(e.writer, f) + case f32: return _encode_f32(e, f) + case f64: return _encode_f64(e, f) + + case f16le: return _encode_f16(e.writer, f16(f)) + case f32le: return _encode_f32(e, f32(f)) + case f64le: return _encode_f64(e, f64(f)) + + case f16be: return _encode_f16(e.writer, f16(f)) + case f32be: return _encode_f32(e, f32(f)) + case f64be: return _encode_f64(e, f64(f)) + } + + case runtime.Type_Info_Complex: + switch z in a { + case complex32: + arr: [2]Value = {real(z), imag(z)} + return err_conv(_encode_array(e, arr[:])) + case complex64: + arr: [2]Value = {real(z), imag(z)} + return err_conv(_encode_array(e, arr[:])) + case complex128: + arr: [2]Value = {real(z), imag(z)} + return err_conv(_encode_array(e, arr[:])) + } + + case runtime.Type_Info_Quaternion: + switch q in a { + case quaternion64: + arr: [4]Value = {imag(q), jmag(q), kmag(q), real(q)} + return err_conv(_encode_array(e, arr[:])) + case quaternion128: + arr: [4]Value = {imag(q), jmag(q), kmag(q), real(q)} + return err_conv(_encode_array(e, arr[:])) + case quaternion256: + arr: [4]Value = {imag(q), jmag(q), kmag(q), real(q)} + return err_conv(_encode_array(e, arr[:])) + } + + case runtime.Type_Info_String: + switch s in a { + case string: return err_conv(_encode_text(e, s)) + case cstring: return err_conv(_encode_text(e, string(s))) + } + + case runtime.Type_Info_Boolean: + switch b in a { + case bool: return _encode_bool(e.writer, b) + case b8: return _encode_bool(e.writer, bool(b)) + case b16: return _encode_bool(e.writer, bool(b)) + case b32: return _encode_bool(e.writer, bool(b)) + case b64: return _encode_bool(e.writer, bool(b)) + } + + case runtime.Type_Info_Array: + if info.elem.id == byte { + raw := ([^]byte)(v.data) + return err_conv(_encode_bytes(e, raw[:info.count])) + } + + err_conv(_encode_u64(e, u64(info.count), .Array)) or_return + for i in 0.. (res: [10]byte) { + e := e + builder := strings.builder_from_slice(res[:]) + e.writer = strings.to_stream(&builder) + + assert(_encode_u64(e, u64(len(str)), .Text) == nil) + res[9] = u8(len(builder.buf)) + assert(res[9] < 10) + return + } + + Encoded_Entry_Fast :: struct($T: typeid) { + pre_key: [10]byte, + key: T, + val_idx: uintptr, + } + + Encoded_Entry :: struct { + key: ^[dynamic]byte, + val_idx: uintptr, + } + + switch info.key.id { + case string: + entries := make([dynamic]Encoded_Entry_Fast(^[]byte), 0, map_cap, e.temp_allocator) or_return + defer delete(entries) + + for bucket_index in 0.. slice.Ordering { + a, b := a, b + pre_cmp := slice.Ordering(bytes.compare(a.pre_key[:a.pre_key[9]], b.pre_key[:b.pre_key[9]])) + if pre_cmp != .Equal { + return pre_cmp + } + + return slice.Ordering(bytes.compare(a.key^, b.key^)) + }) + + for &entry in entries { + io.write_full(e.writer, entry.pre_key[:entry.pre_key[9]]) or_return + io.write_full(e.writer, entry.key^) or_return + + value := rawptr(runtime.map_cell_index_dynamic(vs, info.map_info.vs, entry.val_idx)) + marshal_into(e, any{ value, info.value.id }) or_return + } + return + + case cstring: + entries := make([dynamic]Encoded_Entry_Fast(^cstring), 0, map_cap, e.temp_allocator) or_return + defer delete(entries) + + for bucket_index in 0.. slice.Ordering { + a, b := a, b + pre_cmp := slice.Ordering(bytes.compare(a.pre_key[:a.pre_key[9]], b.pre_key[:b.pre_key[9]])) + if pre_cmp != .Equal { + return pre_cmp + } + + ab := transmute([]byte)string(a.key^) + bb := transmute([]byte)string(b.key^) + return slice.Ordering(bytes.compare(ab, bb)) + }) + + for &entry in entries { + io.write_full(e.writer, entry.pre_key[:entry.pre_key[9]]) or_return + io.write_full(e.writer, transmute([]byte)string(entry.key^)) or_return + + value := rawptr(runtime.map_cell_index_dynamic(vs, info.map_info.vs, entry.val_idx)) + marshal_into(e, any{ value, info.value.id }) or_return + } + return + + case: + entries := make([dynamic]Encoded_Entry, 0, map_cap, e.temp_allocator) or_return + defer delete(entries) + + for bucket_index in 0.. slice.Ordering { + return slice.Ordering(bytes.compare(a.key[:], b.key[:])) + }) + + for entry in entries { + io.write_full(e.writer, entry.key[:]) or_return + delete(entry.key^) + + value := rawptr(runtime.map_cell_index_dynamic(vs, info.map_info.vs, entry.val_idx)) + marshal_into(e, any{ value, info.value.id }) or_return + } + return + } + } + + case runtime.Type_Info_Struct: + switch vv in v { + case Tag: return err_conv(_encode_tag(e, vv)) + } + + field_name :: #force_inline proc(info: runtime.Type_Info_Struct, i: int) -> string { + if cbor_name := string(reflect.struct_tag_get(reflect.Struct_Tag(info.tags[i]), "cbor")); cbor_name != "" { + return cbor_name + } else { + return info.names[i] + } + } + + marshal_entry :: #force_inline proc(e: Encoder, info: runtime.Type_Info_Struct, v: any, name: string, i: int) -> Marshal_Error { + err_conv(_encode_text(e, name)) or_return + + id := info.types[i].id + data := rawptr(uintptr(v.data) + info.offsets[i]) + field_any := any{data, id} + + if tag := string(reflect.struct_tag_get(reflect.Struct_Tag(info.tags[i]), "cbor_tag")); tag != "" { + if impl, ok := _tag_implementations_id[tag]; ok { + return impl->marshal(e, field_any) + } + + nr, ok := strconv.parse_u64_of_base(tag, 10) + if !ok { return .Invalid_CBOR_Tag } + + if impl, nok := _tag_implementations_nr[nr]; nok { + return impl->marshal(e, field_any) + } + + err_conv(_encode_u64(e, nr, .Tag)) or_return + } + + return marshal_into(e, field_any) + } + + n: u64; { + for _, i in info.names { + if field_name(info, i) != "-" { + n += 1 + } + } + err_conv(_encode_u64(e, n, .Map)) or_return + } + + if .Deterministic_Map_Sorting in e.flags { + Name :: struct { + name: string, + field: int, + } + entries := make([dynamic]Name, 0, n, e.temp_allocator) or_return + defer delete(entries) + + for _, i in info.names { + fname := field_name(info, i) + if fname == "-" { + continue + } + + append(&entries, Name{fname, i}) or_return + } + + // Sort lexicographic on the bytes of the key. + slice.sort_by_cmp(entries[:], proc(a, b: Name) -> slice.Ordering { + return slice.Ordering(bytes.compare(transmute([]byte)a.name, transmute([]byte)b.name)) + }) + + for entry in entries { + marshal_entry(e, info, v, entry.name, entry.field) or_return + } + } else { + for _, i in info.names { + fname := field_name(info, i) + if fname == "-" { + continue + } + + marshal_entry(e, info, v, fname, i) or_return + } + } + return + + case runtime.Type_Info_Union: + switch vv in v { + case Value: return err_conv(encode(e, vv)) + } + + id := reflect.union_variant_typeid(v) + if v.data == nil || id == nil { + return _encode_nil(e.writer) + } + + if len(info.variants) == 1 { + return marshal_into(e, any{v.data, id}) + } + + // Encode a non-nil multi-variant union as the `TAG_OBJECT_TYPE`. + // Which is a tag of an array, where the first element is the textual id/type of the object + // that follows it. + + err_conv(_encode_u16(e, TAG_OBJECT_TYPE, .Tag)) or_return + _encode_u8(e.writer, 2, .Array) or_return + + vti := reflect.union_variant_type_info(v) + #partial switch vt in vti.variant { + case reflect.Type_Info_Named: + err_conv(_encode_text(e, vt.name)) or_return + case: + builder := strings.builder_make(e.temp_allocator) or_return + defer strings.builder_destroy(&builder) + reflect.write_type(&builder, vti) + err_conv(_encode_text(e, strings.to_string(builder))) or_return + } + + return marshal_into(e, any{v.data, vti.id}) + + case runtime.Type_Info_Enum: + return marshal_into(e, any{v.data, info.base.id}) + + case runtime.Type_Info_Bit_Set: + // Store bit_set as big endian just like the protocol. + do_byte_swap := !reflect.bit_set_is_big_endian(v) + switch ti.size * 8 { + case 0: + return _encode_u8(e.writer, 0) + case 8: + x := (^u8)(v.data)^ + return _encode_u8(e.writer, x) + case 16: + x := (^u16)(v.data)^ + if do_byte_swap { x = intrinsics.byte_swap(x) } + return err_conv(_encode_u16(e, x)) + case 32: + x := (^u32)(v.data)^ + if do_byte_swap { x = intrinsics.byte_swap(x) } + return err_conv(_encode_u32(e, x)) + case 64: + x := (^u64)(v.data)^ + if do_byte_swap { x = intrinsics.byte_swap(x) } + return err_conv(_encode_u64(e, x)) + case: + panic("unknown bit_size size") + } + } + + return _unsupported(v.id, nil) +} diff --git a/core/encoding/cbor/tags.odin b/core/encoding/cbor/tags.odin new file mode 100644 index 000000000..3dc79a5dd --- /dev/null +++ b/core/encoding/cbor/tags.odin @@ -0,0 +1,381 @@ +package encoding_cbor + +import "base:runtime" + +import "core:encoding/base64" +import "core:io" +import "core:math" +import "core:math/big" +import "core:mem" +import "core:reflect" +import "core:strings" +import "core:time" + +// Tags defined in RFC 7049 that we provide implementations for. + +// UTC time in seconds, unmarshalled into a `core:time` `time.Time` or integer. +// Use the struct tag `cbor_tag:"1"` or `cbor_tag:"epoch"` to have your `time.Time` field en/decoded as epoch time. +TAG_EPOCH_TIME_NR :: 1 +TAG_EPOCH_TIME_ID :: "epoch" + +// Using `core:math/big`, big integers are properly encoded and decoded during marshal and unmarshal. +// These fields use this tag by default, no struct tag required. +TAG_UNSIGNED_BIG_NR :: 2 +// Using `core:math/big`, big integers are properly encoded and decoded during marshal and unmarshal. +// These fields use this tag by default, no struct tag required. +TAG_NEGATIVE_BIG_NR :: 3 + +// TAG_DECIMAL_FRACTION :: 4 // NOTE: We could probably implement this with `math/fixed`. + +// Sometimes it is beneficial to carry an embedded CBOR data item that is not meant to be decoded +// immediately at the time the enclosing data item is being decoded. Tag number 24 (CBOR data item) +// can be used to tag the embedded byte string as a single data item encoded in CBOR format. +// Use the struct tag `cbor_tag:"24"` or `cbor_tag:"cbor"` to keep a non-decoded field (string or bytes) of raw CBOR. +TAG_CBOR_NR :: 24 +TAG_CBOR_ID :: "cbor" + +// The contents of this tag are base64 encoded during marshal and decoded during unmarshal. +// Use the struct tag `cbor_tag:"34"` or `cbor_tag:"base64"` to have your field string or bytes field en/decoded as base64. +TAG_BASE64_NR :: 34 +TAG_BASE64_ID :: "base64" + +// A tag that is used to detect the contents of a binary buffer (like a file) are CBOR. +// This tag would wrap everything else, decoders can then check for this header and see if the +// given content is definitely CBOR. +// Added by the encoder if it has the flag `.Self_Described_CBOR`, decoded by default. +TAG_SELF_DESCRIBED_CBOR :: 55799 + +// A tag that is used to assign a textual type to the object following it. +// The tag's value must be an array of 2 items, where the first is text (describing the following type) +// and the second is any valid CBOR value. +// +// See the registration: https://datatracker.ietf.org/doc/draft-rundgren-cotx/05/ +// +// We use this in Odin to marshal and unmarshal unions. +TAG_OBJECT_TYPE :: 1010 + +// A tag implementation that handles marshals and unmarshals for the tag it is registered on. +Tag_Implementation :: struct { + data: rawptr, + unmarshal: Tag_Unmarshal_Proc, + marshal: Tag_Marshal_Proc, +} + +// Procedure responsible for umarshalling the tag out of the reader into the given `any`. +Tag_Unmarshal_Proc :: #type proc(self: ^Tag_Implementation, d: Decoder, tag_nr: Tag_Number, v: any) -> Unmarshal_Error + +// Procedure responsible for marshalling the tag in the given `any` into the given encoder. +Tag_Marshal_Proc :: #type proc(self: ^Tag_Implementation, e: Encoder, v: any) -> Marshal_Error + +// When encountering a tag in the CBOR being unmarshalled, the implementation is used to unmarshal it. +// When encountering a struct tag like `cbor_tag:"Tag_Number"`, the implementation is used to marshal it. +_tag_implementations_nr: map[Tag_Number]Tag_Implementation + +// Same as the number implementations but friendlier to use as a struct tag. +// Instead of `cbor_tag:"34"` you can use `cbor_tag:"base64"`. +_tag_implementations_id: map[string]Tag_Implementation + +// Tag implementations that are always used by a type, if that type is encountered in marshal it +// will rely on the implementation to marshal it. +// +// This is good for types that don't make sense or can't marshal in its default form. +_tag_implementations_type: map[typeid]Tag_Implementation + +// Register a custom tag implementation to be used when marshalling that type and unmarshalling that tag number. +tag_register_type :: proc(impl: Tag_Implementation, nr: Tag_Number, type: typeid) { + _tag_implementations_nr[nr] = impl + _tag_implementations_type[type] = impl +} + +// Register a custom tag implementation to be used when marshalling that tag number or marshalling +// a field with the struct tag `cbor_tag:"nr"`. +tag_register_number :: proc(impl: Tag_Implementation, nr: Tag_Number, id: string) { + _tag_implementations_nr[nr] = impl + _tag_implementations_id[id] = impl +} + +// Controls initialization of default tag implementations. +// JS and WASI default to a panic allocator so we don't want to do it on those. +INITIALIZE_DEFAULT_TAGS :: #config(CBOR_INITIALIZE_DEFAULT_TAGS, !ODIN_DEFAULT_TO_PANIC_ALLOCATOR && !ODIN_DEFAULT_TO_NIL_ALLOCATOR) + +@(private, init, disabled=!INITIALIZE_DEFAULT_TAGS) +tags_initialize_defaults :: proc() { + tags_register_defaults() +} + +// Registers tags that have implementations provided by this package. +// This is done by default and can be controlled with the `CBOR_INITIALIZE_DEFAULT_TAGS` define. +tags_register_defaults :: proc() { + tag_register_number({nil, tag_time_unmarshal, tag_time_marshal}, TAG_EPOCH_TIME_NR, TAG_EPOCH_TIME_ID) + tag_register_number({nil, tag_base64_unmarshal, tag_base64_marshal}, TAG_BASE64_NR, TAG_BASE64_ID) + tag_register_number({nil, tag_cbor_unmarshal, tag_cbor_marshal}, TAG_CBOR_NR, TAG_CBOR_ID) + + // These following tags are registered at the type level and don't require an opt-in struct tag. + // Encoding these types on its own make no sense or no data is lost to encode it. + + // En/Decoding of `big.Int` fields by default. + tag_register_type({nil, tag_big_unmarshal, tag_big_marshal}, TAG_UNSIGNED_BIG_NR, big.Int) + tag_register_type({nil, tag_big_unmarshal, tag_big_marshal}, TAG_NEGATIVE_BIG_NR, big.Int) +} + +// Tag number 1 contains a numerical value counting the number of seconds from 1970-01-01T00:00Z +// in UTC time to the represented point in civil time. +// +// See RFC 8949 section 3.4.2. +@(private) +tag_time_unmarshal :: proc(_: ^Tag_Implementation, d: Decoder, _: Tag_Number, v: any) -> (err: Unmarshal_Error) { + hdr := _decode_header(d.reader) or_return + #partial switch hdr { + case .U8, .U16, .U32, .U64, .Neg_U8, .Neg_U16, .Neg_U32, .Neg_U64: + switch &dst in v { + case time.Time: + i: i64 + _unmarshal_any_ptr(d, &i, hdr) or_return + dst = time.unix(i64(i), 0) + return + case: + return _unmarshal_value(d, v, hdr) + } + + case .F16, .F32, .F64: + switch &dst in v { + case time.Time: + f: f64 + _unmarshal_any_ptr(d, &f, hdr) or_return + whole, fract := math.modf(f) + dst = time.unix(i64(whole), i64(fract * 1e9)) + return + case: + return _unmarshal_value(d, v, hdr) + } + + case: + maj, add := _header_split(hdr) + if maj == .Other { + i := _decode_tiny_u8(add) or_return + + switch &dst in v { + case time.Time: + dst = time.unix(i64(i), 0) + case: + if _assign_int(v, i) { return } + } + } + + // Only numbers and floats are allowed in this tag. + return .Bad_Tag_Value + } + + return _unsupported(v, hdr) +} + +@(private) +tag_time_marshal :: proc(_: ^Tag_Implementation, e: Encoder, v: any) -> Marshal_Error { + switch vv in v { + case time.Time: + // NOTE: we lose precision here, which is one of the reasons for this tag being opt-in. + i := time.time_to_unix(vv) + + _encode_u8(e.writer, TAG_EPOCH_TIME_NR, .Tag) or_return + return err_conv(_encode_uint(e, _int_to_uint(i))) + case: + unreachable() + } +} + +@(private) +tag_big_unmarshal :: proc(_: ^Tag_Implementation, d: Decoder, tnr: Tag_Number, v: any) -> (err: Unmarshal_Error) { + hdr := _decode_header(d.reader) or_return + maj, add := _header_split(hdr) + if maj != .Bytes { + // Only bytes are supported in this tag. + return .Bad_Tag_Value + } + + switch &dst in v { + case big.Int: + bytes := err_conv(_decode_bytes(d, add)) or_return + defer delete(bytes) + + if err := big.int_from_bytes_big(&dst, bytes); err != nil { + return .Bad_Tag_Value + } + + if tnr == TAG_NEGATIVE_BIG_NR { + dst.sign = .Negative + } + + return + } + + return _unsupported(v, hdr) +} + +@(private) +tag_big_marshal :: proc(_: ^Tag_Implementation, e: Encoder, v: any) -> Marshal_Error { + switch &vv in v { + case big.Int: + if !big.int_is_initialized(&vv) { + _encode_u8(e.writer, TAG_UNSIGNED_BIG_NR, .Tag) or_return + return _encode_u8(e.writer, 0, .Bytes) + } + + // NOTE: using the panic_allocator because all procedures should only allocate if the Int + // is uninitialized (which we checked). + + is_neg, err := big.is_negative(&vv, mem.panic_allocator()) + assert(err == nil, "should only error if not initialized, which has been checked") + + tnr: u8 = TAG_NEGATIVE_BIG_NR if is_neg else TAG_UNSIGNED_BIG_NR + _encode_u8(e.writer, tnr, .Tag) or_return + + size_in_bytes, berr := big.int_to_bytes_size(&vv, false, mem.panic_allocator()) + assert(berr == nil, "should only error if not initialized, which has been checked") + assert(size_in_bytes >= 0) + + err_conv(_encode_u64(e, u64(size_in_bytes), .Bytes)) or_return + + for offset := (size_in_bytes*8)-8; offset >= 0; offset -= 8 { + bits, derr := big.int_bitfield_extract(&vv, offset, 8, mem.panic_allocator()) + assert(derr == nil, "should only error if not initialized or invalid argument (offset and count), which won't happen") + + io.write_full(e.writer, {u8(bits & 255)}) or_return + } + return nil + + case: unreachable() + } +} + +@(private) +tag_cbor_unmarshal :: proc(_: ^Tag_Implementation, d: Decoder, _: Tag_Number, v: any) -> Unmarshal_Error { + hdr := _decode_header(d.reader) or_return + major, add := _header_split(hdr) + #partial switch major { + case .Bytes: + ti := reflect.type_info_base(type_info_of(v.id)) + return _unmarshal_bytes(d, v, ti, hdr, add) + + case: return .Bad_Tag_Value + } +} + +@(private) +tag_cbor_marshal :: proc(_: ^Tag_Implementation, e: Encoder, v: any) -> Marshal_Error { + _encode_u8(e.writer, TAG_CBOR_NR, .Tag) or_return + ti := runtime.type_info_base(type_info_of(v.id)) + #partial switch t in ti.variant { + case runtime.Type_Info_String: + return marshal_into(e, v) + case runtime.Type_Info_Array: + elem_base := reflect.type_info_base(t.elem) + if elem_base.id != byte { return .Bad_Tag_Value } + return marshal_into(e, v) + case runtime.Type_Info_Slice: + elem_base := reflect.type_info_base(t.elem) + if elem_base.id != byte { return .Bad_Tag_Value } + return marshal_into(e, v) + case runtime.Type_Info_Dynamic_Array: + elem_base := reflect.type_info_base(t.elem) + if elem_base.id != byte { return .Bad_Tag_Value } + return marshal_into(e, v) + case: + return .Bad_Tag_Value + } +} + +@(private) +tag_base64_unmarshal :: proc(_: ^Tag_Implementation, d: Decoder, _: Tag_Number, v: any) -> (err: Unmarshal_Error) { + hdr := _decode_header(d.reader) or_return + major, add := _header_split(hdr) + ti := reflect.type_info_base(type_info_of(v.id)) + + if major != .Text && major != .Bytes { + return .Bad_Tag_Value + } + + bytes := string(err_conv(_decode_bytes(d, add, allocator=context.temp_allocator)) or_return) + defer delete(bytes, context.temp_allocator) + + #partial switch t in ti.variant { + case reflect.Type_Info_String: + + if t.is_cstring { + length := base64.decoded_len(bytes) + builder := strings.builder_make(0, length+1) + base64.decode_into(strings.to_stream(&builder), bytes) or_return + + raw := (^cstring)(v.data) + raw^ = cstring(raw_data(builder.buf)) + } else { + raw := (^string)(v.data) + raw^ = string(base64.decode(bytes) or_return) + } + + return + + case reflect.Type_Info_Slice: + elem_base := reflect.type_info_base(t.elem) + + if elem_base.id != byte { return _unsupported(v, hdr) } + + raw := (^[]byte)(v.data) + raw^ = base64.decode(bytes) or_return + return + + case reflect.Type_Info_Dynamic_Array: + elem_base := reflect.type_info_base(t.elem) + + if elem_base.id != byte { return _unsupported(v, hdr) } + + decoded := base64.decode(bytes) or_return + + raw := (^mem.Raw_Dynamic_Array)(v.data) + raw.data = raw_data(decoded) + raw.len = len(decoded) + raw.cap = len(decoded) + raw.allocator = context.allocator + return + + case reflect.Type_Info_Array: + elem_base := reflect.type_info_base(t.elem) + + if elem_base.id != byte { return _unsupported(v, hdr) } + + if base64.decoded_len(bytes) > t.count { return _unsupported(v, hdr) } + + slice := ([^]byte)(v.data)[:len(bytes)] + copy(slice, base64.decode(bytes) or_return) + return + } + + return _unsupported(v, hdr) +} + +@(private) +tag_base64_marshal :: proc(_: ^Tag_Implementation, e: Encoder, v: any) -> Marshal_Error { + _encode_u8(e.writer, TAG_BASE64_NR, .Tag) or_return + + ti := runtime.type_info_base(type_info_of(v.id)) + a := any{v.data, ti.id} + + bytes: []byte + switch val in a { + case string: bytes = transmute([]byte)val + case cstring: bytes = transmute([]byte)string(val) + case []byte: bytes = val + case [dynamic]byte: bytes = val[:] + case: + #partial switch t in ti.variant { + case runtime.Type_Info_Array: + if t.elem.id != byte { return .Bad_Tag_Value } + bytes = ([^]byte)(v.data)[:t.count] + case: + return .Bad_Tag_Value + } + } + + out_len := base64.encoded_len(bytes) + err_conv(_encode_u64(e, u64(out_len), .Text)) or_return + return base64.encode_into(e.writer, bytes) +} diff --git a/core/encoding/cbor/unmarshal.odin b/core/encoding/cbor/unmarshal.odin new file mode 100644 index 000000000..a1524d9f4 --- /dev/null +++ b/core/encoding/cbor/unmarshal.odin @@ -0,0 +1,932 @@ +package encoding_cbor + +import "base:intrinsics" +import "base:runtime" + +import "core:io" +import "core:mem" +import "core:reflect" +import "core:strings" +import "core:unicode/utf8" + +/* +Unmarshals the given CBOR into the given pointer using reflection. +Types that require allocation are allocated using the given allocator. + +Some temporary allocations are done on the given `temp_allocator`, but, if you want to, +this can be set to a "normal" allocator, because the necessary `delete` and `free` calls are still made. +This is helpful when the CBOR size is so big that you don't want to collect all the temporary allocations until the end. + +Disable streaming/indeterminate lengths with the `.Disallow_Streaming` flag. + +Shrink excess bytes in buffers and containers with the `.Shrink_Excess` flag. + +Mark the input as trusted input with the `.Trusted_Input` flag, this turns off the safety feature +of not pre-allocating more than `max_pre_alloc` bytes before reading into the bytes. You should only +do this when you own both sides of the encoding and are sure there can't be malicious bytes used as +an input. +*/ +unmarshal :: proc { + unmarshal_from_reader, + unmarshal_from_string, +} + +unmarshal_from_reader :: proc(r: io.Reader, ptr: ^$T, flags := Decoder_Flags{}, allocator := context.allocator, temp_allocator := context.temp_allocator) -> (err: Unmarshal_Error) { + err = unmarshal_from_decoder(Decoder{ DEFAULT_MAX_PRE_ALLOC, flags, r }, ptr, allocator, temp_allocator) + + // Normal EOF does not exist here, we try to read the exact amount that is said to be provided. + if err == .EOF { err = .Unexpected_EOF } + return +} + +// Unmarshals from a string, see docs on the proc group `Unmarshal` for more info. +unmarshal_from_string :: proc(s: string, ptr: ^$T, flags := Decoder_Flags{}, allocator := context.allocator, temp_allocator := context.temp_allocator) -> (err: Unmarshal_Error) { + sr: strings.Reader + r := strings.to_reader(&sr, s) + + err = unmarshal_from_reader(r, ptr, flags, allocator, temp_allocator) + + // Normal EOF does not exist here, we try to read the exact amount that is said to be provided. + if err == .EOF { err = .Unexpected_EOF } + return +} + +unmarshal_from_decoder :: proc(d: Decoder, ptr: ^$T, allocator := context.allocator, temp_allocator := context.temp_allocator) -> (err: Unmarshal_Error) { + d := d + + err = _unmarshal_any_ptr(d, ptr, nil, allocator, temp_allocator) + + // Normal EOF does not exist here, we try to read the exact amount that is said to be provided. + if err == .EOF { err = .Unexpected_EOF } + return + +} + +_unmarshal_any_ptr :: proc(d: Decoder, v: any, hdr: Maybe(Header) = nil, allocator := context.allocator, temp_allocator := context.temp_allocator) -> Unmarshal_Error { + context.allocator = allocator + context.temp_allocator = temp_allocator + v := v + + if v == nil || v.id == nil { + return .Invalid_Parameter + } + + v = reflect.any_base(v) + ti := type_info_of(v.id) + if !reflect.is_pointer(ti) || ti.id == rawptr { + return .Non_Pointer_Parameter + } + + data := any{(^rawptr)(v.data)^, ti.variant.(reflect.Type_Info_Pointer).elem.id} + return _unmarshal_value(d, data, hdr.? or_else (_decode_header(d.reader) or_return)) +} + +_unmarshal_value :: proc(d: Decoder, v: any, hdr: Header) -> (err: Unmarshal_Error) { + v := v + ti := reflect.type_info_base(type_info_of(v.id)) + r := d.reader + + // If it's a union with only one variant, then treat it as that variant + if u, ok := ti.variant.(reflect.Type_Info_Union); ok && len(u.variants) == 1 { + #partial switch hdr { + case .Nil, .Undefined, nil: // no-op. + case: + variant := u.variants[0] + v.id = variant.id + ti = reflect.type_info_base(variant) + if !reflect.is_pointer_internally(variant) { + tag := any{rawptr(uintptr(v.data) + u.tag_offset), u.tag_type.id} + assert(_assign_int(tag, 1)) + } + } + } + + // Allow generic unmarshal by doing it into a `Value`. + switch &dst in v { + case Value: + dst = err_conv(_decode_from_decoder(d, hdr)) or_return + return + } + + switch hdr { + case .U8: + decoded := _decode_u8(r) or_return + if !_assign_int(v, decoded) { return _unsupported(v, hdr) } + return + + case .U16: + decoded := _decode_u16(r) or_return + if !_assign_int(v, decoded) { return _unsupported(v, hdr) } + return + + case .U32: + decoded := _decode_u32(r) or_return + if !_assign_int(v, decoded) { return _unsupported(v, hdr) } + return + + case .U64: + decoded := _decode_u64(r) or_return + if !_assign_int(v, decoded) { return _unsupported(v, hdr) } + return + + case .Neg_U8: + decoded := Negative_U8(_decode_u8(r) or_return) + + switch &dst in v { + case Negative_U8: + dst = decoded + return + case Negative_U16: + dst = Negative_U16(decoded) + return + case Negative_U32: + dst = Negative_U32(decoded) + return + case Negative_U64: + dst = Negative_U64(decoded) + return + } + + if reflect.is_unsigned(ti) { return _unsupported(v, hdr) } + + if !_assign_int(v, negative_to_int(decoded)) { return _unsupported(v, hdr) } + return + + case .Neg_U16: + decoded := Negative_U16(_decode_u16(r) or_return) + + switch &dst in v { + case Negative_U16: + dst = decoded + return + case Negative_U32: + dst = Negative_U32(decoded) + return + case Negative_U64: + dst = Negative_U64(decoded) + return + } + + if reflect.is_unsigned(ti) { return _unsupported(v, hdr) } + + if !_assign_int(v, negative_to_int(decoded)) { return _unsupported(v, hdr) } + return + + case .Neg_U32: + decoded := Negative_U32(_decode_u32(r) or_return) + + switch &dst in v { + case Negative_U32: + dst = decoded + return + case Negative_U64: + dst = Negative_U64(decoded) + return + } + + if reflect.is_unsigned(ti) { return _unsupported(v, hdr) } + + if !_assign_int(v, negative_to_int(decoded)) { return _unsupported(v, hdr) } + return + + case .Neg_U64: + decoded := Negative_U64(_decode_u64(r) or_return) + + switch &dst in v { + case Negative_U64: + dst = decoded + return + } + + if reflect.is_unsigned(ti) { return _unsupported(v, hdr) } + + if !_assign_int(v, negative_to_int(decoded)) { return _unsupported(v, hdr) } + return + + case .Simple: + decoded := _decode_simple(r) or_return + + // NOTE: Because this is a special type and not to be treated as a general integer, + // We only put the value of it in fields that are explicitly of type `Simple`. + switch &dst in v { + case Simple: + dst = decoded + return + case: + return _unsupported(v, hdr) + } + + case .F16: + decoded := _decode_f16(r) or_return + if !_assign_float(v, decoded) { return _unsupported(v, hdr) } + return + + case .F32: + decoded := _decode_f32(r) or_return + if !_assign_float(v, decoded) { return _unsupported(v, hdr) } + return + + case .F64: + decoded := _decode_f64(r) or_return + if !_assign_float(v, decoded) { return _unsupported(v, hdr) } + return + + case .True: + if !_assign_bool(v, true) { return _unsupported(v, hdr) } + return + + case .False: + if !_assign_bool(v, false) { return _unsupported(v, hdr) } + return + + case .Nil, .Undefined: + mem.zero(v.data, ti.size) + return + + case .Break: + return .Break + } + + maj, add := _header_split(hdr) + switch maj { + case .Unsigned: + decoded := _decode_tiny_u8(add) or_return + if !_assign_int(v, decoded) { return _unsupported(v, hdr, add) } + return + + case .Negative: + decoded := Negative_U8(_decode_tiny_u8(add) or_return) + + switch &dst in v { + case Negative_U8: + dst = decoded + return + } + + if reflect.is_unsigned(ti) { return _unsupported(v, hdr, add) } + + if !_assign_int(v, negative_to_int(decoded)) { return _unsupported(v, hdr, add) } + return + + case .Other: + decoded := _decode_tiny_simple(add) or_return + + // NOTE: Because this is a special type and not to be treated as a general integer, + // We only put the value of it in fields that are explicitly of type `Simple`. + switch &dst in v { + case Simple: + dst = decoded + return + case: + return _unsupported(v, hdr, add) + } + + case .Tag: + switch &dst in v { + case ^Tag: + tval := err_conv(_decode_tag_ptr(d, add)) or_return + if t, is_tag := tval.(^Tag); is_tag { + dst = t + return + } + + destroy(tval) + return .Bad_Tag_Value + case Tag: + t := err_conv(_decode_tag(d, add)) or_return + if t, is_tag := t.?; is_tag { + dst = t + return + } + + return .Bad_Tag_Value + } + + nr := err_conv(_decode_uint_as_u64(r, add)) or_return + + // Custom tag implementations. + if impl, ok := _tag_implementations_nr[nr]; ok { + return impl->unmarshal(d, nr, v) + } else if nr == TAG_OBJECT_TYPE { + return _unmarshal_union(d, v, ti, hdr) + } else { + // Discard the tag info and unmarshal as its value. + return _unmarshal_value(d, v, _decode_header(r) or_return) + } + + return _unsupported(v, hdr, add) + + case .Bytes: return _unmarshal_bytes(d, v, ti, hdr, add) + case .Text: return _unmarshal_string(d, v, ti, hdr, add) + case .Array: return _unmarshal_array(d, v, ti, hdr, add) + case .Map: return _unmarshal_map(d, v, ti, hdr, add) + + case: return .Bad_Major + } +} + +_unmarshal_bytes :: proc(d: Decoder, v: any, ti: ^reflect.Type_Info, hdr: Header, add: Add) -> (err: Unmarshal_Error) { + #partial switch t in ti.variant { + case reflect.Type_Info_String: + bytes := err_conv(_decode_bytes(d, add)) or_return + + if t.is_cstring { + raw := (^cstring)(v.data) + assert_safe_for_cstring(string(bytes)) + raw^ = cstring(raw_data(bytes)) + } else { + // String has same memory layout as a slice, so we can directly use it as a slice. + raw := (^mem.Raw_String)(v.data) + raw^ = transmute(mem.Raw_String)bytes + } + + return + + case reflect.Type_Info_Slice: + elem_base := reflect.type_info_base(t.elem) + + if elem_base.id != byte { return _unsupported(v, hdr) } + + bytes := err_conv(_decode_bytes(d, add)) or_return + raw := (^mem.Raw_Slice)(v.data) + raw^ = transmute(mem.Raw_Slice)bytes + return + + case reflect.Type_Info_Dynamic_Array: + elem_base := reflect.type_info_base(t.elem) + + if elem_base.id != byte { return _unsupported(v, hdr) } + + bytes := err_conv(_decode_bytes(d, add)) or_return + raw := (^mem.Raw_Dynamic_Array)(v.data) + raw.data = raw_data(bytes) + raw.len = len(bytes) + raw.cap = len(bytes) + raw.allocator = context.allocator + return + + case reflect.Type_Info_Array: + elem_base := reflect.type_info_base(t.elem) + + if elem_base.id != byte { return _unsupported(v, hdr) } + + bytes := err_conv(_decode_bytes(d, add, allocator=context.temp_allocator)) or_return + defer delete(bytes, context.temp_allocator) + + if len(bytes) > t.count { return _unsupported(v, hdr) } + + // Copy into array type, delete original. + slice := ([^]byte)(v.data)[:len(bytes)] + n := copy(slice, bytes) + assert(n == len(bytes)) + return + } + + return _unsupported(v, hdr) +} + +_unmarshal_string :: proc(d: Decoder, v: any, ti: ^reflect.Type_Info, hdr: Header, add: Add) -> (err: Unmarshal_Error) { + #partial switch t in ti.variant { + case reflect.Type_Info_String: + text := err_conv(_decode_text(d, add)) or_return + + if t.is_cstring { + raw := (^cstring)(v.data) + + assert_safe_for_cstring(text) + raw^ = cstring(raw_data(text)) + } else { + raw := (^string)(v.data) + raw^ = text + } + return + + // Enum by its variant name. + case reflect.Type_Info_Enum: + text := err_conv(_decode_text(d, add, allocator=context.temp_allocator)) or_return + defer delete(text, context.temp_allocator) + + for name, i in t.names { + if name == text { + if !_assign_int(any{v.data, ti.id}, t.values[i]) { return _unsupported(v, hdr) } + return + } + } + + case reflect.Type_Info_Rune: + text := err_conv(_decode_text(d, add, allocator=context.temp_allocator)) or_return + defer delete(text, context.temp_allocator) + + r := (^rune)(v.data) + dr, n := utf8.decode_rune(text) + if dr == utf8.RUNE_ERROR || n < len(text) { + return _unsupported(v, hdr) + } + + r^ = dr + return + } + + return _unsupported(v, hdr) +} + +_unmarshal_array :: proc(d: Decoder, v: any, ti: ^reflect.Type_Info, hdr: Header, add: Add) -> (err: Unmarshal_Error) { + assign_array :: proc( + d: Decoder, + da: ^mem.Raw_Dynamic_Array, + elemt: ^reflect.Type_Info, + length: int, + growable := true, + ) -> (out_of_space: bool, err: Unmarshal_Error) { + for idx: uintptr = 0; length == -1 || idx < uintptr(length); idx += 1 { + elem_ptr := rawptr(uintptr(da.data) + idx*uintptr(elemt.size)) + elem := any{elem_ptr, elemt.id} + + hdr := _decode_header(d.reader) or_return + + // Double size if out of capacity. + if da.cap <= da.len { + // Not growable, error out. + if !growable { return true, .Out_Of_Memory } + + cap := 2 * da.cap + ok := runtime.__dynamic_array_reserve(da, elemt.size, elemt.align, cap) + + // NOTE: Might be lying here, but it is at least an allocator error. + if !ok { return false, .Out_Of_Memory } + } + + err = _unmarshal_value(d, elem, hdr) + if length == -1 && err == .Break { break } + if err != nil { return } + + da.len += 1 + } + + return false, nil + } + + // Allow generically storing the values array. + switch &dst in v { + case ^Array: + dst = err_conv(_decode_array_ptr(d, add)) or_return + return + case Array: + dst = err_conv(_decode_array(d, add)) or_return + return + } + + #partial switch t in ti.variant { + case reflect.Type_Info_Slice: + length, scap := err_conv(_decode_len_container(d, add)) or_return + + data := mem.alloc_bytes_non_zeroed(t.elem.size * scap, t.elem.align) or_return + defer if err != nil { mem.free_bytes(data) } + + da := mem.Raw_Dynamic_Array{raw_data(data), 0, length, context.allocator } + + assign_array(d, &da, t.elem, length) or_return + + if .Shrink_Excess in d.flags { + // Ignoring an error here, but this is not critical to succeed. + _ = runtime.__dynamic_array_shrink(&da, t.elem.size, t.elem.align, da.len) + } + + raw := (^mem.Raw_Slice)(v.data) + raw.data = da.data + raw.len = da.len + return + + case reflect.Type_Info_Dynamic_Array: + length, scap := err_conv(_decode_len_container(d, add)) or_return + + data := mem.alloc_bytes_non_zeroed(t.elem.size * scap, t.elem.align) or_return + defer if err != nil { mem.free_bytes(data) } + + raw := (^mem.Raw_Dynamic_Array)(v.data) + raw.data = raw_data(data) + raw.len = 0 + raw.cap = length + raw.allocator = context.allocator + + _ = assign_array(d, raw, t.elem, length) or_return + + if .Shrink_Excess in d.flags { + // Ignoring an error here, but this is not critical to succeed. + _ = runtime.__dynamic_array_shrink(raw, t.elem.size, t.elem.align, raw.len) + } + return + + case reflect.Type_Info_Array: + _, scap := err_conv(_decode_len_container(d, add)) or_return + length := min(scap, t.count) + + if length > t.count { + return _unsupported(v, hdr) + } + + da := mem.Raw_Dynamic_Array{rawptr(v.data), 0, length, context.allocator } + + out_of_space := assign_array(d, &da, t.elem, length, growable=false) or_return + if out_of_space { return _unsupported(v, hdr) } + return + + case reflect.Type_Info_Enumerated_Array: + _, scap := err_conv(_decode_len_container(d, add)) or_return + length := min(scap, t.count) + + if length > t.count { + return _unsupported(v, hdr) + } + + da := mem.Raw_Dynamic_Array{rawptr(v.data), 0, length, context.allocator } + + out_of_space := assign_array(d, &da, t.elem, length, growable=false) or_return + if out_of_space { return _unsupported(v, hdr) } + return + + case reflect.Type_Info_Complex: + _, scap := err_conv(_decode_len_container(d, add)) or_return + length := min(scap, 2) + + if length > 2 { + return _unsupported(v, hdr) + } + + da := mem.Raw_Dynamic_Array{rawptr(v.data), 0, 2, context.allocator } + + info: ^runtime.Type_Info + switch ti.id { + case complex32: info = type_info_of(f16) + case complex64: info = type_info_of(f32) + case complex128: info = type_info_of(f64) + case: unreachable() + } + + out_of_space := assign_array(d, &da, info, 2, growable=false) or_return + if out_of_space { return _unsupported(v, hdr) } + return + + case reflect.Type_Info_Quaternion: + _, scap := err_conv(_decode_len_container(d, add)) or_return + length := min(scap, 4) + + if length > 4 { + return _unsupported(v, hdr) + } + + da := mem.Raw_Dynamic_Array{rawptr(v.data), 0, 4, context.allocator } + + info: ^runtime.Type_Info + switch ti.id { + case quaternion64: info = type_info_of(f16) + case quaternion128: info = type_info_of(f32) + case quaternion256: info = type_info_of(f64) + case: unreachable() + } + + out_of_space := assign_array(d, &da, info, 4, growable=false) or_return + if out_of_space { return _unsupported(v, hdr) } + return + + case: return _unsupported(v, hdr) + } +} + +_unmarshal_map :: proc(d: Decoder, v: any, ti: ^reflect.Type_Info, hdr: Header, add: Add) -> (err: Unmarshal_Error) { + r := d.reader + decode_key :: proc(d: Decoder, v: any, allocator := context.allocator) -> (k: string, err: Unmarshal_Error) { + entry_hdr := _decode_header(d.reader) or_return + entry_maj, entry_add := _header_split(entry_hdr) + #partial switch entry_maj { + case .Text: + k = err_conv(_decode_text(d, entry_add, allocator)) or_return + return + case .Bytes: + bytes := err_conv(_decode_bytes(d, entry_add, allocator=allocator)) or_return + k = string(bytes) + return + case: + err = _unsupported(v, entry_hdr) + return + } + } + + // Allow generically storing the map array. + switch &dst in v { + case ^Map: + dst = err_conv(_decode_map_ptr(d, add)) or_return + return + case Map: + dst = err_conv(_decode_map(d, add)) or_return + return + } + + #partial switch t in ti.variant { + case reflect.Type_Info_Struct: + if t.is_raw_union { + return _unsupported(v, hdr) + } + + length, _ := err_conv(_decode_len_container(d, add)) or_return + unknown := length == -1 + fields := reflect.struct_fields_zipped(ti.id) + + for idx := 0; idx < len(fields) && (unknown || idx < length); idx += 1 { + // Decode key, keys can only be strings. + key: string + if keyv, kerr := decode_key(d, v, context.temp_allocator); unknown && kerr == .Break { + break + } else if kerr != nil { + err = kerr + return + } else { + key = keyv + } + defer delete(key, context.temp_allocator) + + // Find matching field. + use_field_idx := -1 + { + for field, field_idx in fields { + tag_value := string(reflect.struct_tag_get(field.tag, "cbor")) + if tag_value == "-" { + continue + } + + if key == tag_value { + use_field_idx = field_idx + break + } + + if key == field.name { + // No break because we want to still check remaining struct tags. + use_field_idx = field_idx + } + } + + // Skips unused map entries. + if use_field_idx < 0 { + continue + } + } + + field := fields[use_field_idx] + // name := field.name + ptr := rawptr(uintptr(v.data) + field.offset) + fany := any{ptr, field.type.id} + _unmarshal_value(d, fany, _decode_header(r) or_return) or_return + } + return + + case reflect.Type_Info_Map: + if !reflect.is_string(t.key) { + return _unsupported(v, hdr) + } + + raw_map := (^mem.Raw_Map)(v.data) + if raw_map.allocator.procedure == nil { + raw_map.allocator = context.allocator + } + + defer if err != nil { + _ = runtime.map_free_dynamic(raw_map^, t.map_info) + } + + length, scap := err_conv(_decode_len_container(d, add)) or_return + unknown := length == -1 + if !unknown { + // Reserve space before setting so we can return allocation errors and be efficient on big maps. + new_len := uintptr(min(scap, runtime.map_len(raw_map^)+length)) + runtime.map_reserve_dynamic(raw_map, t.map_info, new_len) or_return + } + + // Temporary memory to unmarshal keys into before inserting them into the map. + elem_backing := mem.alloc_bytes_non_zeroed(t.value.size, t.value.align, context.temp_allocator) or_return + defer delete(elem_backing, context.temp_allocator) + + map_backing_value := any{raw_data(elem_backing), t.value.id} + + for idx := 0; unknown || idx < length; idx += 1 { + // Decode key, keys can only be strings. + key: string + if keyv, kerr := decode_key(d, v); unknown && kerr == .Break { + break + } else if kerr != nil { + err = kerr + return + } else { + key = keyv + } + + if unknown || idx > scap { + // Reserve space for new element so we can return allocator errors. + new_len := uintptr(runtime.map_len(raw_map^)+1) + runtime.map_reserve_dynamic(raw_map, t.map_info, new_len) or_return + } + + mem.zero_slice(elem_backing) + _unmarshal_value(d, map_backing_value, _decode_header(r) or_return) or_return + + key_ptr := rawptr(&key) + key_cstr: cstring + if reflect.is_cstring(t.key) { + assert_safe_for_cstring(key) + key_cstr = cstring(raw_data(key)) + key_ptr = &key_cstr + } + + set_ptr := runtime.__dynamic_map_set_without_hash(raw_map, t.map_info, key_ptr, map_backing_value.data) + // We already reserved space for it, so this shouldn't fail. + assert(set_ptr != nil) + } + + if .Shrink_Excess in d.flags { + _, _ = runtime.map_shrink_dynamic(raw_map, t.map_info) + } + return + + case: + return _unsupported(v, hdr) + } +} + +// Unmarshal into a union, based on the `TAG_OBJECT_TYPE` tag of the spec, it denotes a tag which +// contains an array of exactly two elements, the first is a textual representation of the following +// CBOR value's type. +_unmarshal_union :: proc(d: Decoder, v: any, ti: ^reflect.Type_Info, hdr: Header) -> (err: Unmarshal_Error) { + r := d.reader + #partial switch t in ti.variant { + case reflect.Type_Info_Union: + idhdr: Header + target_name: string + { + vhdr := _decode_header(r) or_return + vmaj, vadd := _header_split(vhdr) + if vmaj != .Array { + return .Bad_Tag_Value + } + + n_items, _ := err_conv(_decode_len_container(d, vadd)) or_return + if n_items != 2 { + return .Bad_Tag_Value + } + + idhdr = _decode_header(r) or_return + idmaj, idadd := _header_split(idhdr) + if idmaj != .Text { + return .Bad_Tag_Value + } + + target_name = err_conv(_decode_text(d, idadd, context.temp_allocator)) or_return + } + defer delete(target_name, context.temp_allocator) + + for variant, i in t.variants { + tag := i64(i) + if !t.no_nil { + tag += 1 + } + + #partial switch vti in variant.variant { + case reflect.Type_Info_Named: + if vti.name == target_name { + reflect.set_union_variant_raw_tag(v, tag) + return _unmarshal_value(d, any{v.data, variant.id}, _decode_header(r) or_return) + } + + case: + builder := strings.builder_make(context.temp_allocator) + defer strings.builder_destroy(&builder) + + reflect.write_type(&builder, variant) + variant_name := strings.to_string(builder) + + if variant_name == target_name { + reflect.set_union_variant_raw_tag(v, tag) + return _unmarshal_value(d, any{v.data, variant.id}, _decode_header(r) or_return) + } + } + } + + // No variant matched. + return _unsupported(v, idhdr) + + case: + // Not a union. + return _unsupported(v, hdr) + } +} + +_assign_int :: proc(val: any, i: $T) -> bool { + v := reflect.any_core(val) + + // NOTE: should under/over flow be checked here? `encoding/json` doesn't, but maybe that is a + // less strict encoding?. + + switch &dst in v { + case i8: dst = i8 (i) + case i16: dst = i16 (i) + case i16le: dst = i16le (i) + case i16be: dst = i16be (i) + case i32: dst = i32 (i) + case i32le: dst = i32le (i) + case i32be: dst = i32be (i) + case i64: dst = i64 (i) + case i64le: dst = i64le (i) + case i64be: dst = i64be (i) + case i128: dst = i128 (i) + case i128le: dst = i128le (i) + case i128be: dst = i128be (i) + case u8: dst = u8 (i) + case u16: dst = u16 (i) + case u16le: dst = u16le (i) + case u16be: dst = u16be (i) + case u32: dst = u32 (i) + case u32le: dst = u32le (i) + case u32be: dst = u32be (i) + case u64: dst = u64 (i) + case u64le: dst = u64le (i) + case u64be: dst = u64be (i) + case u128: dst = u128 (i) + case u128le: dst = u128le (i) + case u128be: dst = u128be (i) + case int: dst = int (i) + case uint: dst = uint (i) + case uintptr: dst = uintptr(i) + case: + ti := type_info_of(v.id) + if _, ok := ti.variant.(runtime.Type_Info_Bit_Set); ok { + do_byte_swap := !reflect.bit_set_is_big_endian(v) + switch ti.size * 8 { + case 0: // no-op. + case 8: + x := (^u8)(v.data) + x^ = u8(i) + case 16: + x := (^u16)(v.data) + x^ = do_byte_swap ? intrinsics.byte_swap(u16(i)) : u16(i) + case 32: + x := (^u32)(v.data) + x^ = do_byte_swap ? intrinsics.byte_swap(u32(i)) : u32(i) + case 64: + x := (^u64)(v.data) + x^ = do_byte_swap ? intrinsics.byte_swap(u64(i)) : u64(i) + case: + panic("unknown bit_size size") + } + return true + } + return false + } + return true +} + +_assign_float :: proc(val: any, f: $T) -> bool { + v := reflect.any_core(val) + + // NOTE: should under/over flow be checked here? `encoding/json` doesn't, but maybe that is a + // less strict encoding?. + + switch &dst in v { + case f16: dst = f16 (f) + case f16le: dst = f16le(f) + case f16be: dst = f16be(f) + case f32: dst = f32 (f) + case f32le: dst = f32le(f) + case f32be: dst = f32be(f) + case f64: dst = f64 (f) + case f64le: dst = f64le(f) + case f64be: dst = f64be(f) + + case complex32: dst = complex(f16(f), 0) + case complex64: dst = complex(f32(f), 0) + case complex128: dst = complex(f64(f), 0) + + case quaternion64: dst = quaternion(w=f16(f), x=0, y=0, z=0) + case quaternion128: dst = quaternion(w=f32(f), x=0, y=0, z=0) + case quaternion256: dst = quaternion(w=f64(f), x=0, y=0, z=0) + + case: return false + } + return true +} + +_assign_bool :: proc(val: any, b: bool) -> bool { + v := reflect.any_core(val) + switch &dst in v { + case bool: dst = bool(b) + case b8: dst = b8 (b) + case b16: dst = b16 (b) + case b32: dst = b32 (b) + case b64: dst = b64 (b) + case: return false + } + return true +} + +// Sanity check that the decoder added a nil byte to the end. +@(private, disabled=ODIN_DISABLE_ASSERT) +assert_safe_for_cstring :: proc(s: string, loc := #caller_location) { + assert(([^]byte)(raw_data(s))[len(s)] == 0, loc = loc) +} diff --git a/core/encoding/csv/reader.odin b/core/encoding/csv/reader.odin index 44a9fdcc4..f8c72c423 100644 --- a/core/encoding/csv/reader.odin +++ b/core/encoding/csv/reader.odin @@ -1,6 +1,6 @@ // package csv reads and writes comma-separated values (CSV) files. // This package supports the format described in RFC 4180 -package csv +package encoding_csv import "core:bufio" import "core:bytes" @@ -91,7 +91,10 @@ DEFAULT_RECORD_BUFFER_CAPACITY :: 256 // reader_init initializes a new Reader from r reader_init :: proc(reader: ^Reader, r: io.Reader, buffer_allocator := context.allocator) { - reader.comma = ',' + switch reader.comma { + case '\x00', '\n', '\r', 0xfffd: + reader.comma = ',' + } context.allocator = buffer_allocator reserve(&reader.record_buffer, DEFAULT_RECORD_BUFFER_CAPACITY) @@ -121,6 +124,7 @@ reader_destroy :: proc(r: ^Reader) { // read reads a single record (a slice of fields) from r // // All \r\n sequences are normalized to \n, including multi-line field +@(require_results) read :: proc(r: ^Reader, allocator := context.allocator) -> (record: []string, err: Error) { if r.reuse_record { record, err = _read_record(r, &r.last_record, allocator) @@ -133,6 +137,7 @@ read :: proc(r: ^Reader, allocator := context.allocator) -> (record: []string, e } // is_io_error checks where an Error is a specific io.Error kind +@(require_results) is_io_error :: proc(err: Error, io_err: io.Error) -> bool { if v, ok := err.(io.Error); ok { return v == io_err @@ -140,10 +145,10 @@ is_io_error :: proc(err: Error, io_err: io.Error) -> bool { return false } - // read_all reads all the remaining records from r. // Each record is a slice of fields. // read_all is defined to read until an EOF, and does not treat, and does not treat EOF as an error +@(require_results) read_all :: proc(r: ^Reader, allocator := context.allocator) -> ([][]string, Error) { context.allocator = allocator records: [dynamic][]string @@ -153,13 +158,18 @@ read_all :: proc(r: ^Reader, allocator := context.allocator) -> ([][]string, Err return records[:], nil } if rerr != nil { - return nil, rerr + // allow for a partial read + if record != nil { + append(&records, record) + } + return records[:], rerr } append(&records, record) } } // read reads a single record (a slice of fields) from the provided input. +@(require_results) read_from_string :: proc(input: string, record_allocator := context.allocator, buffer_allocator := context.allocator) -> (record: []string, n: int, err: Error) { ir: strings.Reader strings.reader_init(&ir, input) @@ -175,6 +185,7 @@ read_from_string :: proc(input: string, record_allocator := context.allocator, b // read_all reads all the remaining records from the provided input. +@(require_results) read_all_from_string :: proc(input: string, records_allocator := context.allocator, buffer_allocator := context.allocator) -> ([][]string, Error) { ir: strings.Reader strings.reader_init(&ir, input) @@ -186,7 +197,7 @@ read_all_from_string :: proc(input: string, records_allocator := context.allocat return read_all(&r, records_allocator) } -@private +@(private, require_results) is_valid_delim :: proc(r: rune) -> bool { switch r { case 0, '"', '\r', '\n', utf8.RUNE_ERROR: @@ -195,8 +206,9 @@ is_valid_delim :: proc(r: rune) -> bool { return utf8.valid_rune(r) } -@private +@(private, require_results) _read_record :: proc(r: ^Reader, dst: ^[dynamic]string, allocator := context.allocator) -> ([]string, Error) { + @(require_results) read_line :: proc(r: ^Reader) -> ([]byte, io.Error) { if !r.multiline_fields { line, err := bufio.reader_read_slice(&r.r, '\n') @@ -266,6 +278,7 @@ _read_record :: proc(r: ^Reader, dst: ^[dynamic]string, allocator := context.all unreachable() } + @(require_results) length_newline :: proc(b: []byte) -> int { if len(b) > 0 && b[len(b)-1] == '\n' { return 1 @@ -273,6 +286,7 @@ _read_record :: proc(r: ^Reader, dst: ^[dynamic]string, allocator := context.all return 0 } + @(require_results) next_rune :: proc(b: []byte) -> rune { r, _ := utf8.decode_rune(b) return r diff --git a/core/encoding/csv/writer.odin b/core/encoding/csv/writer.odin index d519104f2..132fa0a51 100644 --- a/core/encoding/csv/writer.odin +++ b/core/encoding/csv/writer.odin @@ -1,4 +1,4 @@ -package csv +package encoding_csv import "core:io" import "core:strings" @@ -17,7 +17,10 @@ Writer :: struct { // writer_init initializes a Writer that writes to w writer_init :: proc(writer: ^Writer, w: io.Writer) { - writer.comma = ',' + switch writer.comma { + case '\x00', '\n', '\r', 0xfffd: + writer.comma = ',' + } writer.w = w } diff --git a/core/encoding/entity/entity.odin b/core/encoding/entity/entity.odin index ec640c69f..cee6230ef 100644 --- a/core/encoding/entity/entity.odin +++ b/core/encoding/entity/entity.odin @@ -1,4 +1,4 @@ -package unicode_entity +package encoding_unicode_entity /* A unicode entity encoder/decoder diff --git a/core/encoding/entity/generated.odin b/core/encoding/entity/generated.odin index 3d1c02513..d2acde20d 100644 --- a/core/encoding/entity/generated.odin +++ b/core/encoding/entity/generated.odin @@ -1,4 +1,4 @@ -package unicode_entity +package encoding_unicode_entity /* ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ DO NOT EDIT ------ GENERATED ------ diff --git a/core/encoding/hex/hex.odin b/core/encoding/hex/hex.odin index ef0bab1d0..dbffe216b 100644 --- a/core/encoding/hex/hex.odin +++ b/core/encoding/hex/hex.odin @@ -1,4 +1,4 @@ -package hex +package encoding_hex import "core:strings" diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 3d57316b3..f45cdb1f1 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -1,4 +1,4 @@ -package json +package encoding_json import "core:mem" import "core:math/bits" @@ -420,7 +420,7 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: data := rawptr(uintptr(v.data) + info.offsets[i]) the_value := any{data, id} - if is_omitempty(the_value) { + if omitempty && is_omitempty(the_value) { continue } diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index 8bcef1339..3973725dc 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -1,4 +1,4 @@ -package json +package encoding_json import "core:mem" import "core:unicode/utf8" diff --git a/core/encoding/json/tokenizer.odin b/core/encoding/json/tokenizer.odin index a406a73a5..5c20a2cc3 100644 --- a/core/encoding/json/tokenizer.odin +++ b/core/encoding/json/tokenizer.odin @@ -1,4 +1,4 @@ -package json +package encoding_json import "core:unicode/utf8" diff --git a/core/encoding/json/types.odin b/core/encoding/json/types.odin index 20c806236..73e183615 100644 --- a/core/encoding/json/types.odin +++ b/core/encoding/json/types.odin @@ -1,4 +1,4 @@ -package json +package encoding_json import "core:strings" diff --git a/core/encoding/json/unmarshal.odin b/core/encoding/json/unmarshal.odin index b2052e43c..eb59e7838 100644 --- a/core/encoding/json/unmarshal.odin +++ b/core/encoding/json/unmarshal.odin @@ -1,4 +1,4 @@ -package json +package encoding_json import "core:mem" import "core:math" @@ -348,7 +348,7 @@ json_name_from_tag_value :: proc(value: string) -> (json_name, extra: string) { json_name = value if comma_index := strings.index_byte(json_name, ','); comma_index >= 0 { json_name = json_name[:comma_index] - extra = json_name[comma_index:] + extra = value[1 + comma_index:] } return } diff --git a/core/encoding/json/validator.odin b/core/encoding/json/validator.odin index 961c2dc23..a6873319d 100644 --- a/core/encoding/json/validator.odin +++ b/core/encoding/json/validator.odin @@ -1,4 +1,4 @@ -package json +package encoding_json import "core:mem" diff --git a/core/encoding/varint/doc.odin b/core/encoding/varint/doc.odin index 5e4708a59..c0a09873c 100644 --- a/core/encoding/varint/doc.odin +++ b/core/encoding/varint/doc.odin @@ -25,4 +25,4 @@ ``` */ -package varint \ No newline at end of file +package encoding_varint \ No newline at end of file diff --git a/core/encoding/varint/leb128.odin b/core/encoding/varint/leb128.odin index 1cdbb81b0..ca6513f04 100644 --- a/core/encoding/varint/leb128.odin +++ b/core/encoding/varint/leb128.odin @@ -8,7 +8,7 @@ // package varint implements variable length integer encoding and decoding using // the LEB128 format as used by DWARF debug info, Android .dex and other file formats. -package varint +package encoding_varint // In theory we should use the bigint package. In practice, varints bigger than this indicate a corrupted file. // Instead we'll set limits on the values we'll encode/decode diff --git a/core/encoding/xml/debug_print.odin b/core/encoding/xml/debug_print.odin index 2607bec23..be958baaa 100644 --- a/core/encoding/xml/debug_print.odin +++ b/core/encoding/xml/debug_print.odin @@ -1,4 +1,4 @@ -package xml +package encoding_xml /* An XML 1.0 / 1.1 parser diff --git a/core/encoding/xml/helpers.odin b/core/encoding/xml/helpers.odin index 42a5258b3..a9d4ad493 100644 --- a/core/encoding/xml/helpers.odin +++ b/core/encoding/xml/helpers.odin @@ -1,4 +1,4 @@ -package xml +package encoding_xml /* An XML 1.0 / 1.1 parser diff --git a/core/encoding/xml/tokenizer.odin b/core/encoding/xml/tokenizer.odin index a223a75d6..0f87c366b 100644 --- a/core/encoding/xml/tokenizer.odin +++ b/core/encoding/xml/tokenizer.odin @@ -1,4 +1,4 @@ -package xml +package encoding_xml /* An XML 1.0 / 1.1 parser diff --git a/core/encoding/xml/xml_reader.odin b/core/encoding/xml/xml_reader.odin index bf8646bc3..5b4b12948 100644 --- a/core/encoding/xml/xml_reader.odin +++ b/core/encoding/xml/xml_reader.odin @@ -24,7 +24,7 @@ MAYBE: List of contributors: - Jeroen van Rijn: Initial implementation. */ -package xml +package encoding_xml // An XML 1.0 / 1.1 parser import "core:bytes" diff --git a/core/fmt/fmt.odin b/core/fmt/fmt.odin index d3b9d7d69..867257491 100644 --- a/core/fmt/fmt.odin +++ b/core/fmt/fmt.odin @@ -120,11 +120,11 @@ register_user_formatter :: proc(id: typeid, formatter: User_Formatter) -> Regist // // Returns: A formatted string. // +@(require_results) aprint :: proc(args: ..any, sep := " ", allocator := context.allocator) -> string { str: strings.Builder strings.builder_init(&str, allocator) - sbprint(&str, ..args, sep=sep) - return strings.to_string(str) + return sbprint(&str, ..args, sep=sep) } // Creates a formatted string with a newline character at the end // @@ -136,11 +136,11 @@ aprint :: proc(args: ..any, sep := " ", allocator := context.allocator) -> strin // // Returns: A formatted string with a newline character at the end. // +@(require_results) aprintln :: proc(args: ..any, sep := " ", allocator := context.allocator) -> string { str: strings.Builder strings.builder_init(&str, allocator) - sbprintln(&str, ..args, sep=sep) - return strings.to_string(str) + return sbprintln(&str, ..args, sep=sep) } // Creates a formatted string using a format string and arguments // @@ -153,11 +153,11 @@ aprintln :: proc(args: ..any, sep := " ", allocator := context.allocator) -> str // // Returns: A formatted string. The returned string must be freed accordingly. // +@(require_results) aprintf :: proc(fmt: string, args: ..any, allocator := context.allocator, newline := false) -> string { str: strings.Builder strings.builder_init(&str, allocator) - sbprintf(&str, fmt, ..args, newline=newline) - return strings.to_string(str) + return sbprintf(&str, fmt, ..args, newline=newline) } // Creates a formatted string using a format string and arguments, followed by a newline. // @@ -169,6 +169,7 @@ aprintf :: proc(fmt: string, args: ..any, allocator := context.allocator, newlin // // Returns: A formatted string. The returned string must be freed accordingly. // +@(require_results) aprintfln :: proc(fmt: string, args: ..any, allocator := context.allocator) -> string { return aprintf(fmt, ..args, allocator=allocator, newline=true) } @@ -182,11 +183,11 @@ aprintfln :: proc(fmt: string, args: ..any, allocator := context.allocator) -> s // // Returns: A formatted string. // +@(require_results) tprint :: proc(args: ..any, sep := " ") -> string { str: strings.Builder strings.builder_init(&str, context.temp_allocator) - sbprint(&str, ..args, sep=sep) - return strings.to_string(str) + return sbprint(&str, ..args, sep=sep) } // Creates a formatted string with a newline character at the end // @@ -198,11 +199,11 @@ tprint :: proc(args: ..any, sep := " ") -> string { // // Returns: A formatted string with a newline character at the end. // +@(require_results) tprintln :: proc(args: ..any, sep := " ") -> string { str: strings.Builder strings.builder_init(&str, context.temp_allocator) - sbprintln(&str, ..args, sep=sep) - return strings.to_string(str) + return sbprintln(&str, ..args, sep=sep) } // Creates a formatted string using a format string and arguments // @@ -215,11 +216,11 @@ tprintln :: proc(args: ..any, sep := " ") -> string { // // Returns: A formatted string. // +@(require_results) tprintf :: proc(fmt: string, args: ..any, newline := false) -> string { str: strings.Builder strings.builder_init(&str, context.temp_allocator) - sbprintf(&str, fmt, ..args, newline=newline) - return strings.to_string(str) + return sbprintf(&str, fmt, ..args, newline=newline) } // Creates a formatted string using a format string and arguments, followed by a newline. // @@ -231,6 +232,7 @@ tprintf :: proc(fmt: string, args: ..any, newline := false) -> string { // // Returns: A formatted string. // +@(require_results) tprintfln :: proc(fmt: string, args: ..any) -> string { return tprintf(fmt, ..args, newline=true) } @@ -339,6 +341,7 @@ panicf :: proc(fmt: string, args: ..any, loc := #caller_location) -> ! { // // Returns: A formatted C string // +@(require_results) caprintf :: proc(format: string, args: ..any, newline := false) -> cstring { str: strings.Builder strings.builder_init(&str) @@ -357,6 +360,7 @@ caprintf :: proc(format: string, args: ..any, newline := false) -> cstring { // // Returns: A formatted C string // +@(require_results) caprintfln :: proc(format: string, args: ..any) -> cstring { return caprintf(format, ..args, newline=true) } @@ -371,6 +375,7 @@ caprintfln :: proc(format: string, args: ..any) -> cstring { // // Returns: A formatted C string // +@(require_results) ctprintf :: proc(format: string, args: ..any, newline := false) -> cstring { str: strings.Builder strings.builder_init(&str, context.temp_allocator) @@ -389,6 +394,7 @@ ctprintf :: proc(format: string, args: ..any, newline := false) -> cstring { // // Returns: A formatted C string // +@(require_results) ctprintfln :: proc(format: string, args: ..any) -> cstring { return ctprintf(format, ..args, newline=true) } @@ -1900,7 +1906,7 @@ fmt_struct :: proc(fi: ^Info, v: any, the_verb: rune, info: runtime.Type_Info_St // fi.hash = false; fi.indent += 1 - if hash { + if !is_soa && hash { io.write_byte(fi.writer, '\n', &fi.n) } defer { @@ -1934,6 +1940,9 @@ fmt_struct :: proc(fi: ^Info, v: any, the_verb: rune, info: runtime.Type_Info_St n = uintptr((^int)(uintptr(v.data) + info.offsets[actual_field_count])^) } + if hash && n > 0 { + io.write_byte(fi.writer, '\n', &fi.n) + } for index in 0.. 0 { io.write_string(fi.writer, ", ", &fi.n) } @@ -1942,9 +1951,23 @@ fmt_struct :: proc(fi: ^Info, v: any, the_verb: rune, info: runtime.Type_Info_St if !hash && field_count > 0 { io.write_string(fi.writer, ", ", &fi.n) } + if hash { + fi.indent -= 1 + fmt_write_indent(fi) + fi.indent += 1 + } io.write_string(fi.writer, base_type_name, &fi.n) io.write_byte(fi.writer, '{', &fi.n) - defer io.write_byte(fi.writer, '}', &fi.n) + if hash { io.write_byte(fi.writer, '\n', &fi.n) } + defer { + if hash { + fi.indent -= 1 + fmt_write_indent(fi) + fi.indent += 1 + } + io.write_byte(fi.writer, '}', &fi.n) + if hash { io.write_string(fi.writer, ",\n", &fi.n) } + } fi.record_level += 1 defer fi.record_level -= 1 @@ -2156,14 +2179,18 @@ fmt_named :: proc(fi: ^Info, v: any, verb: rune, info: runtime.Type_Info_Named) when ODIN_ERROR_POS_STYLE == .Default { io.write_byte(fi.writer, '(', &fi.n) io.write_int(fi.writer, int(a.line), 10, &fi.n) - io.write_byte(fi.writer, ':', &fi.n) - io.write_int(fi.writer, int(a.column), 10, &fi.n) + if a.column != 0 { + io.write_byte(fi.writer, ':', &fi.n) + io.write_int(fi.writer, int(a.column), 10, &fi.n) + } io.write_byte(fi.writer, ')', &fi.n) } else when ODIN_ERROR_POS_STYLE == .Unix { io.write_byte(fi.writer, ':', &fi.n) io.write_int(fi.writer, int(a.line), 10, &fi.n) - io.write_byte(fi.writer, ':', &fi.n) - io.write_int(fi.writer, int(a.column), 10, &fi.n) + if a.column != 0 { + io.write_byte(fi.writer, ':', &fi.n) + io.write_int(fi.writer, int(a.column), 10, &fi.n) + } io.write_byte(fi.writer, ':', &fi.n) } else { #panic("Unhandled ODIN_ERROR_POS_STYLE") @@ -2505,8 +2532,11 @@ fmt_bit_field :: proc(fi: ^Info, v: any, verb: rune, info: runtime.Type_Info_Bit bit_offset := info.bit_offsets[i] bit_size := info.bit_sizes[i] - value := read_bits(([^]byte)(v.data), bit_offset, bit_size) type := info.types[i] + value := read_bits(([^]byte)(v.data), bit_offset, bit_size) + if reflect.is_endian_big(type) { + value <<= u64(8*type.size) - u64(bit_size) + } if !reflect.is_unsigned(runtime.type_info_core(type)) { // Sign Extension diff --git a/core/fmt/fmt_js.odin b/core/fmt/fmt_js.odin index c70b7c1c0..acf218eb5 100644 --- a/core/fmt/fmt_js.odin +++ b/core/fmt/fmt_js.odin @@ -1,7 +1,9 @@ //+build js package fmt +import "core:bufio" import "core:io" +import "core:os" foreign import "odin_env" @@ -31,12 +33,63 @@ stderr := io.Writer{ data = rawptr(uintptr(2)), } +@(private="file") +fd_to_writer :: proc(fd: os.Handle, loc := #caller_location) -> io.Writer { + switch fd { + case 1: return stdout + case 2: return stderr + case: panic("`fmt.fprint` variant called with invalid file descriptor for JS, only 1 (stdout) and 2 (stderr) are supported", loc) + } +} + +// fprint formats using the default print settings and writes to fd +fprint :: proc(fd: os.Handle, args: ..any, sep := " ", flush := true, loc := #caller_location) -> int { + buf: [1024]byte + b: bufio.Writer + defer bufio.writer_flush(&b) + + bufio.writer_init_with_buf(&b, fd_to_writer(fd, loc), buf[:]) + w := bufio.writer_to_writer(&b) + return wprint(w, ..args, sep=sep, flush=flush) +} + +// fprintln formats using the default print settings and writes to fd +fprintln :: proc(fd: os.Handle, args: ..any, sep := " ", flush := true, loc := #caller_location) -> int { + buf: [1024]byte + b: bufio.Writer + defer bufio.writer_flush(&b) + + bufio.writer_init_with_buf(&b, fd_to_writer(fd, loc), buf[:]) + + w := bufio.writer_to_writer(&b) + return wprintln(w, ..args, sep=sep, flush=flush) +} + +// fprintf formats according to the specified format string and writes to fd +fprintf :: proc(fd: os.Handle, fmt: string, args: ..any, flush := true, newline := false, loc := #caller_location) -> int { + buf: [1024]byte + b: bufio.Writer + defer bufio.writer_flush(&b) + + bufio.writer_init_with_buf(&b, fd_to_writer(fd, loc), buf[:]) + + w := bufio.writer_to_writer(&b) + return wprintf(w, fmt, ..args, flush=flush, newline=newline) +} + +// fprintfln formats according to the specified format string and writes to fd, followed by a newline. +fprintfln :: proc(fd: os.Handle, fmt: string, args: ..any, flush := true, loc := #caller_location) -> int { + return fprintf(fd, fmt, ..args, flush=flush, newline=true, loc=loc) +} + // print formats using the default print settings and writes to stdout print :: proc(args: ..any, sep := " ", flush := true) -> int { return wprint(w=stdout, args=args, sep=sep, flush=flush) } // println formats using the default print settings and writes to stdout println :: proc(args: ..any, sep := " ", flush := true) -> int { return wprintln(w=stdout, args=args, sep=sep, flush=flush) } // printf formats according to the specififed format string and writes to stdout printf :: proc(fmt: string, args: ..any, flush := true) -> int { return wprintf(stdout, fmt, ..args, flush=flush) } +// printfln formats according to the specified format string and writes to stdout, followed by a newline. +printfln :: proc(fmt: string, args: ..any, flush := true) -> int { return wprintf(stdout, fmt, ..args, flush=flush, newline=true) } // eprint formats using the default print settings and writes to stderr eprint :: proc(args: ..any, sep := " ", flush := true) -> int { return wprint(w=stderr, args=args, sep=sep, flush=flush) } @@ -44,3 +97,5 @@ eprint :: proc(args: ..any, sep := " ", flush := true) -> int { return wprint( eprintln :: proc(args: ..any, sep := " ", flush := true) -> int { return wprintln(w=stderr, args=args, sep=sep, flush=flush) } // eprintf formats according to the specififed format string and writes to stderr eprintf :: proc(fmt: string, args: ..any, flush := true) -> int { return wprintf(stderr, fmt, ..args, flush=flush) } +// eprintfln formats according to the specified format string and writes to stderr, followed by a newline. +eprintfln :: proc(fmt: string, args: ..any, flush := true) -> int { return wprintf(stdout, fmt, ..args, flush=flush, newline=true) } diff --git a/core/image/netpbm/doc.odin b/core/image/netpbm/doc.odin index 1b5b46856..7106e023e 100644 --- a/core/image/netpbm/doc.odin +++ b/core/image/netpbm/doc.odin @@ -1,5 +1,6 @@ /* Formats: + PBM (P1, P4): Portable Bit Map, stores black and white images (1 channel) PGM (P2, P5): Portable Gray Map, stores greyscale images (1 channel, 1 or 2 bytes per value) PPM (P3, P6): Portable Pixel Map, stores colour images (3 channel, 1 or 2 bytes per value) @@ -7,27 +8,29 @@ Formats: PFM (Pf, PF): Portable Float Map, stores floating-point images (Pf: 1 channel, PF: 3 channel) Reading: - All formats fill out header fields `format`, `width`, `height`, `channels`, `depth` - Specific formats use more fields - PGM, PPM, and PAM set `maxval` (maximum of 65535) - PAM sets `tupltype` if there is one, and can set `channels` to any value (not just 1 or 3) - PFM sets `scale` (float equivalent of `maxval`) and `little_endian` (endianness of stored floats) - Currently doesn't support reading multiple images from one binary-format file + +- All formats fill out header fields `format`, `width`, `height`, `channels`, `depth`. +- Specific formats use more fields: + PGM, PPM, and PAM set `maxval` (maximum of 65535) + PAM sets `tupltype` if there is one, and can set `channels` to any value (not just 1 or 3) + PFM sets `scale` (float equivalent of `maxval`) and `little_endian` (endianness of stored floats) +- Currently doesn't support reading multiple images from one binary-format file. Writing: - You can use your own `Netpbm_Info` struct to control how images are written - All formats require the header field `format` to be specified - Additional header fields are required for specific formats - PGM, PPM, and PAM require `maxval` (maximum of 65535) - PAM also uses `tupltype`, though it may be left as default (empty or nil string) - PFM requires `scale`, and optionally `little_endian` + +- You can use your own `Netpbm_Info` struct to control how images are written. +- All formats require the header field `format` to be specified. +- Additional header fields are required for specific formats: + PGM, PPM, and PAM require `maxval` (maximum of 65535) + PAM also uses `tupltype`, though it may be left as default (empty or nil string) + PFM requires `scale`, and optionally `little_endian` Some syntax differences from the specifications: - `channels` stores the number of values per pixel, what the PAM specification calls `depth` - `depth` instead is the number of bits for a single value (32 for PFM, 16 or 8 otherwise) - `scale` and `little_endian` are separated, so the `header` will always store a positive `scale` - `little_endian` will only be true for a negative `scale` PFM, every other format will be false - `little_endian` only describes the netpbm data being read/written, the image buffer will be native -*/ +- `channels` stores the number of values per pixel, what the PAM specification calls `depth` +- `depth` instead is the number of bits for a single value (32 for PFM, 16 or 8 otherwise) +- `scale` and `little_endian` are separated, so the `header` will always store a positive `scale` +- `little_endian` will only be true for a negative `scale` PFM, every other format will be false +- `little_endian` only describes the netpbm data being read/written, the image buffer will be native +*/ package netpbm diff --git a/core/io/io.odin b/core/io/io.odin index ea8e240b0..961dbe43e 100644 --- a/core/io/io.odin +++ b/core/io/io.odin @@ -29,7 +29,7 @@ Error :: enum i32 { // Invalid_Write means that a write returned an impossible count Invalid_Write, - // Short_Buffer means that a read required a longer buffer than was provided + // Short_Buffer means that a read/write required a longer buffer than was provided Short_Buffer, // No_Progress is returned by some implementations of `io.Reader` when many calls @@ -359,6 +359,29 @@ read_at_least :: proc(r: Reader, buf: []byte, min: int) -> (n: int, err: Error) return } +// write_full writes until the entire contents of `buf` has been written or an error occurs. +write_full :: proc(w: Writer, buf: []byte) -> (n: int, err: Error) { + return write_at_least(w, buf, len(buf)) +} + +// write_at_least writes at least `buf[:min]` to the writer and returns the amount written. +// If an error occurs before writing everything it is returned. +write_at_least :: proc(w: Writer, buf: []byte, min: int) -> (n: int, err: Error) { + if len(buf) < min { + return 0, .Short_Buffer + } + for n < min && err == nil { + nn: int + nn, err = write(w, buf[n:]) + n += nn + } + + if err == nil && n < min { + err = .Short_Write + } + return +} + // copy copies from src to dst till either EOF is reached on src or an error occurs // It returns the number of bytes copied and the first error that occurred whilst copying, if any. copy :: proc(dst: Writer, src: Reader) -> (written: i64, err: Error) { diff --git a/core/math/big/prime.odin b/core/math/big/prime.odin index b02b7cb4e..5e7c02f37 100644 --- a/core/math/big/prime.odin +++ b/core/math/big/prime.odin @@ -1247,6 +1247,20 @@ internal_random_prime :: proc(a: ^Int, size_in_bits: int, trials: int, flags := a.digit[0] |= 3 } if .Second_MSB_On in flags { + /* + Ensure there's enough space for the bit to be set. + */ + if a.used * _DIGIT_BITS < size_in_bits - 1 { + new_size := (size_in_bits - 1) / _DIGIT_BITS + + if new_size % _DIGIT_BITS > 0 { + new_size += 1 + } + + internal_grow(a, new_size) or_return + a.used = new_size + } + internal_int_bitfield_set_single(a, size_in_bits - 2) or_return } diff --git a/core/math/linalg/glsl/linalg_glsl.odin b/core/math/linalg/glsl/linalg_glsl.odin index bda1f1723..363a95887 100644 --- a/core/math/linalg/glsl/linalg_glsl.odin +++ b/core/math/linalg/glsl/linalg_glsl.odin @@ -1724,7 +1724,7 @@ quatFromMat4 :: proc "c" (m: mat4) -> (q: quat) { @(require_results) quatMulVec3 :: proc "c" (q: quat, v: vec3) -> vec3 { xyz := vec3{q.x, q.y, q.z} - t := cross(xyz, v) + t := cross(2.0 * xyz, v) return v + q.w*t + cross(xyz, t) } @@ -1832,7 +1832,7 @@ dquatFromDmat4 :: proc "c" (m: dmat4) -> (q: dquat) { @(require_results) dquatMulDvec3 :: proc "c" (q: dquat, v: dvec3) -> dvec3 { xyz := dvec3{q.x, q.y, q.z} - t := cross(xyz, v) + t := cross(2.0 * xyz, v) return v + q.w*t + cross(xyz, t) } diff --git a/core/math/math.odin b/core/math/math.odin index 570c2d255..8d85c2381 100644 --- a/core/math/math.odin +++ b/core/math/math.odin @@ -60,6 +60,7 @@ sqrt :: proc{ @(require_results) sin_f32be :: proc "contextless" (θ: f32be) -> f32be { return #force_inline f32be(sin_f32(f32(θ))) } @(require_results) sin_f64le :: proc "contextless" (θ: f64le) -> f64le { return #force_inline f64le(sin_f64(f64(θ))) } @(require_results) sin_f64be :: proc "contextless" (θ: f64be) -> f64be { return #force_inline f64be(sin_f64(f64(θ))) } +// Return the sine of θ in radians. sin :: proc{ sin_f16, sin_f16le, sin_f16be, sin_f32, sin_f32le, sin_f32be, @@ -72,6 +73,7 @@ sin :: proc{ @(require_results) cos_f32be :: proc "contextless" (θ: f32be) -> f32be { return #force_inline f32be(cos_f32(f32(θ))) } @(require_results) cos_f64le :: proc "contextless" (θ: f64le) -> f64le { return #force_inline f64le(cos_f64(f64(θ))) } @(require_results) cos_f64be :: proc "contextless" (θ: f64be) -> f64be { return #force_inline f64be(cos_f64(f64(θ))) } +// Return the cosine of θ in radians. cos :: proc{ cos_f16, cos_f16le, cos_f16be, cos_f32, cos_f32le, cos_f32be, @@ -378,6 +380,7 @@ log10 :: proc{ @(require_results) tan_f64 :: proc "contextless" (θ: f64) -> f64 { return sin(θ)/cos(θ) } @(require_results) tan_f64le :: proc "contextless" (θ: f64le) -> f64le { return f64le(tan_f64(f64(θ))) } @(require_results) tan_f64be :: proc "contextless" (θ: f64be) -> f64be { return f64be(tan_f64(f64(θ))) } +// Return the tangent of θ in radians. tan :: proc{ tan_f16, tan_f16le, tan_f16be, tan_f32, tan_f32le, tan_f32be, @@ -1752,7 +1755,28 @@ atan2_f64be :: proc "contextless" (y, x: f64be) -> f64be { // TODO(bill): Better atan2_f32 return f64be(atan2_f64(f64(y), f64(x))) } +/* + Return the arc tangent of y/x in radians. Defined on the domain [-∞, ∞] for x and y with a range of [-π, π] + Special cases: + atan2(y, NaN) = NaN + atan2(NaN, x) = NaN + atan2(+0, x>=0) = + 0 + atan2(-0, x>=0) = - 0 + atan2(+0, x<=-0) = + π + atan2(-0, x<=-0) = - π + atan2(y>0, 0) = + π/2 + atan2(y<0, 0) = - π/2 + atan2(+∞, +∞) = + π/4 + atan2(-∞, +∞) = - π/4 + atan2(+∞, -∞) = 3π/4 + atan2(-∞, -∞) = - 3π/4 + atan2(y, +∞) = 0 + atan2(y>0, -∞) = + π + atan2(y<0, -∞) = - π + atan2(+∞, x) = + π/2 + atan2(-∞, x) = - π/2 +*/ atan2 :: proc{ atan2_f64, atan2_f32, atan2_f16, atan2_f64le, atan2_f64be, @@ -1760,6 +1784,7 @@ atan2 :: proc{ atan2_f16le, atan2_f16be, } +// Return the arc tangent of x, in radians. Defined on the domain of [-∞, ∞] with a range of [-π/2, π/2] @(require_results) atan :: proc "contextless" (x: $T) -> T where intrinsics.type_is_float(T) { return atan2(x, 1) @@ -1871,6 +1896,7 @@ asin_f16le :: proc "contextless" (x: f16le) -> f16le { asin_f16be :: proc "contextless" (x: f16be) -> f16be { return f16be(asin_f64(f64(x))) } +// Return the arc sine of x, in radians. Defined on the domain of [-1, 1] with a range of [-π/2, π/2] asin :: proc{ asin_f64, asin_f32, asin_f16, asin_f64le, asin_f64be, @@ -1985,6 +2011,7 @@ acos_f16le :: proc "contextless" (x: f16le) -> f16le { acos_f16be :: proc "contextless" (x: f16be) -> f16be { return f16be(acos_f64(f64(x))) } +// Return the arc cosine of x, in radians. Defined on the domain of [-1, 1] with a range of [0, π]. acos :: proc{ acos_f64, acos_f32, acos_f16, acos_f64le, acos_f64be, diff --git a/core/math/rand/rand.odin b/core/math/rand/rand.odin index 560dc8379..664d6abc9 100644 --- a/core/math/rand/rand.odin +++ b/core/math/rand/rand.odin @@ -5,6 +5,7 @@ Package core:math/rand implements various random number generators package rand import "base:intrinsics" +import "core:crypto" import "core:math" import "core:mem" @@ -104,27 +105,30 @@ init :: proc(r: ^Rand, seed: u64) { } /* -Initialises a random number generator to use the system random number generator. -The system random number generator is platform specific. -On `linux` refer to the `getrandom` syscall. -On `darwin` refer to `getentropy`. -On `windows` refer to `BCryptGenRandom`. - -All other platforms are not supported +Initialises a random number generator to use the system random number generator. +The system random number generator is platform specific, and not supported +on all targets. Inputs: - r: The random number generator to use the system random number generator -WARNING: Panics if the system is not either `windows`, `darwin` or `linux` +WARNING: Panics if the system random number generator is not supported. +Support can be determined via the `core:crypto.HAS_RAND_BYTES` constant. Example: + import "core:crypto" import "core:math/rand" import "core:fmt" init_as_system_example :: proc() { my_rand: rand.Rand - rand.init_as_system(&my_rand) - fmt.println(rand.uint64(&my_rand)) + switch crypto.HAS_RAND_BYTES { + case true: + rand.init_as_system(&my_rand) + fmt.println(rand.uint64(&my_rand)) + case false: + fmt.println("system random not supported!") + } } Possible Output: @@ -133,7 +137,7 @@ Possible Output: */ init_as_system :: proc(r: ^Rand) { - if !#defined(_system_random) { + if !crypto.HAS_RAND_BYTES { panic(#procedure + " is not supported on this platform yet") } r.state = 0 @@ -144,15 +148,14 @@ init_as_system :: proc(r: ^Rand) { @(private) _random_u64 :: proc(r: ^Rand) -> u64 { r := r - if r == nil { + switch { + case r == nil: r = &global_rand + case r.is_system: + value: u64 + crypto.rand_bytes((cast([^]u8)&value)[:size_of(u64)]) + return value } - when #defined(_system_random) { - if r.is_system { - return _system_random() - } - } - old_state := r.state r.state = old_state * 6364136223846793005 + (r.inc|1) @@ -789,8 +792,8 @@ shuffle :: proc(array: $T/[]$E, r: ^Rand = nil) { return } - for i := i64(0); i < n; i += 1 { - j := int63_max(n, r) + for i := i64(n - 1); i > 0; i -= 1 { + j := int63_max(i + 1, r) array[i], array[j] = array[j], array[i] } } diff --git a/core/math/rand/system_darwin.odin b/core/math/rand/system_darwin.odin deleted file mode 100644 index 756f7fcae..000000000 --- a/core/math/rand/system_darwin.odin +++ /dev/null @@ -1,22 +0,0 @@ -package rand - -import "core:sys/darwin" - -@(require_results) -_system_random :: proc() -> u64 { - for { - value: u64 - ret := darwin.syscall_getentropy(([^]u8)(&value), size_of(value)) - if ret < 0 { - switch ret { - case -4: // EINTR - continue - case -78: // ENOSYS - panic("getentropy not available in kernel") - case: - panic("getentropy failed") - } - } - return value - } -} \ No newline at end of file diff --git a/core/math/rand/system_js.odin b/core/math/rand/system_js.odin deleted file mode 100644 index b9b71c4a6..000000000 --- a/core/math/rand/system_js.odin +++ /dev/null @@ -1,14 +0,0 @@ -package rand - -foreign import "odin_env" -foreign odin_env { - @(link_name = "rand_bytes") - env_rand_bytes :: proc "contextless" (buf: []byte) --- -} - -@(require_results) -_system_random :: proc() -> u64 { - buf: [8]u8 - env_rand_bytes(buf[:]) - return transmute(u64)buf -} diff --git a/core/math/rand/system_linux.odin b/core/math/rand/system_linux.odin deleted file mode 100644 index 42c9f86fa..000000000 --- a/core/math/rand/system_linux.odin +++ /dev/null @@ -1,29 +0,0 @@ -package rand - -import "core:sys/linux" - -@(require_results) -_system_random :: proc() -> u64 { - for { - value: u64 - value_buf := (cast([^]u8)&value)[:size_of(u64)] - _, errno := linux.getrandom(value_buf, {}) - #partial switch errno { - case .NONE: - // Do nothing - case .EINTR: - // Call interupted by a signal handler, just retry the request. - continue - case .ENOSYS: - // The kernel is apparently prehistoric (< 3.17 circa 2014) - // and does not support getrandom. - panic("getrandom not available in kernel") - case: - // All other failures are things that should NEVER happen - // unless the kernel interface changes (ie: the Linux - // developers break userland). - panic("getrandom failed") - } - return value - } -} \ No newline at end of file diff --git a/core/math/rand/system_windows.odin b/core/math/rand/system_windows.odin deleted file mode 100644 index c6d68816d..000000000 --- a/core/math/rand/system_windows.odin +++ /dev/null @@ -1,13 +0,0 @@ -package rand - -import win32 "core:sys/windows" - -@(require_results) -_system_random :: proc() -> u64 { - value: u64 - status := win32.BCryptGenRandom(nil, ([^]u8)(&value), size_of(value), win32.BCRYPT_USE_SYSTEM_PREFERRED_RNG) - if status < 0 { - panic("BCryptGenRandom failed") - } - return value -} \ No newline at end of file diff --git a/core/mem/allocators.odin b/core/mem/allocators.odin index eba79eacf..1d79e09c1 100644 --- a/core/mem/allocators.odin +++ b/core/mem/allocators.odin @@ -1124,7 +1124,7 @@ buddy_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, case .Query_Info: info := (^Allocator_Query_Info)(old_memory) if info != nil && info.pointer != nil { - ptr := old_memory + ptr := info.pointer if !(b.head <= ptr && ptr <= b.tail) { return nil, .Invalid_Pointer } diff --git a/core/mem/tracking_allocator.odin b/core/mem/tracking_allocator.odin index d6d189731..bc624617d 100644 --- a/core/mem/tracking_allocator.odin +++ b/core/mem/tracking_allocator.odin @@ -22,6 +22,13 @@ Tracking_Allocator :: struct { bad_free_array: [dynamic]Tracking_Allocator_Bad_Free_Entry, mutex: sync.Mutex, clear_on_free_all: bool, + + total_memory_allocated: i64, + total_allocation_count: i64, + total_memory_freed: i64, + total_free_count: i64, + peak_memory_allocated: i64, + current_memory_allocated: i64, } tracking_allocator_init :: proc(t: ^Tracking_Allocator, backing_allocator: Allocator, internals_allocator := context.allocator) { @@ -44,6 +51,7 @@ tracking_allocator_clear :: proc(t: ^Tracking_Allocator) { sync.mutex_lock(&t.mutex) clear(&t.allocation_map) clear(&t.bad_free_array) + t.current_memory_allocated = 0 sync.mutex_unlock(&t.mutex) } @@ -59,6 +67,21 @@ tracking_allocator :: proc(data: ^Tracking_Allocator) -> Allocator { tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, size, alignment: int, old_memory: rawptr, old_size: int, loc := #caller_location) -> (result: []byte, err: Allocator_Error) { + track_alloc :: proc(data: ^Tracking_Allocator, entry: ^Tracking_Allocator_Entry) { + data.total_memory_allocated += i64(entry.size) + data.total_allocation_count += 1 + data.current_memory_allocated += i64(entry.size) + if data.current_memory_allocated > data.peak_memory_allocated { + data.peak_memory_allocated = data.current_memory_allocated + } + } + + track_free :: proc(data: ^Tracking_Allocator, entry: ^Tracking_Allocator_Entry) { + data.total_memory_freed += i64(entry.size) + data.total_free_count += 1 + data.current_memory_allocated -= i64(entry.size) + } + data := (^Tracking_Allocator)(allocator_data) sync.mutex_guard(&data.mutex) @@ -100,13 +123,21 @@ tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, err = err, location = loc, } + track_alloc(data, &data.allocation_map[result_ptr]) case .Free: + if old_memory != nil && old_memory in data.allocation_map { + track_free(data, &data.allocation_map[old_memory]) + } delete_key(&data.allocation_map, old_memory) case .Free_All: if data.clear_on_free_all { clear_map(&data.allocation_map) + data.current_memory_allocated = 0 } case .Resize, .Resize_Non_Zeroed: + if old_memory != nil && old_memory in data.allocation_map { + track_free(data, &data.allocation_map[old_memory]) + } if old_memory != result_ptr { delete_key(&data.allocation_map, old_memory) } @@ -118,6 +149,7 @@ tracking_allocator_proc :: proc(allocator_data: rawptr, mode: Allocator_Mode, err = err, location = loc, } + track_alloc(data, &data.allocation_map[result_ptr]) case .Query_Features: set := (^Allocator_Mode_Set)(old_memory) diff --git a/core/net/socket_linux.odin b/core/net/socket_linux.odin index ba48959fb..a4d75b92b 100644 --- a/core/net/socket_linux.odin +++ b/core/net/socket_linux.odin @@ -258,8 +258,12 @@ _send_tcp :: proc(tcp_sock: TCP_Socket, buf: []byte) -> (int, Network_Error) { for total_written < len(buf) { limit := min(int(max(i32)), len(buf) - total_written) remaining := buf[total_written:][:limit] - res, errno := linux.send(linux.Fd(tcp_sock), remaining, {}) - if errno != .NONE { + res, errno := linux.send(linux.Fd(tcp_sock), remaining, {.NOSIGNAL}) + if errno == .EPIPE { + // If the peer is disconnected when we are trying to send we will get an `EPIPE` error, + // so we turn that into a clearer error + return total_written, TCP_Send_Error.Connection_Closed + } else if errno != .NONE { return total_written, TCP_Send_Error(errno) } total_written += int(res) diff --git a/core/net/url.odin b/core/net/url.odin index 7ad88bd1f..16aa57ec5 100644 --- a/core/net/url.odin +++ b/core/net/url.odin @@ -21,7 +21,7 @@ import "core:strconv" import "core:unicode/utf8" import "core:encoding/hex" -split_url :: proc(url: string, allocator := context.allocator) -> (scheme, host, path: string, queries: map[string]string) { +split_url :: proc(url: string, allocator := context.allocator) -> (scheme, host, path: string, queries: map[string]string, fragment: string) { s := url i := strings.index(s, "://") @@ -30,6 +30,12 @@ split_url :: proc(url: string, allocator := context.allocator) -> (scheme, host, s = s[i+3:] } + i = strings.index(s, "#") + if i != -1 { + fragment = s[i+1:] + s = s[:i] + } + i = strings.index(s, "?") if i != -1 { query_str := s[i+1:] @@ -62,7 +68,7 @@ split_url :: proc(url: string, allocator := context.allocator) -> (scheme, host, return } -join_url :: proc(scheme, host, path: string, queries: map[string]string, allocator := context.allocator) -> string { +join_url :: proc(scheme, host, path: string, queries: map[string]string, fragment: string, allocator := context.allocator) -> string { b := strings.builder_make(allocator) strings.builder_grow(&b, len(scheme) + 3 + len(host) + 1 + len(path)) @@ -95,6 +101,13 @@ join_url :: proc(scheme, host, path: string, queries: map[string]string, allocat i += 1 } + if fragment != "" { + if fragment[0] != '#' { + strings.write_string(&b, "#") + } + strings.write_string(&b, strings.trim_space(fragment)) + } + return strings.to_string(b) } diff --git a/core/odin/parser/parser.odin b/core/odin/parser/parser.odin index 9eaef4655..b2ffd3888 100644 --- a/core/odin/parser/parser.odin +++ b/core/odin/parser/parser.odin @@ -416,24 +416,28 @@ end_of_line_pos :: proc(p: ^Parser, tok: tokenizer.Token) -> tokenizer.Pos { } expect_closing_brace_of_field_list :: proc(p: ^Parser) -> tokenizer.Token { + return expect_closing_token_of_field_list(p, .Close_Brace, "field list") +} + +expect_closing_token_of_field_list :: proc(p: ^Parser, closing_kind: tokenizer.Token_Kind, msg: string) -> tokenizer.Token { token := p.curr_tok - if allow_token(p, .Close_Brace) { + if allow_token(p, closing_kind) { return token } if allow_token(p, .Semicolon) && !tokenizer.is_newline(token) { str := tokenizer.token_to_string(token) error(p, end_of_line_pos(p, p.prev_tok), "expected a comma, got %s", str) } - expect_brace := expect_token(p, .Close_Brace) + expect_closing := expect_token_after(p, closing_kind, msg) - if expect_brace.kind != .Close_Brace { - for p.curr_tok.kind != .Close_Brace && p.curr_tok.kind != .EOF && !is_non_inserted_semicolon(p.curr_tok) { + if expect_closing.kind != closing_kind { + for p.curr_tok.kind != closing_kind && p.curr_tok.kind != .EOF && !is_non_inserted_semicolon(p.curr_tok) { advance_token(p) } return p.curr_tok } - return expect_brace + return expect_closing } expect_closing_parentheses_of_field_list :: proc(p: ^Parser) -> tokenizer.Token { @@ -1354,6 +1358,7 @@ parse_stmt :: proc(p: ^Parser) -> ^ast.Stmt { rs := ast.new(ast.Return_Stmt, tok.pos, end) rs.results = results[:] + expect_semicolon(p, rs) return rs case .Break, .Continue, .Fallthrough: @@ -2990,8 +2995,8 @@ parse_literal_value :: proc(p: ^Parser, type: ^ast.Expr) -> ^ast.Comp_Lit { } p.expr_level -= 1 - skip_possible_newline(p) - close := expect_token_after(p, .Close_Brace, "compound literal") + skip_possible_newline(p) + close := expect_closing_brace_of_field_list(p) pos := type.pos if type != nil else open.pos lit := ast.new(ast.Comp_Lit, pos, end_pos(close)) @@ -3054,7 +3059,7 @@ parse_call_expr :: proc(p: ^Parser, operand: ^ast.Expr) -> ^ast.Expr { allow_token(p, .Comma) or_break } - close := expect_token_after(p, .Close_Paren, "argument list") + close := expect_closing_token_of_field_list(p, .Close_Paren, "argument list") p.expr_level -= 1 ce := ast.new(ast.Call_Expr, operand.pos, end_pos(close)) diff --git a/core/odin/tokenizer/tokenizer.odin b/core/odin/tokenizer/tokenizer.odin index 41de3ac8b..62170aa10 100644 --- a/core/odin/tokenizer/tokenizer.odin +++ b/core/odin/tokenizer/tokenizer.odin @@ -39,6 +39,7 @@ init :: proc(t: ^Tokenizer, src: string, path: string, err: Error_Handler = defa t.read_offset = 0 t.line_offset = 0 t.line_count = len(src) > 0 ? 1 : 0 + t.insert_semicolon = false t.error_count = 0 t.path = path diff --git a/core/os/os_js.odin b/core/os/os_js.odin index 910cb8155..8b61cb7ed 100644 --- a/core/os/os_js.odin +++ b/core/os/os_js.odin @@ -1,9 +1,7 @@ //+build js package os -import "base:intrinsics" import "base:runtime" -import "core:unicode/utf16" is_path_separator :: proc(c: byte) -> bool { return c == '/' || c == '\\' @@ -64,13 +62,8 @@ write_at :: proc(fd: Handle, data: []byte, offset: i64) -> (n: int, err: Errno) unimplemented("core:os procedure not supported on JS target") } - - -// NOTE(bill): Uses startup to initialize it -//stdin := get_std_handle(uint(win32.STD_INPUT_HANDLE)) -//stdout := get_std_handle(uint(win32.STD_OUTPUT_HANDLE)) -//stderr := get_std_handle(uint(win32.STD_ERROR_HANDLE)) - +stdout: Handle = 1 +stderr: Handle = 2 get_std_handle :: proc "contextless" (h: uint) -> Handle { context = runtime.default_context() diff --git a/core/os/stat.odin b/core/os/stat.odin index 1b64ad33b..21a4961d1 100644 --- a/core/os/stat.odin +++ b/core/os/stat.odin @@ -3,8 +3,8 @@ package os import "core:time" File_Info :: struct { - fullpath: string, - name: string, + fullpath: string, // allocated + name: string, // uses `fullpath` as underlying data size: i64, mode: File_Mode, is_dir: bool, diff --git a/core/reflect/reflect.odin b/core/reflect/reflect.odin index de5dec2e3..de7379ecc 100644 --- a/core/reflect/reflect.odin +++ b/core/reflect/reflect.odin @@ -934,6 +934,27 @@ set_union_value :: proc(dst: any, value: any) -> bool { panic("expected a union to reflect.set_union_variant_typeid") } +@(require_results) +bit_set_is_big_endian :: proc(value: any, loc := #caller_location) -> bool { + if value == nil { return ODIN_ENDIAN == .Big } + + ti := runtime.type_info_base(type_info_of(value.id)) + if info, ok := ti.variant.(runtime.Type_Info_Bit_Set); ok { + if info.underlying == nil { return ODIN_ENDIAN == .Big } + + underlying_ti := runtime.type_info_base(info.underlying) + if underlying_info, uok := underlying_ti.variant.(runtime.Type_Info_Integer); uok { + switch underlying_info.endianness { + case .Platform: return ODIN_ENDIAN == .Big + case .Little: return false + case .Big: return true + } + } + + return ODIN_ENDIAN == .Big + } + panic("expected a bit_set to reflect.bit_set_is_big_endian", loc) +} @(require_results) diff --git a/core/reflect/types.odin b/core/reflect/types.odin index 9cff46a00..f242dfd5c 100644 --- a/core/reflect/types.odin +++ b/core/reflect/types.odin @@ -408,7 +408,68 @@ is_relative_multi_pointer :: proc(info: ^Type_Info) -> bool { } +@(require_results) +is_endian_platform :: proc(info: ^Type_Info) -> bool { + if info == nil { return false} + info := info + info = type_info_core(info) + #partial switch v in info.variant { + case Type_Info_Integer: + return v.endianness == .Platform + case Type_Info_Bit_Set: + if v.underlying != nil { + return is_endian_platform(v.underlying) + } + return true + case Type_Info_Pointer: + return true + } + return false +} +@(require_results) +is_endian_little :: proc(info: ^Type_Info) -> bool { + if info == nil { return false} + info := info + info = type_info_core(info) + #partial switch v in info.variant { + case Type_Info_Integer: + if v.endianness == .Platform { + return ODIN_ENDIAN == .Little + } + return v.endianness == .Little + case Type_Info_Bit_Set: + if v.underlying != nil { + return is_endian_platform(v.underlying) + } + return ODIN_ENDIAN == .Little + case Type_Info_Pointer: + return ODIN_ENDIAN == .Little + } + return ODIN_ENDIAN == .Little +} + +@(require_results) +is_endian_big :: proc(info: ^Type_Info) -> bool { + if info == nil { return false} + info := info + info = type_info_core(info) + #partial switch v in info.variant { + case Type_Info_Integer: + if v.endianness == .Platform { + return ODIN_ENDIAN == .Big + } + return v.endianness == .Big + case Type_Info_Bit_Set: + if v.underlying != nil { + return is_endian_platform(v.underlying) + } + return ODIN_ENDIAN == .Big + case Type_Info_Pointer: + return ODIN_ENDIAN == .Big + } + return ODIN_ENDIAN == .Big +} diff --git a/core/sync/chan/chan.odin b/core/sync/chan/chan.odin index cbcfdf3bf..f4774e4f8 100644 --- a/core/sync/chan/chan.odin +++ b/core/sync/chan/chan.odin @@ -75,6 +75,7 @@ create_raw_unbuffered :: proc(#any_int msg_size, msg_alignment: int, allocator: ptr := mem.alloc(size, align, allocator) or_return c = (^Raw_Chan)(ptr) + c.allocator = allocator c.allocation_size = size c.unbuffered_data = ([^]byte)(ptr)[offset:] c.msg_size = u16(msg_size) @@ -99,6 +100,7 @@ create_raw_buffered :: proc(#any_int msg_size, msg_alignment: int, #any_int cap: ptr := mem.alloc(size, align, allocator) or_return c = (^Raw_Chan)(ptr) + c.allocator = allocator c.allocation_size = size bptr := ([^]byte)(ptr) diff --git a/core/sync/futex_netbsd.odin b/core/sync/futex_netbsd.odin index 4f52e952f..08f7cd6c9 100644 --- a/core/sync/futex_netbsd.odin +++ b/core/sync/futex_netbsd.odin @@ -1,165 +1,74 @@ //+private package sync -import "core:c" +import "base:intrinsics" import "core:time" +import "core:c" import "core:sys/unix" -@(private="file") -Wait_Node :: struct { - thread: unix.pthread_t, - futex: ^Futex, - prev, next: ^Wait_Node, +foreign import libc "system:c" + +FUTEX_PRIVATE_FLAG :: 128 + +FUTEX_WAIT_PRIVATE :: 0 | FUTEX_PRIVATE_FLAG +FUTEX_WAKE_PRIVATE :: 1 | FUTEX_PRIVATE_FLAG + +EINTR :: 4 /* Interrupted system call */ +EAGAIN :: 35 /* Resource temporarily unavailable */ +ETIMEDOUT :: 60 /* Operation timed out */ + +Time_Spec :: struct { + time_sec: uint, + time_nsec: uint, } -@(private="file") -atomic_flag :: distinct bool -@(private="file") -Wait_Queue :: struct { - lock: atomic_flag, - list: Wait_Node, -} -@(private="file") -waitq_lock :: proc "contextless" (waitq: ^Wait_Queue) { - for cast(bool)atomic_exchange_explicit(&waitq.lock, atomic_flag(true), .Acquire) { - cpu_relax() // spin... + +get_last_error :: proc "contextless" () -> int { + foreign libc { + __errno :: proc() -> ^c.int --- } -} -@(private="file") -waitq_unlock :: proc "contextless" (waitq: ^Wait_Queue) { - atomic_store_explicit(&waitq.lock, atomic_flag(false), .Release) + return int(__errno()^) } -// FIXME: This approach may scale badly in the future, -// possible solution - hash map (leads to deadlocks now). -@(private="file") -g_waitq: Wait_Queue - -@(init, private="file") -g_waitq_init :: proc() { - g_waitq = { - list = { - prev = &g_waitq.list, - next = &g_waitq.list, - }, +_futex_wait :: proc "contextless" (futex: ^Futex, expected: u32) -> bool { + if cast(int) intrinsics.syscall(unix.SYS___futex, uintptr(futex), FUTEX_WAIT_PRIVATE, uintptr(expected), 0, 0, 0) == -1 { + switch get_last_error() { + case EINTR, EAGAIN: + return true + case: + _panic("futex_wait failure") + } } + return true } -@(private="file") -get_waitq :: #force_inline proc "contextless" (f: ^Futex) -> ^Wait_Queue { - _ = f - return &g_waitq -} - -_futex_wait :: proc "contextless" (f: ^Futex, expect: u32) -> (ok: bool) { - waitq := get_waitq(f) - waitq_lock(waitq) - defer waitq_unlock(waitq) - - head := &waitq.list - waiter := Wait_Node{ - thread = unix.pthread_self(), - futex = f, - prev = head, - next = head.next, - } - - waiter.prev.next = &waiter - waiter.next.prev = &waiter - - old_mask, mask: unix.sigset_t - unix.sigemptyset(&mask) - unix.sigaddset(&mask, unix.SIGCONT) - unix.pthread_sigmask(unix.SIG_BLOCK, &mask, &old_mask) - - if u32(atomic_load_explicit(f, .Acquire)) == expect { - waitq_unlock(waitq) - defer waitq_lock(waitq) - - sig: c.int - unix.sigwait(&mask, &sig) - errno := unix.errno() - ok = errno == unix.ERROR_NONE - } - - waiter.prev.next = waiter.next - waiter.next.prev = waiter.prev - - unix.pthread_sigmask(unix.SIG_SETMASK, &old_mask, nil) - - // FIXME: Add error handling! - return -} - -_futex_wait_with_timeout :: proc "contextless" (f: ^Futex, expect: u32, duration: time.Duration) -> (ok: bool) { +_futex_wait_with_timeout :: proc "contextless" (futex: ^Futex, expected: u32, duration: time.Duration) -> bool { if duration <= 0 { return false } - waitq := get_waitq(f) - waitq_lock(waitq) - defer waitq_unlock(waitq) - - head := &waitq.list - waiter := Wait_Node{ - thread = unix.pthread_self(), - futex = f, - prev = head, - next = head.next, - } - - waiter.prev.next = &waiter - waiter.next.prev = &waiter - - old_mask, mask: unix.sigset_t - unix.sigemptyset(&mask) - unix.sigaddset(&mask, unix.SIGCONT) - unix.pthread_sigmask(unix.SIG_BLOCK, &mask, &old_mask) - - if u32(atomic_load_explicit(f, .Acquire)) == expect { - waitq_unlock(waitq) - defer waitq_lock(waitq) - - info: unix.siginfo_t - ts := unix.timespec{ - tv_sec = i64(duration / 1e9), - tv_nsec = i64(duration % 1e9), + if cast(int) intrinsics.syscall(unix.SYS___futex, uintptr(futex), FUTEX_WAIT_PRIVATE, uintptr(expected), cast(uintptr) &Time_Spec{ + time_sec = cast(uint)(duration / 1e9), + time_nsec = cast(uint)(duration % 1e9), + }, 0, 0) == -1 { + switch get_last_error() { + case EINTR, EAGAIN: + return true + case ETIMEDOUT: + return false + case: + _panic("futex_wait_with_timeout failure") } - unix.sigtimedwait(&mask, &info, &ts) - errno := unix.errno() - ok = errno == unix.EAGAIN || errno == unix.ERROR_NONE } - - waiter.prev.next = waiter.next - waiter.next.prev = waiter.prev - - unix.pthread_sigmask(unix.SIG_SETMASK, &old_mask, nil) - - // FIXME: Add error handling! - return + return true } -_futex_signal :: proc "contextless" (f: ^Futex) { - waitq := get_waitq(f) - waitq_lock(waitq) - defer waitq_unlock(waitq) - - head := &waitq.list - for waiter := head.next; waiter != head; waiter = waiter.next { - if waiter.futex == f { - unix.pthread_kill(waiter.thread, unix.SIGCONT) - break - } +_futex_signal :: proc "contextless" (futex: ^Futex) { + if cast(int) intrinsics.syscall(unix.SYS___futex, uintptr(futex), FUTEX_WAKE_PRIVATE, 1, 0, 0, 0) == -1 { + _panic("futex_wake_single failure") } } -_futex_broadcast :: proc "contextless" (f: ^Futex) { - waitq := get_waitq(f) - waitq_lock(waitq) - defer waitq_unlock(waitq) - - head := &waitq.list - for waiter := head.next; waiter != head; waiter = waiter.next { - if waiter.futex == f { - unix.pthread_kill(waiter.thread, unix.SIGCONT) - } +_futex_broadcast :: proc "contextless" (futex: ^Futex) { + if cast(int) intrinsics.syscall(unix.SYS___futex, uintptr(futex), FUTEX_WAKE_PRIVATE, uintptr(max(i32)), 0, 0, 0) == -1 { + _panic("_futex_wake_all failure") } } diff --git a/core/sys/darwin/CoreFoundation/CFBase.odin b/core/sys/darwin/CoreFoundation/CFBase.odin new file mode 100644 index 000000000..7335f087b --- /dev/null +++ b/core/sys/darwin/CoreFoundation/CFBase.odin @@ -0,0 +1,34 @@ +package CoreFoundation + +foreign import CoreFoundation "system:CoreFoundation.framework" + +TypeID :: distinct uint +OptionFlags :: distinct uint +HashCode :: distinct uint +Index :: distinct int +TypeRef :: distinct rawptr + +Range :: struct { + location: Index, + length: Index, +} + +foreign CoreFoundation { + // Releases a Core Foundation object. + CFRelease :: proc(cf: TypeRef) --- +} + +// Releases a Core Foundation object. +Release :: proc { + ReleaseObject, + ReleaseString, +} + +ReleaseObject :: #force_inline proc(cf: TypeRef) { + CFRelease(cf) +} + +// Releases a Core Foundation string. +ReleaseString :: #force_inline proc(theString: String) { + CFRelease(TypeRef(theString)) +} diff --git a/core/sys/darwin/CoreFoundation/CFString.odin b/core/sys/darwin/CoreFoundation/CFString.odin new file mode 100644 index 000000000..4a167c604 --- /dev/null +++ b/core/sys/darwin/CoreFoundation/CFString.odin @@ -0,0 +1,203 @@ +package CoreFoundation + +import "base:runtime" + +foreign import CoreFoundation "system:CoreFoundation.framework" + +String :: distinct TypeRef // same as CFStringRef + +StringEncoding :: distinct u32 + +StringBuiltInEncodings :: enum StringEncoding { + MacRoman = 0, + WindowsLatin1 = 0x0500, + ISOLatin1 = 0x0201, + NextStepLatin = 0x0B01, + ASCII = 0x0600, + Unicode = 0x0100, + UTF8 = 0x08000100, + NonLossyASCII = 0x0BFF, + + UTF16 = 0x0100, + UTF16BE = 0x10000100, + UTF16LE = 0x14000100, + + UTF32 = 0x0c000100, + UTF32BE = 0x18000100, + UTF32LE = 0x1c000100, +} + +StringEncodings :: enum Index { + MacJapanese = 1, + MacChineseTrad = 2, + MacKorean = 3, + MacArabic = 4, + MacHebrew = 5, + MacGreek = 6, + MacCyrillic = 7, + MacDevanagari = 9, + MacGurmukhi = 10, + MacGujarati = 11, + MacOriya = 12, + MacBengali = 13, + MacTamil = 14, + MacTelugu = 15, + MacKannada = 16, + MacMalayalam = 17, + MacSinhalese = 18, + MacBurmese = 19, + MacKhmer = 20, + MacThai = 21, + MacLaotian = 22, + MacGeorgian = 23, + MacArmenian = 24, + MacChineseSimp = 25, + MacTibetan = 26, + MacMongolian = 27, + MacEthiopic = 28, + MacCentralEurRoman = 29, + MacVietnamese = 30, + MacExtArabic = 31, + MacSymbol = 33, + MacDingbats = 34, + MacTurkish = 35, + MacCroatian = 36, + MacIcelandic = 37, + MacRomanian = 38, + MacCeltic = 39, + MacGaelic = 40, + MacFarsi = 0x8C, + MacUkrainian = 0x98, + MacInuit = 0xEC, + MacVT100 = 0xFC, + MacHFS = 0xFF, + ISOLatin2 = 0x0202, + ISOLatin3 = 0x0203, + ISOLatin4 = 0x0204, + ISOLatinCyrillic = 0x0205, + ISOLatinArabic = 0x0206, + ISOLatinGreek = 0x0207, + ISOLatinHebrew = 0x0208, + ISOLatin5 = 0x0209, + ISOLatin6 = 0x020A, + ISOLatinThai = 0x020B, + ISOLatin7 = 0x020D, + ISOLatin8 = 0x020E, + ISOLatin9 = 0x020F, + ISOLatin10 = 0x0210, + DOSLatinUS = 0x0400, + DOSGreek = 0x0405, + DOSBalticRim = 0x0406, + DOSLatin1 = 0x0410, + DOSGreek1 = 0x0411, + DOSLatin2 = 0x0412, + DOSCyrillic = 0x0413, + DOSTurkish = 0x0414, + DOSPortuguese = 0x0415, + DOSIcelandic = 0x0416, + DOSHebrew = 0x0417, + DOSCanadianFrench = 0x0418, + DOSArabic = 0x0419, + DOSNordic = 0x041A, + DOSRussian = 0x041B, + DOSGreek2 = 0x041C, + DOSThai = 0x041D, + DOSJapanese = 0x0420, + DOSChineseSimplif = 0x0421, + DOSKorean = 0x0422, + DOSChineseTrad = 0x0423, + WindowsLatin2 = 0x0501, + WindowsCyrillic = 0x0502, + WindowsGreek = 0x0503, + WindowsLatin5 = 0x0504, + WindowsHebrew = 0x0505, + WindowsArabic = 0x0506, + WindowsBalticRim = 0x0507, + WindowsVietnamese = 0x0508, + WindowsKoreanJohab = 0x0510, + ANSEL = 0x0601, + JIS_X0201_76 = 0x0620, + JIS_X0208_83 = 0x0621, + JIS_X0208_90 = 0x0622, + JIS_X0212_90 = 0x0623, + JIS_C6226_78 = 0x0624, + ShiftJIS_X0213 = 0x0628, + ShiftJIS_X0213_MenKuTen = 0x0629, + GB_2312_80 = 0x0630, + GBK_95 = 0x0631, + GB_18030_2000 = 0x0632, + KSC_5601_87 = 0x0640, + KSC_5601_92_Johab = 0x0641, + CNS_11643_92_P1 = 0x0651, + CNS_11643_92_P2 = 0x0652, + CNS_11643_92_P3 = 0x0653, + ISO_2022_JP = 0x0820, + ISO_2022_JP_2 = 0x0821, + ISO_2022_JP_1 = 0x0822, + ISO_2022_JP_3 = 0x0823, + ISO_2022_CN = 0x0830, + ISO_2022_CN_EXT = 0x0831, + ISO_2022_KR = 0x0840, + EUC_JP = 0x0920, + EUC_CN = 0x0930, + EUC_TW = 0x0931, + EUC_KR = 0x0940, + ShiftJIS = 0x0A01, + KOI8_R = 0x0A02, + Big5 = 0x0A03, + MacRomanLatin1 = 0x0A04, + HZ_GB_2312 = 0x0A05, + Big5_HKSCS_1999 = 0x0A06, + VISCII = 0x0A07, + KOI8_U = 0x0A08, + Big5_E = 0x0A09, + NextStepJapanese = 0x0B02, + EBCDIC_US = 0x0C01, + EBCDIC_CP037 = 0x0C02, + UTF7 = 0x04000100, + UTF7_IMAP = 0x0A10, + ShiftJIS_X0213_00 = 0x0628, // Deprecated. Use `ShiftJIS_X0213` instead. +} + +@(link_prefix = "CF", default_calling_convention = "c") +foreign CoreFoundation { + // Copies the character contents of a string to a local C string buffer after converting the characters to a given encoding. + StringGetCString :: proc(theString: String, buffer: [^]byte, bufferSize: Index, encoding: StringEncoding) -> b8 --- + + // Returns the number (in terms of UTF-16 code pairs) of Unicode characters in a string. + StringGetLength :: proc(theString: String) -> Index --- + + // Returns the maximum number of bytes a string of a specified length (in Unicode characters) will take up if encoded in a specified encoding. + StringGetMaximumSizeForEncoding :: proc(length: Index, encoding: StringEncoding) -> Index --- + + // Fetches a range of the characters from a string into a byte buffer after converting the characters to a specified encoding. + StringGetBytes :: proc(thestring: String, range: Range, encoding: StringEncoding, lossByte: u8, isExternalRepresentation: b8, buffer: [^]byte, maxBufLen: Index, usedBufLen: ^Index) -> Index --- + + StringIsEncodingAvailable :: proc(encoding: StringEncoding) -> bool --- + + @(link_name = "__CFStringMakeConstantString") + StringMakeConstantString :: proc "c" (#const c: cstring) -> String --- +} + +STR :: StringMakeConstantString + +StringCopyToOdinString :: proc( + theString: String, + allocator := context.allocator, +) -> ( + str: string, + ok: bool, +) #optional_ok { + length := StringGetLength(theString) + max := StringGetMaximumSizeForEncoding(length, StringEncoding(StringBuiltInEncodings.UTF8)) + + buf, err := make([]byte, max, allocator) + if err != nil do return + + raw_str := runtime.Raw_String { + data = raw_data(buf), + } + StringGetBytes(theString, {0, length}, StringEncoding(StringBuiltInEncodings.UTF8), 0, false, raw_data(buf), max, (^Index)(&raw_str.len)) + + return transmute(string)raw_str, true +} diff --git a/core/sys/darwin/Foundation/NSApplication.odin b/core/sys/darwin/Foundation/NSApplication.odin index d332345f9..34221aed6 100644 --- a/core/sys/darwin/Foundation/NSApplication.odin +++ b/core/sys/darwin/Foundation/NSApplication.odin @@ -132,7 +132,7 @@ Application_nextEventMatchingMask :: proc "c" (self: ^Application, mask: EventMa @(objc_type=Application, objc_name="sendEvent") Application_sendEvent :: proc "c" (self: ^Application, event: ^Event) { - msgSend(Event, self, "sendEvent:", event) + msgSend(nil, self, "sendEvent:", event) } @(objc_type=Application, objc_name="updateWindows") Application_updateWindows :: proc "c" (self: ^Application) { diff --git a/core/sys/darwin/Foundation/NSString.odin b/core/sys/darwin/Foundation/NSString.odin index d3c6c454d..b4918b3fb 100644 --- a/core/sys/darwin/Foundation/NSString.odin +++ b/core/sys/darwin/Foundation/NSString.odin @@ -23,12 +23,9 @@ StringEncoding :: enum UInteger { WindowsCP1250 = 15, ISO2022JP = 21, MacOSRoman = 30, - UTF16 = Unicode, - UTF16BigEndian = 0x90000100, UTF16LittleEndian = 0x94000100, - UTF32 = 0x8c000100, UTF32BigEndian = 0x98000100, UTF32LittleEndian = 0x9c000100, @@ -49,12 +46,9 @@ StringCompareOption :: enum UInteger { unichar :: distinct u16 -@(link_prefix="NS", default_calling_convention="c") -foreign Foundation { - StringFromClass :: proc(cls: Class) -> ^String --- -} - AT :: MakeConstantString + +// CFString is 'toll-free bridged' with its Cocoa Foundation counterpart, NSString. MakeConstantString :: proc "c" (#const c: cstring) -> ^String { foreign Foundation { __CFStringMakeConstantString :: proc "c" (c: cstring) -> ^String --- @@ -62,6 +56,10 @@ MakeConstantString :: proc "c" (#const c: cstring) -> ^String { return __CFStringMakeConstantString(c) } +@(link_prefix="NS", default_calling_convention="c") +foreign Foundation { + StringFromClass :: proc(cls: Class) -> ^String --- +} @(objc_type=String, objc_name="alloc", objc_is_class_method=true) String_alloc :: proc "c" () -> ^String { @@ -73,7 +71,6 @@ String_init :: proc "c" (self: ^String) -> ^String { return msgSend(^String, self, "init") } - @(objc_type=String, objc_name="initWithString") String_initWithString :: proc "c" (self: ^String, other: ^String) -> ^String { return msgSend(^String, self, "initWithString:", other) diff --git a/core/sys/darwin/Security/SecBase.odin b/core/sys/darwin/Security/SecBase.odin new file mode 100644 index 000000000..9cc82d6f5 --- /dev/null +++ b/core/sys/darwin/Security/SecBase.odin @@ -0,0 +1,386 @@ +package Security + +OSStatus :: distinct i32 + +errSec :: enum OSStatus { + Success = 0, // No error. + Unimplemented = -4, // Function or operation not implemented. + DiskFull = -34, // The disk is full. + IO = -36, // I/O error. + OpWr = -49, // File already open with with write permission. + Param = -50, // One or more parameters passed to a function were not valid. + WrPerm = -61, // Write permissions error. + Allocate = -108, // Failed to allocate memory. + UserCanceled = -128, // User canceled the operation. + BadReq = -909, // Bad parameter or invalid state for operation. + InternalComponent = -2070, + CoreFoundationUnknown = -4960, + MissingEntitlement, // A required entitlement isn't present. + RestrictedAPI, // Client is restricted and is not permitted to perform this operation. + NotAvailable = -25291, // No keychain is available. You may need to restart your computer. + ReadOnly = -25292, // This keychain cannot be modified. + AuthFailed = -25293, // The user name or passphrase you entered is not correct. + NoSuchKeychain = -25294, // The specified keychain could not be found. + InvalidKeychain = -25295, // The specified keychain is not a valid keychain file. + DuplicateKeychain = -25296, // A keychain with the same name already exists. + DuplicateCallback = -25297, // The specified callback function is already installed. + InvalidCallback = -25298, // The specified callback function is not valid. + DuplicateItem = -25299, // The specified item already exists in the keychain. + ItemNotFound = -25300, // The specified item could not be found in the keychain. + BufferTooSmall = -25301, // There is not enough memory available to use the specified item. + DataTooLarge = -25302, // This item contains information which is too large or in a format that cannot be displayed. + NoSuchAttr = -25303, // The specified attribute does not exist. + InvalidItemRef = -25304, // The specified item is no longer valid. It may have been deleted from the keychain. + InvalidSearchRef = -25305, // Unable to search the current keychain. + NoSuchClass = -25306, // The specified item does not appear to be a valid keychain item. + NoDefaultKeychain = -25307, // A default keychain could not be found. + InteractionNotAllowed = -25308, // User interaction is not allowed. + ReadOnlyAttr = -25309, // The specified attribute could not be modified. + WrongSecVersion = -25310, // This keychain was created by a different version of the system software and cannot be opened. + KeySizeNotAllowed = -25311, // This item specifies a key size which is too large or too small. + NoStorageModule = -25312, // A required component (data storage module) could not be loaded. You may need to restart your computer. + NoCertificateModule = -25313, // A required component (certificate module) could not be loaded. You may need to restart your computer. + NoPolicyModule = -25314, // A required component (policy module) could not be loaded. You may need to restart your computer. + InteractionRequired = -25315, // User interaction is required, but is currently not allowed. + DataNotAvailable = -25316, // The contents of this item cannot be retrieved. + DataNotModifiable = -25317, // The contents of this item cannot be modified. + CreateChainFailed = -25318, // One or more certificates required to validate this certificate cannot be found. + InvalidPrefsDomain = -25319, // The specified preferences domain is not valid. + InDarkWake = -25320, // In dark wake, no UI possible + ACLNotSimple = -25240, // The specified access control list is not in standard (simple) form. + PolicyNotFound = -25241, // The specified policy cannot be found. + InvalidTrustSetting = -25242, // The specified trust setting is invalid. + NoAccessForItem = -25243, // The specified item has no access control. + InvalidOwnerEdit = -25244, // Invalid attempt to change the owner of this item. + TrustNotAvailable = -25245, // No trust results are available. + UnsupportedFormat = -25256, // Import/Export format unsupported. + UnknownFormat = -25257, // Unknown format in import. + KeyIsSensitive = -25258, // Key material must be wrapped for export. + MultiplePrivKeys = -25259, // An attempt was made to import multiple private keys. + PassphraseRequired = -25260, // Passphrase is required for import/export. + InvalidPasswordRef = -25261, // The password reference was invalid. + InvalidTrustSettings = -25262, // The Trust Settings Record was corrupted. + NoTrustSettings = -25263, // No Trust Settings were found. + Pkcs12VerifyFailure = -25264, // MAC verification failed during PKCS12 import (wrong password?) + NotSigner = -26267, // A certificate was not signed by its proposed parent. + Decode = -26275, // Unable to decode the provided data. + ServiceNotAvailable = -67585, // The required service is not available. + InsufficientClientID = -67586, // The client ID is not correct. + DeviceReset = -67587, // A device reset has occurred. + DeviceFailed = -67588, // A device failure has occurred. + AppleAddAppACLSubject = -67589, // Adding an application ACL subject failed. + ApplePublicKeyIncomplete = -67590, // The public key is incomplete. + AppleSignatureMismatch = -67591, // A signature mismatch has occurred. + AppleInvalidKeyStartDate = -67592, // The specified key has an invalid start date. + AppleInvalidKeyEndDate = -67593, // The specified key has an invalid end date. + ConversionError = -67594, // A conversion error has occurred. + AppleSSLv2Rollback = -67595, // A SSLv2 rollback error has occurred. + QuotaExceeded = -67596, // The quota was exceeded. + FileTooBig = -67597, // The file is too big. + InvalidDatabaseBlob = -67598, // The specified database has an invalid blob. + InvalidKeyBlob = -67599, // The specified database has an invalid key blob. + IncompatibleDatabaseBlob = -67600, // The specified database has an incompatible blob. + IncompatibleKeyBlob = -67601, // The specified database has an incompatible key blob. + HostNameMismatch = -67602, // A host name mismatch has occurred. + UnknownCriticalExtensionFlag = -67603, // There is an unknown critical extension flag. + NoBasicConstraints = -67604, // No basic constraints were found. + NoBasicConstraintsCA = -67605, // No basic CA constraints were found. + InvalidAuthorityKeyID = -67606, // The authority key ID is not valid. + InvalidSubjectKeyID = -67607, // The subject key ID is not valid. + InvalidKeyUsageForPolicy = -67608, // The key usage is not valid for the specified policy. + InvalidExtendedKeyUsage = -67609, // The extended key usage is not valid. + InvalidIDLinkage = -67610, // The ID linkage is not valid. + PathLengthConstraintExceeded = -67611, // The path length constraint was exceeded. + InvalidRoot = -67612, // The root or anchor certificate is not valid. + CRLExpired = -67613, // The CRL has expired. + CRLNotValidYet = -67614, // The CRL is not yet valid. + CRLNotFound = -67615, // The CRL was not found. + CRLServerDown = -67616, // The CRL server is down. + CRLBadURI = -67617, // The CRL has a bad Uniform Resource Identifier. + UnknownCertExtension = -67618, // An unknown certificate extension was encountered. + UnknownCRLExtension = -67619, // An unknown CRL extension was encountered. + CRLNotTrusted = -67620, // The CRL is not trusted. + CRLPolicyFailed = -67621, // The CRL policy failed. + IDPFailure = -67622, // The issuing distribution point was not valid. + SMIMEEmailAddressesNotFound = -67623, // An email address mismatch was encountered. + SMIMEBadExtendedKeyUsage = -67624, // The appropriate extended key usage for SMIME was not found. + SMIMEBadKeyUsage = -67625, // The key usage is not compatible with SMIME. + SMIMEKeyUsageNotCritical = -67626, // The key usage extension is not marked as critical. + SMIMENoEmailAddress = -67627, // No email address was found in the certificate. + SMIMESubjAltNameNotCritical = -67628, // The subject alternative name extension is not marked as critical. + SSLBadExtendedKeyUsage = -67629, // The appropriate extended key usage for SSL was not found. + OCSPBadResponse = -67630, // The OCSP response was incorrect or could not be parsed. + OCSPBadRequest = -67631, // The OCSP request was incorrect or could not be parsed. + OCSPUnavailable = -67632, // OCSP service is unavailable. + OCSPStatusUnrecognized = -67633, // The OCSP server did not recognize this certificate. + EndOfData = -67634, // An end-of-data was detected. + IncompleteCertRevocationCheck = -67635, // An incomplete certificate revocation check occurred. + NetworkFailure = -67636, // A network failure occurred. + OCSPNotTrustedToAnchor = -67637, // The OCSP response was not trusted to a root or anchor certificate. + RecordModified = -67638, // The record was modified. + OCSPSignatureError = -67639, // The OCSP response had an invalid signature. + OCSPNoSigner = -67640, // The OCSP response had no signer. + OCSPResponderMalformedReq = -67641, // The OCSP responder was given a malformed request. + OCSPResponderInternalError = -67642, // The OCSP responder encountered an internal error. + OCSPResponderTryLater = -67643, // The OCSP responder is busy, try again later. + OCSPResponderSignatureRequired = -67644, // The OCSP responder requires a signature. + OCSPResponderUnauthorized = -67645, // The OCSP responder rejected this request as unauthorized. + OCSPResponseNonceMismatch = -67646, // The OCSP response nonce did not match the request. + CodeSigningBadCertChainLength = -67647, // Code signing encountered an incorrect certificate chain length. + CodeSigningNoBasicConstraints = -67648, // Code signing found no basic constraints. + CodeSigningBadPathLengthConstraint = -67649, // Code signing encountered an incorrect path length constraint. + CodeSigningNoExtendedKeyUsage = -67650, // Code signing found no extended key usage. + CodeSigningDevelopment = -67651, // Code signing indicated use of a development-only certificate. + ResourceSignBadCertChainLength = -67652, // Resource signing has encountered an incorrect certificate chain length. + ResourceSignBadExtKeyUsage = -67653, // Resource signing has encountered an error in the extended key usage. + TrustSettingDeny = -67654, // The trust setting for this policy was set to Deny. + InvalidSubjectName = -67655, // An invalid certificate subject name was encountered. + UnknownQualifiedCertStatement = -67656, // An unknown qualified certificate statement was encountered. + MobileMeRequestQueued = -67657, + MobileMeRequestRedirected = -67658, + MobileMeServerError = -67659, + MobileMeServerNotAvailable = -67660, + MobileMeServerAlreadyExists = -67661, + MobileMeServerServiceErr = -67662, + MobileMeRequestAlreadyPending = -67663, + MobileMeNoRequestPending = -67664, + MobileMeCSRVerifyFailure = -67665, + MobileMeFailedConsistencyCheck = -67666, + NotInitialized = -67667, // A function was called without initializing CSSM. + InvalidHandleUsage = -67668, // The CSSM handle does not match with the service type. + PVCReferentNotFound = -67669, // A reference to the calling module was not found in the list of authorized callers. + FunctionIntegrityFail = -67670, // A function address was not within the verified module. + InternalError = -67671, // An internal error has occurred. + MemoryError = -67672, // A memory error has occurred. + InvalidData = -67673, // Invalid data was encountered. + MDSError = -67674, // A Module Directory Service error has occurred. + InvalidPointer = -67675, // An invalid pointer was encountered. + SelfCheckFailed = -67676, // Self-check has failed. + FunctionFailed = -67677, // A function has failed. + ModuleManifestVerifyFailed = -67678, // A module manifest verification failure has occurred. + InvalidGUID = -67679, // An invalid GUID was encountered. + InvalidHandle = -67680, // An invalid handle was encountered. + InvalidDBList = -67681, // An invalid DB list was encountered. + InvalidPassthroughID = -67682, // An invalid passthrough ID was encountered. + InvalidNetworkAddress = -67683, // An invalid network address was encountered. + CRLAlreadySigned = -67684, // The certificate revocation list is already signed. + InvalidNumberOfFields = -67685, // An invalid number of fields were encountered. + VerificationFailure = -67686, // A verification failure occurred. + UnknownTag = -67687, // An unknown tag was encountered. + InvalidSignature = -67688, // An invalid signature was encountered. + InvalidName = -67689, // An invalid name was encountered. + InvalidCertificateRef = -67690, // An invalid certificate reference was encountered. + InvalidCertificateGroup = -67691, // An invalid certificate group was encountered. + TagNotFound = -67692, // The specified tag was not found. + InvalidQuery = -67693, // The specified query was not valid. + InvalidValue = -67694, // An invalid value was detected. + CallbackFailed = -67695, // A callback has failed. + ACLDeleteFailed = -67696, // An ACL delete operation has failed. + ACLReplaceFailed = -67697, // An ACL replace operation has failed. + ACLAddFailed = -67698, // An ACL add operation has failed. + ACLChangeFailed = -67699, // An ACL change operation has failed. + InvalidAccessCredentials = -67700, // Invalid access credentials were encountered. + InvalidRecord = -67701, // An invalid record was encountered. + InvalidACL = -67702, // An invalid ACL was encountered. + InvalidSampleValue = -67703, // An invalid sample value was encountered. + IncompatibleVersion = -67704, // An incompatible version was encountered. + PrivilegeNotGranted = -67705, // The privilege was not granted. + InvalidScope = -67706, // An invalid scope was encountered. + PVCAlreadyConfigured = -67707, // The PVC is already configured. + InvalidPVC = -67708, // An invalid PVC was encountered. + EMMLoadFailed = -67709, // The EMM load has failed. + EMMUnloadFailed = -67710, // The EMM unload has failed. + AddinLoadFailed = -67711, // The add-in load operation has failed. + InvalidKeyRef = -67712, // An invalid key was encountered. + InvalidKeyHierarchy = -67713, // An invalid key hierarchy was encountered. + AddinUnloadFailed = -67714, // The add-in unload operation has failed. + LibraryReferenceNotFound = -67715, // A library reference was not found. + InvalidAddinFunctionTable = -67716, // An invalid add-in function table was encountered. + InvalidServiceMask = -67717, // An invalid service mask was encountered. + ModuleNotLoaded = -67718, // A module was not loaded. + InvalidSubServiceID = -67719, // An invalid subservice ID was encountered. + AttributeNotInContext = -67720, // An attribute was not in the context. + ModuleManagerInitializeFailed = -67721, // A module failed to initialize. + ModuleManagerNotFound = -67722, // A module was not found. + EventNotificationCallbackNotFound = -67723, // An event notification callback was not found. + InputLengthError = -67724, // An input length error was encountered. + OutputLengthError = -67725, // An output length error was encountered. + PrivilegeNotSupported = -67726, // The privilege is not supported. + DeviceError = -67727, // A device error was encountered. + AttachHandleBusy = -67728, // The CSP handle was busy. + NotLoggedIn = -67729, // You are not logged in. + AlgorithmMismatch = -67730, // An algorithm mismatch was encountered. + KeyUsageIncorrect = -67731, // The key usage is incorrect. + KeyBlobTypeIncorrect = -67732, // The key blob type is incorrect. + KeyHeaderInconsistent = -67733, // The key header is inconsistent. + UnsupportedKeyFormat = -67734, // The key header format is not supported. + UnsupportedKeySize = -67735, // The key size is not supported. + InvalidKeyUsageMask = -67736, // The key usage mask is not valid. + UnsupportedKeyUsageMask = -67737, // The key usage mask is not supported. + InvalidKeyAttributeMask = -67738, // The key attribute mask is not valid. + UnsupportedKeyAttributeMask = -67739, // The key attribute mask is not supported. + InvalidKeyLabel = -67740, // The key label is not valid. + UnsupportedKeyLabel = -67741, // The key label is not supported. + InvalidKeyFormat = -67742, // The key format is not valid. + UnsupportedVectorOfBuffers = -67743, // The vector of buffers is not supported. + InvalidInputVector = -67744, // The input vector is not valid. + InvalidOutputVector = -67745, // The output vector is not valid. + InvalidContext = -67746, // An invalid context was encountered. + InvalidAlgorithm = -67747, // An invalid algorithm was encountered. + InvalidAttributeKey = -67748, // A key attribute was not valid. + MissingAttributeKey = -67749, // A key attribute was missing. + InvalidAttributeInitVector = -67750, // An init vector attribute was not valid. + MissingAttributeInitVector = -67751, // An init vector attribute was missing. + InvalidAttributeSalt = -67752, // A salt attribute was not valid. + MissingAttributeSalt = -67753, // A salt attribute was missing. + InvalidAttributePadding = -67754, // A padding attribute was not valid. + MissingAttributePadding = -67755, // A padding attribute was missing. + InvalidAttributeRandom = -67756, // A random number attribute was not valid. + MissingAttributeRandom = -67757, // A random number attribute was missing. + InvalidAttributeSeed = -67758, // A seed attribute was not valid. + MissingAttributeSeed = -67759, // A seed attribute was missing. + InvalidAttributePassphrase = -67760, // A passphrase attribute was not valid. + MissingAttributePassphrase = -67761, // A passphrase attribute was missing. + InvalidAttributeKeyLength = -67762, // A key length attribute was not valid. + MissingAttributeKeyLength = -67763, // A key length attribute was missing. + InvalidAttributeBlockSize = -67764, // A block size attribute was not valid. + MissingAttributeBlockSize = -67765, // A block size attribute was missing. + InvalidAttributeOutputSize = -67766, // An output size attribute was not valid. + MissingAttributeOutputSize = -67767, // An output size attribute was missing. + InvalidAttributeRounds = -67768, // The number of rounds attribute was not valid. + MissingAttributeRounds = -67769, // The number of rounds attribute was missing. + InvalidAlgorithmParms = -67770, // An algorithm parameters attribute was not valid. + MissingAlgorithmParms = -67771, // An algorithm parameters attribute was missing. + InvalidAttributeLabel = -67772, // A label attribute was not valid. + MissingAttributeLabel = -67773, // A label attribute was missing. + InvalidAttributeKeyType = -67774, // A key type attribute was not valid. + MissingAttributeKeyType = -67775, // A key type attribute was missing. + InvalidAttributeMode = -67776, // A mode attribute was not valid. + MissingAttributeMode = -67777, // A mode attribute was missing. + InvalidAttributeEffectiveBits = -67778, // An effective bits attribute was not valid. + MissingAttributeEffectiveBits = -67779, // An effective bits attribute was missing. + InvalidAttributeStartDate = -67780, // A start date attribute was not valid. + MissingAttributeStartDate = -67781, // A start date attribute was missing. + InvalidAttributeEndDate = -67782, // An end date attribute was not valid. + MissingAttributeEndDate = -67783, // An end date attribute was missing. + InvalidAttributeVersion = -67784, // A version attribute was not valid. + MissingAttributeVersion = -67785, // A version attribute was missing. + InvalidAttributePrime = -67786, // A prime attribute was not valid. + MissingAttributePrime = -67787, // A prime attribute was missing. + InvalidAttributeBase = -67788, // A base attribute was not valid. + MissingAttributeBase = -67789, // A base attribute was missing. + InvalidAttributeSubprime = -67790, // A subprime attribute was not valid. + MissingAttributeSubprime = -67791, // A subprime attribute was missing. + InvalidAttributeIterationCount = -67792, // An iteration count attribute was not valid. + MissingAttributeIterationCount = -67793, // An iteration count attribute was missing. + InvalidAttributeDLDBHandle = -67794, // A database handle attribute was not valid. + MissingAttributeDLDBHandle = -67795, // A database handle attribute was missing. + InvalidAttributeAccessCredentials = -67796, // An access credentials attribute was not valid. + MissingAttributeAccessCredentials = -67797, // An access credentials attribute was missing. + InvalidAttributePublicKeyFormat = -67798, // A public key format attribute was not valid. + MissingAttributePublicKeyFormat = -67799, // A public key format attribute was missing. + InvalidAttributePrivateKeyFormat = -67800, // A private key format attribute was not valid. + MissingAttributePrivateKeyFormat = -67801, // A private key format attribute was missing. + InvalidAttributeSymmetricKeyFormat = -67802, // A symmetric key format attribute was not valid. + MissingAttributeSymmetricKeyFormat = -67803, // A symmetric key format attribute was missing. + InvalidAttributeWrappedKeyFormat = -67804, // A wrapped key format attribute was not valid. + MissingAttributeWrappedKeyFormat = -67805, // A wrapped key format attribute was missing. + StagedOperationInProgress = -67806, // A staged operation is in progress. + StagedOperationNotStarted = -67807, // A staged operation was not started. + VerifyFailed = -67808, // A cryptographic verification failure has occurred. + QuerySizeUnknown = -67809, // The query size is unknown. + BlockSizeMismatch = -67810, // A block size mismatch occurred. + PublicKeyInconsistent = -67811, // The public key was inconsistent. + DeviceVerifyFailed = -67812, // A device verification failure has occurred. + InvalidLoginName = -67813, // An invalid login name was detected. + AlreadyLoggedIn = -67814, // The user is already logged in. + InvalidDigestAlgorithm = -67815, // An invalid digest algorithm was detected. + InvalidCRLGroup = -67816, // An invalid CRL group was detected. + CertificateCannotOperate = -67817, // The certificate cannot operate. + CertificateExpired = -67818, // An expired certificate was detected. + CertificateNotValidYet = -67819, // The certificate is not yet valid. + CertificateRevoked = -67820, // The certificate was revoked. + CertificateSuspended = -67821, // The certificate was suspended. + InsufficientCredentials = -67822, // Insufficient credentials were detected. + InvalidAction = -67823, // The action was not valid. + InvalidAuthority = -67824, // The authority was not valid. + VerifyActionFailed = -67825, // A verify action has failed. + InvalidCertAuthority = -67826, // The certificate authority was not valid. + InvalidCRLAuthority = -67827, // The CRL authority was not valid. + InvalidCRLEncoding = -67828, // The CRL encoding was not valid. + InvalidCRLType = -67829, // The CRL type was not valid. + InvalidCRL = -67830, // The CRL was not valid. + InvalidFormType = -67831, // The form type was not valid. + InvalidID = -67832, // The ID was not valid. + InvalidIdentifier = -67833, // The identifier was not valid. + InvalidIndex = -67834, // The index was not valid. + InvalidPolicyIdentifiers = -67835, // The policy identifiers are not valid. + InvalidTimeString = -67836, // The time specified was not valid. + InvalidReason = -67837, // The trust policy reason was not valid. + InvalidRequestInputs = -67838, // The request inputs are not valid. + InvalidResponseVector = -67839, // The response vector was not valid. + InvalidStopOnPolicy = -67840, // The stop-on policy was not valid. + InvalidTuple = -67841, // The tuple was not valid. + MultipleValuesUnsupported = -67842, // Multiple values are not supported. + NotTrusted = -67843, // The certificate was not trusted. + NoDefaultAuthority = -67844, // No default authority was detected. + RejectedForm = -67845, // The trust policy had a rejected form. + RequestLost = -67846, // The request was lost. + RequestRejected = -67847, // The request was rejected. + UnsupportedAddressType = -67848, // The address type is not supported. + UnsupportedService = -67849, // The service is not supported. + InvalidTupleGroup = -67850, // The tuple group was not valid. + InvalidBaseACLs = -67851, // The base ACLs are not valid. + InvalidTupleCredentials = -67852, // The tuple credentials are not valid. + InvalidEncoding = -67853, // The encoding was not valid. + InvalidValidityPeriod = -67854, // The validity period was not valid. + InvalidRequestor = -67855, // The requestor was not valid. + RequestDescriptor = -67856, // The request descriptor was not valid. + InvalidBundleInfo = -67857, // The bundle information was not valid. + InvalidCRLIndex = -67858, // The CRL index was not valid. + NoFieldValues = -67859, // No field values were detected. + UnsupportedFieldFormat = -67860, // The field format is not supported. + UnsupportedIndexInfo = -67861, // The index information is not supported. + UnsupportedLocality = -67862, // The locality is not supported. + UnsupportedNumAttributes = -67863, // The number of attributes is not supported. + UnsupportedNumIndexes = -67864, // The number of indexes is not supported. + UnsupportedNumRecordTypes = -67865, // The number of record types is not supported. + FieldSpecifiedMultiple = -67866, // Too many fields were specified. + IncompatibleFieldFormat = -67867, // The field format was incompatible. + InvalidParsingModule = -67868, // The parsing module was not valid. + DatabaseLocked = -67869, // The database is locked. + DatastoreIsOpen = -67870, // The data store is open. + MissingValue = -67871, // A missing value was detected. + UnsupportedQueryLimits = -67872, // The query limits are not supported. + UnsupportedNumSelectionPreds = -67873, // The number of selection predicates is not supported. + UnsupportedOperator = -67874, // The operator is not supported. + InvalidDBLocation = -67875, // The database location is not valid. + InvalidAccessRequest = -67876, // The access request is not valid. + InvalidIndexInfo = -67877, // The index information is not valid. + InvalidNewOwner = -67878, // The new owner is not valid. + InvalidModifyMode = -67879, // The modify mode is not valid. + MissingRequiredExtension = -67880, // A required certificate extension is missing. + ExtendedKeyUsageNotCritical = -67881, // The extended key usage extension was not marked critical. + TimestampMissing = -67882, // A timestamp was expected but was not found. + TimestampInvalid = -67883, // The timestamp was not valid. + TimestampNotTrusted = -67884, // The timestamp was not trusted. + TimestampServiceNotAvailable = -67885, // The timestamp service is not available. + TimestampBadAlg = -67886, // An unrecognized or unsupported Algorithm Identifier in timestamp. + TimestampBadRequest = -67887, // The timestamp transaction is not permitted or supported. + TimestampBadDataFormat = -67888, // The timestamp data submitted has the wrong format. + TimestampTimeNotAvailable = -67889, // The time source for the Timestamp Authority is not available. + TimestampUnacceptedPolicy = -67890, // The requested policy is not supported by the Timestamp Authority. + TimestampUnacceptedExtension = -67891, // The requested extension is not supported by the Timestamp Authority. + TimestampAddInfoNotAvailable = -67892, // The additional information requested is not available. + TimestampSystemFailure = -67893, // The timestamp request cannot be handled due to system failure. + SigningTimeMissing = -67894, // A signing time was expected but was not found. + TimestampRejection = -67895, // A timestamp transaction was rejected. + TimestampWaiting = -67896, // A timestamp transaction is waiting. + TimestampRevocationWarning = -67897, // A timestamp authority revocation warning was issued. + TimestampRevocationNotification = -67898, // A timestamp authority revocation notification was issued. + CertificatePolicyNotAllowed = -67899, // The requested policy is not allowed for this certificate. + CertificateNameNotAllowed = -67900, // The requested name is not allowed for this certificate. + CertificateValidityPeriodTooLong = -67901, // The validity period in the certificate exceeds the maximum allowed. + CertificateIsCA = -67902, // The verified certificate is a CA rather than an end-entity. + CertificateDuplicateExtension = -67903, // The certificate contains multiple extensions with the same extension ID. +} diff --git a/core/sys/darwin/Security/SecRandom.odin b/core/sys/darwin/Security/SecRandom.odin new file mode 100644 index 000000000..0527baca1 --- /dev/null +++ b/core/sys/darwin/Security/SecRandom.odin @@ -0,0 +1,19 @@ +package Security + +import CF "core:sys/darwin/CoreFoundation" + +foreign import Security "system:Security.framework" + +// A reference to a random number generator. +RandomRef :: distinct rawptr + +@(link_prefix="Sec", default_calling_convention="c") +foreign Security { + // Default random ref for /dev/random. Synonym for nil. + @(link_name="kSecRandomDefault") kSecRandomDefault: RandomRef + + // Generates an array of cryptographically secure random bytes. + RandomCopyBytes :: proc(rnd: RandomRef = kSecRandomDefault, count: uint, bytes: [^]byte) -> errSec --- + + CopyErrorMessageString :: proc(status: errSec, reserved: rawptr = nil) -> CF.String --- +} \ No newline at end of file diff --git a/core/sys/darwin/core_foundation.odin b/core/sys/darwin/core_foundation.odin deleted file mode 100644 index 325122216..000000000 --- a/core/sys/darwin/core_foundation.odin +++ /dev/null @@ -1,98 +0,0 @@ -//+build darwin -package darwin - -import "base:runtime" - -foreign import core_foundation "system:CoreFoundation.framework" - -CFTypeRef :: distinct rawptr - -CFStringRef :: distinct CFTypeRef - -CFIndex :: int - -CFRange :: struct { - location: CFIndex, - length: CFIndex, -} - -CFStringEncoding :: enum u32 { - ASCII = 1, - NEXTSTEP = 2, - JapaneseEUC = 3, - UTF8 = 4, - ISOLatin1 = 5, - Symbol = 6, - NonLossyASCII = 7, - ShiftJIS = 8, - ISOLatin2 = 9, - Unicode = 10, - WindowsCP1251 = 11, - WindowsCP1252 = 12, - WindowsCP1253 = 13, - WindowsCP1254 = 14, - WindowsCP1250 = 15, - ISO2022JP = 21, - MacOSRoman = 30, - - UTF16 = Unicode, - - UTF16BigEndian = 0x90000100, - UTF16LittleEndian = 0x94000100, - - UTF32 = 0x8c000100, - UTF32BigEndian = 0x98000100, - UTF32LittleEndian = 0x9c000100, -} - -foreign core_foundation { - // Copies the character contents of a string to a local C string buffer after converting the characters to a given encoding. - CFStringGetCString :: proc(theString: CFStringRef, buffer: [^]byte, bufferSize: CFIndex, encoding: CFStringEncoding) -> Bool --- - - // Returns the number (in terms of UTF-16 code pairs) of Unicode characters in a string. - CFStringGetLength :: proc(theString: CFStringRef) -> CFIndex --- - - // Returns the maximum number of bytes a string of a specified length (in Unicode characters) will take up if encoded in a specified encoding. - CFStringGetMaximumSizeForEncoding :: proc(length: CFIndex, encoding: CFStringEncoding) -> CFIndex --- - - // Fetches a range of the characters from a string into a byte buffer after converting the characters to a specified encoding. - CFStringGetBytes :: proc( - thestring: CFStringRef, - range: CFRange, - encoding: CFStringEncoding, - lossByte: u8, - isExternalRepresentation: Bool, - buffer: [^]byte, - maxBufLen: CFIndex, - usedBufLen: ^CFIndex, - ) -> CFIndex --- - - // Releases a Core Foundation object. - @(link_name="CFRelease") - _CFRelease :: proc(cf: CFTypeRef) --- -} - -// Releases a Core Foundation object. -CFRelease :: proc { - CFReleaseString, -} - -// Releases a Core Foundation string. -CFReleaseString :: #force_inline proc(theString: CFStringRef) { - _CFRelease(CFTypeRef(theString)) -} - -CFStringCopyToOdinString :: proc(theString: CFStringRef, allocator := context.allocator) -> (str: string, ok: bool) #optional_ok { - length := CFStringGetLength(theString) - max := CFStringGetMaximumSizeForEncoding(length, .UTF8) - - buf, err := make([]byte, max, allocator) - if err != nil { return } - - raw_str := runtime.Raw_String{ - data = raw_data(buf), - } - CFStringGetBytes(theString, {0, length}, .UTF8, 0, false, raw_data(buf), max, &raw_str.len) - - return transmute(string)raw_str, true -} diff --git a/core/sys/darwin/security.odin b/core/sys/darwin/security.odin deleted file mode 100644 index 0c58260e7..000000000 --- a/core/sys/darwin/security.odin +++ /dev/null @@ -1,26 +0,0 @@ -//+build darwin -package darwin - -foreign import security "system:Security.framework" - -// A reference to a random number generator. -SecRandomRef :: distinct rawptr - -OSStatus :: distinct i32 - -errSec :: enum OSStatus { - Success = 0, // No error. - Unimplemented = -4, // Function or operation not implemented. - - // Many more... -} - -foreign security { - // Synonym for nil, uses a cryptographically secure random number generator. - kSecRandomDefault: SecRandomRef - - // Generates an array of cryptographically secure random bytes. - SecRandomCopyBytes :: proc(rnd: SecRandomRef = kSecRandomDefault, count: uint, bytes: [^]byte) -> errSec --- - - SecCopyErrorMessageString :: proc(status: errSec, reserved: rawptr = nil) -> CFStringRef --- -} diff --git a/core/sys/darwin/xnu_system_call_wrappers.odin b/core/sys/darwin/xnu_system_call_wrappers.odin index b69877cc9..7100da4f1 100644 --- a/core/sys/darwin/xnu_system_call_wrappers.odin +++ b/core/sys/darwin/xnu_system_call_wrappers.odin @@ -337,7 +337,7 @@ syscall_ftruncate :: #force_inline proc "contextless" (fd: c.int, length: off_t) return cast(c.int)intrinsics.syscall(unix_offset_syscall(.ftruncate), uintptr(fd), uintptr(length)) } -syscall_sysctl :: #force_inline proc "contextless" (name: ^c.int, namelen: c.uint, oldp: rawptr, oldlenp: ^i64, newp: ^i8, newlen: i64) -> c.int { +syscall_sysctl :: #force_inline proc "contextless" (name: [^]c.int, namelen: c.size_t, oldp: rawptr, oldlenp: ^c.size_t, newp: rawptr, newlen: c.size_t) -> c.int { return cast(c.int)intrinsics.syscall(unix_offset_syscall(.sysctl), uintptr(name), uintptr(namelen), uintptr(oldp), uintptr(oldlenp), uintptr(newp), uintptr(newlen)) } @@ -390,8 +390,8 @@ syscall_adjtime :: #force_inline proc "contextless" (delta: ^timeval, old_delta: return cast(c.int)intrinsics.syscall(unix_offset_syscall(.adjtime), uintptr(delta), uintptr(old_delta)) } -syscall_sysctlbyname :: #force_inline proc "contextless" (name: cstring, oldp: rawptr, oldlenp: ^i64, newp: rawptr, newlen: i64) -> c.int { - return cast(c.int)intrinsics.syscall(unix_offset_syscall(.sysctlbyname), transmute(uintptr)name, uintptr(oldp), uintptr(oldlenp), uintptr(newp), uintptr(newlen)) +syscall_sysctlbyname :: #force_inline proc "contextless" (name: string, oldp: rawptr, oldlenp: ^c.size_t, newp: rawptr, newlen: c.size_t) -> c.int { + return cast(c.int)intrinsics.syscall(unix_offset_syscall(.sysctlbyname), uintptr(raw_data(name)), uintptr(len(name)), uintptr(oldp), uintptr(oldlenp), uintptr(newp), uintptr(newlen)) } syscall_proc_info :: #force_inline proc "contextless" (num: c.int, pid: u32, flavor: c.int, arg: u64, buffer: rawptr, buffer_size: c.int) -> c.int { diff --git a/core/sys/info/cpu_arm.odin b/core/sys/info/cpu_arm.odin index f66f0e780..aa4bb368a 100644 --- a/core/sys/info/cpu_arm.odin +++ b/core/sys/info/cpu_arm.odin @@ -1,26 +1,70 @@ //+build arm32, arm64 package sysinfo -// TODO: Set up an enum with the ARM equivalent of the above. -CPU_Feature :: enum u64 {} +import "core:sys/unix" -cpu_features: Maybe(CPU_Feature) -cpu_name: Maybe(string) +_ :: unix -@(init, private) -init_cpu_features :: proc "c" () { +CPU_Feature :: enum u64 { + // Advanced SIMD & floating-point capabilities: + asimd, // General support for Advanced SIMD instructions/neon. + floatingpoint, // General support for floating-point instructions. + asimdhp, // Advanced SIMD half-precision conversion instructions. + bf16, // Storage and arithmetic instructions of the Brain Floating Point (BFloat16) data type. + fcma, // Floating-point complex number instructions. + fhm, // Floating-point half-precision multiplication instructions. + fp16, // General half-precision floating-point data processing instructions. + frint, // Floating-point to integral valued floating-point number rounding instructions. + i8mm, // Advanced SIMD int8 matrix multiplication instructions. + jscvt, // JavaScript conversion instruction. + rdm, // Advanced SIMD rounding double multiply accumulate instructions. + + flagm, // Condition flag manipulation instructions. + flagm2, // Enhancements to condition flag manipulation instructions. + crc32, // CRC32 instructions. + + lse, // Atomic instructions to support large systems. + lse2, // Changes to single-copy atomicity and alignment requirements for loads and stores for large systems. + lrcpc, // Load-acquire Release Consistency processor consistent (RCpc) instructions. + lrcpc2, // Load-acquire Release Consistency processor consistent (RCpc) instructions version 2. + + aes, + pmull, + sha1, + sha256, + sha512, + sha3, + + sb, // Barrier instruction to control speculation. + ssbs, // Instructions to control speculation of loads and stores. } +CPU_Features :: distinct bit_set[CPU_Feature; u64] + +cpu_features: Maybe(CPU_Features) +cpu_name: Maybe(string) + @(private) -_cpu_name_buf: [72]u8 +cpu_name_buf: [128]byte @(init, private) -init_cpu_name :: proc "c" () { - when ODIN_ARCH == .arm32 { - copy(_cpu_name_buf[:], "ARM") - cpu_name = string(_cpu_name_buf[:3]) - } else { - copy(_cpu_name_buf[:], "ARM64") - cpu_name = string(_cpu_name_buf[:5]) +init_cpu_name :: proc "contextless" () { + generic := true + + when ODIN_OS == .Darwin { + if unix.sysctlbyname("machdep.cpu.brand_string", &cpu_name_buf) { + cpu_name = string(cstring(rawptr(&cpu_name_buf))) + generic = false + } } -} \ No newline at end of file + + if generic { + when ODIN_ARCH == .arm64 { + copy(cpu_name_buf[:], "ARM64") + cpu_name = string(cpu_name_buf[:len("ARM64")]) + } else { + copy(cpu_name_buf[:], "ARM") + cpu_name = string(cpu_name_buf[:len("ARM")]) + } + } +} diff --git a/core/sys/info/cpu_darwin_arm64.odin b/core/sys/info/cpu_darwin_arm64.odin new file mode 100644 index 000000000..336334bc0 --- /dev/null +++ b/core/sys/info/cpu_darwin_arm64.odin @@ -0,0 +1,98 @@ +package sysinfo + +import "core:sys/unix" + +@(init, private) +init_cpu_features :: proc "contextless" () { + @(static) features: CPU_Features + defer cpu_features = features + + try_set :: proc "contextless" (name: string, feature: CPU_Feature) -> (ok: bool) { + support: b32 + if ok = unix.sysctlbyname(name, &support); ok && support { + features += { feature } + } + return + } + + // Docs from Apple: https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics + // Features from there that do not have (or I didn't find) an equivalent on Linux are commented out below. + + // Advanced SIMD & floating-point capabilities: + { + if !try_set("hw.optional.AdvSIMD", .asimd) { + try_set("hw.optional.neon", .asimd) + } + + try_set("hw.optional.floatingpoint", .floatingpoint) + + if !try_set("hw.optional.AdvSIMD_HPFPCvt", .asimdhp) { + try_set("hw.optional.neon_hpfp", .asimdhp) + } + + try_set("hw.optional.arm.FEAT_BF16", .bf16) + // try_set("hw.optional.arm.FEAT_DotProd", .dotprod) + + if !try_set("hw.optional.arm.FEAT_FCMA", .fcma) { + try_set("hw.optional.armv8_3_compnum", .fcma) + } + + if !try_set("hw.optional.arm.FEAT_FHM", .fhm) { + try_set("hw.optional.armv8_2_fhm", .fhm) + } + + if !try_set("hw.optional.arm.FEAT_FP16", .fp16) { + try_set("hw.optional.neon_fp16", .fp16) + } + + try_set("hw.optional.arm.FEAT_FRINTTS", .frint) + try_set("hw.optional.arm.FEAT_I8MM", .i8mm) + try_set("hw.optional.arm.FEAT_JSCVT", .jscvt) + try_set("hw.optional.arm.FEAT_RDM", .rdm) + } + + // Integer capabilities: + { + try_set("hw.optional.arm.FEAT_FlagM", .flagm) + try_set("hw.optional.arm.FEAT_FlagM2", .flagm2) + try_set("hw.optional.armv8_crc32", .crc32) + } + + // Atomic and memory ordering instruction capabilities: + { + try_set("hw.optional.arm.FEAT_LRCPC", .lrcpc) + try_set("hw.optional.arm.FEAT_LRCPC2", .lrcpc2) + + if !try_set("hw.optional.arm.FEAT_LSE", .lse) { + try_set("hw.optional.armv8_1_atomics", .lse) + } + + // try_set("hw.optional.arm.FEAT_LSE2", .lse2) + } + + // Encryption capabilities: + { + try_set("hw.optional.arm.FEAT_AES", .aes) + try_set("hw.optional.arm.FEAT_PMULL", .pmull) + try_set("hw.optional.arm.FEAT_SHA1", .sha1) + try_set("hw.optional.arm.FEAT_SHA256", .sha256) + + if !try_set("hw.optional.arm.FEAT_SHA512", .sha512) { + try_set("hw.optional.armv8_2_sha512", .sha512) + } + + if !try_set("hw.optional.arm.FEAT_SHA3", .sha3) { + try_set("hw.optional.armv8_2_sha3", .sha3) + } + } + + // General capabilities: + { + // try_set("hw.optional.arm.FEAT_BTI", .bti) + // try_set("hw.optional.arm.FEAT_DPB", .dpb) + // try_set("hw.optional.arm.FEAT_DPB2", .dpb2) + // try_set("hw.optional.arm.FEAT_ECV", .ecv) + try_set("hw.optional.arm.FEAT_SB", .sb) + try_set("hw.optional.arm.FEAT_SSBS", .ssbs) + } +} diff --git a/core/sys/info/cpu_linux_arm.odin b/core/sys/info/cpu_linux_arm.odin new file mode 100644 index 000000000..dcc252971 --- /dev/null +++ b/core/sys/info/cpu_linux_arm.odin @@ -0,0 +1,65 @@ +//+build arm32, arm64 +//+build linux +package sysinfo + +import "core:sys/linux" +import "core:strings" + +@(init, private) +init_cpu_features :: proc() { + fd, err := linux.open("/proc/cpuinfo", {}) + if err != .NONE { return } + defer linux.close(fd) + + // This is probably enough right? + buf: [4096]byte + n, rerr := linux.read(fd, buf[:]) + if rerr != .NONE || n == 0 { return } + + features: CPU_Features + defer cpu_features = features + + str := string(buf[:n]) + for line in strings.split_lines_iterator(&str) { + key, _, value := strings.partition(line, ":") + key = strings.trim_space(key) + value = strings.trim_space(value) + + if key != "Features" { continue } + + for feature in strings.split_by_byte_iterator(&value, ' ') { + switch feature { + case "asimd", "neon": features += { .asimd } + case "fp": features += { .floatingpoint } + case "asimdhp": features += { .asimdhp } + case "asimdbf16": features += { .bf16 } + case "fcma": features += { .fcma } + case "asimdfhm": features += { .fhm } + case "fphp", "half": features += { .fp16 } + case "frint": features += { .frint } + case "i8mm": features += { .i8mm } + case "jscvt": features += { .jscvt } + case "asimdrdm": features += { .rdm } + + case "flagm": features += { .flagm } + case "flagm2": features += { .flagm2 } + case "crc32": features += { .crc32 } + + case "atomics": features += { .lse } + case "lrcpc": features += { .lrcpc } + case "ilrcpc": features += { .lrcpc2 } + + case "aes": features += { .aes } + case "pmull": features += { .pmull } + case "sha1": features += { .sha1 } + case "sha2": features += { .sha256 } + case "sha3": features += { .sha3 } + case "sha512": features += { .sha512 } + + case "sb": features += { .sb } + case "ssbs": features += { .ssbs } + } + } + break + } +} diff --git a/core/sys/info/doc.odin b/core/sys/info/doc.odin index 81c3fb342..802cd9c60 100644 --- a/core/sys/info/doc.odin +++ b/core/sys/info/doc.odin @@ -1,78 +1,81 @@ /* - Copyright 2022 Jeroen van Rijn . - Made available under Odin's BSD-3 license. +Copyright 2022 Jeroen van Rijn . +Made available under Odin's BSD-3 license. - Package `core:sys/info` gathers system information on: - Windows, Linux, macOS, FreeBSD & OpenBSD. +Package `core:sys/info` gathers system information on: +Windows, Linux, macOS, FreeBSD & OpenBSD. - Simply import the package and you'll have access to the OS version, RAM amount - and CPU information. +Simply import the package and you'll have access to the OS version, RAM amount +and CPU information. - On Windows, GPUs will also be enumerated using the registry. +On Windows, GPUs will also be enumerated using the registry. - CPU feature flags can be tested against `cpu_features`, where applicable, e.g. - `if .aes in si.aes { ... }` -*/ -//+build ignore -package sysinfo +CPU feature flags can be tested against `cpu_features`, where applicable, e.g. +`if .aes in si.aes { ... }` -import "core:fmt" -import si "core:sys/info" +Example: -main :: proc() { - fmt.printf("Odin: %v\n", ODIN_VERSION) - fmt.printf("OS: %v\n", si.os_version.as_string) - fmt.printf("OS: %#v\n", si.os_version) - fmt.printf("CPU: %v\n", si.cpu_name) - fmt.printf("RAM: %v MiB\n", si.ram.total_ram / 1024 / 1024) + import "core:fmt" + import si "core:sys/info" - fmt.println() - for gpu, i in si.gpus { - fmt.printf("GPU #%v:\n", i) - fmt.printf("\tVendor: %v\n", gpu.vendor_name) - fmt.printf("\tModel: %v\n", gpu.model_name) - fmt.printf("\tVRAM: %v MiB\n", gpu.total_ram / 1024 / 1024) + main :: proc() { + fmt.printfln("Odin: %v", ODIN_VERSION) + fmt.printfln("OS: %v", si.os_version.as_string) + fmt.printfln("OS: %#v", si.os_version) + fmt.printfln("CPU: %v", si.cpu_name) + fmt.printfln("RAM: %#.1M", si.ram.total_ram) + + // fmt.printfln("Features: %v", si.cpu_features) + // fmt.printfln("MacOS version: %v", si.macos_version) + + fmt.println() + for gpu, i in si.gpus { + fmt.printfln("GPU #%v:", i) + fmt.printfln("\tVendor: %v", gpu.vendor_name) + fmt.printfln("\tModel: %v", gpu.model_name) + fmt.printfln("\tVRAM: %#.1M", gpu.total_ram) + } } -} -/* - Example Windows output: - Odin: dev-2022-09 - OS: Windows 10 Professional (version: 20H2), build: 19042.1466 - OS: OS_Version{ - platform = "Windows", - major = 10, - minor = 0, +- Example Windows output: + + Odin: dev-2022-09 + OS: Windows 10 Professional (version: 20H2), build: 19042.1466 + OS: OS_Version{ + platform = "Windows", + major = 10, + minor = 0, + patch = 0, + build = [ + 19042, + 1466, + ], + version = "20H2", + as_string = "Windows 10 Professional (version: 20H2), build: 19042.1466", + } + CPU: AMD Ryzen 7 1800X Eight-Core Processor + RAM: 64.0 GiB + GPU #0: + Vendor: Advanced Micro Devices, Inc. + Model: Radeon RX Vega + VRAM: 8.0 GiB + +- Example macOS output: + + ODIN: dev-2022-09 + OS: OS_Version{ + platform = "MacOS", + major = 21, + minor = 5, patch = 0, build = [ - 19042, - 1466, + 0, + 0, ], - version = "20H2", - as_string = "Windows 10 Professional (version: 20H2), build: 19042.1466", - } - CPU: AMD Ryzen 7 1800X Eight-Core Processor - RAM: 65469 MiB - - GPU #0: - Vendor: Advanced Micro Devices, Inc. - Model: Radeon RX Vega - VRAM: 8176 MiB - - Example macOS output: - ODIN: dev-2022-09 - OS: OS_Version{ - platform = "MacOS", - major = 21, - minor = 5, - patch = 0, - build = [ - 0, - 0, - ], - version = "21F79", - as_string = "macOS Monterey 12.4 (build 21F79, kernel 21.5.0)", - } - CPU: Intel(R) Core(TM) i5-7360U CPU @ 2.30GHz - RAM: 8192 MiB + version = "21F79", + as_string = "macOS Monterey 12.4 (build 21F79, kernel 21.5.0)", + } + CPU: Intel(R) Core(TM) i5-7360U CPU @ 2.30GHz + RAM: 8.0 GiB */ +package sysinfo diff --git a/core/sys/info/platform_openbsd.odin b/core/sys/info/platform_bsd.odin similarity index 88% rename from core/sys/info/platform_openbsd.odin rename to core/sys/info/platform_bsd.odin index 772531ceb..e2273d253 100644 --- a/core/sys/info/platform_openbsd.odin +++ b/core/sys/info/platform_bsd.odin @@ -1,4 +1,4 @@ -// +build openbsd +//+build openbsd, netbsd package sysinfo import sys "core:sys/unix" @@ -11,12 +11,16 @@ version_string_buf: [1024]u8 @(init, private) init_os_version :: proc () { - os_version.platform = .OpenBSD + when ODIN_OS == .NetBSD { + os_version.platform = .NetBSD + } else { + os_version.platform = .OpenBSD + } kernel_version_buf: [1024]u8 b := strings.builder_from_bytes(version_string_buf[:]) - // Retrieve kernel info using `sysctl`, e.g. OpenBSD + // Retrieve kernel info using `sysctl`, e.g. OpenBSD and NetBSD mib := []i32{sys.CTL_KERN, sys.KERN_OSTYPE} if !sys.sysctl(mib, &kernel_version_buf) { return @@ -61,7 +65,7 @@ init_os_version :: proc () { os_version.as_string = strings.to_string(b) } -@(init) +@(init, private) init_ram :: proc() { // Retrieve RAM info using `sysctl` mib := []i32{sys.CTL_HW, sys.HW_PHYSMEM64} @@ -69,4 +73,4 @@ init_ram :: proc() { if sys.sysctl(mib, &mem_size) { ram.total_ram = int(mem_size) } -} \ No newline at end of file +} diff --git a/core/sys/info/platform_darwin.odin b/core/sys/info/platform_darwin.odin index b95a48bd0..122dd42ee 100644 --- a/core/sys/info/platform_darwin.odin +++ b/core/sys/info/platform_darwin.odin @@ -1,4 +1,3 @@ -// +build darwin package sysinfo import sys "core:sys/unix" @@ -76,6 +75,8 @@ init_os_version :: proc () { os_version.minor = rel.darwin.y os_version.patch = rel.darwin.z + macos_version = transmute(Version)rel.release.version + strings.write_string(&b, rel.os_name) if match == .Exact || match == .Nearest { strings.write_rune(&b, ' ') @@ -113,7 +114,7 @@ init_os_version :: proc () { os_version.as_string = strings.to_string(b) } -@(init) +@(init, private) init_ram :: proc() { // Retrieve RAM info using `sysctl` diff --git a/core/sys/info/platform_freebsd.odin b/core/sys/info/platform_freebsd.odin index 26b4be7e9..c1429c4b2 100644 --- a/core/sys/info/platform_freebsd.odin +++ b/core/sys/info/platform_freebsd.odin @@ -1,4 +1,3 @@ -// +build freebsd package sysinfo import sys "core:sys/unix" @@ -68,7 +67,7 @@ init_os_version :: proc () { } } -@(init) +@(init, private) init_ram :: proc() { // Retrieve RAM info using `sysctl` mib := []i32{sys.CTL_HW, sys.HW_PHYSMEM} diff --git a/core/sys/info/platform_linux.odin b/core/sys/info/platform_linux.odin index 89b1204a7..45efc3329 100644 --- a/core/sys/info/platform_linux.odin +++ b/core/sys/info/platform_linux.odin @@ -1,11 +1,9 @@ -// +build linux package sysinfo import "base:intrinsics" -import "base:runtime" -import "core:strings" -import "core:strconv" +import "core:strconv" +import "core:strings" import "core:sys/linux" @(private) @@ -14,32 +12,37 @@ version_string_buf: [1024]u8 @(init, private) init_os_version :: proc () { os_version.platform = .Linux - // Try to parse `/etc/os-release` for `PRETTY_NAME="Ubuntu 20.04.3 LTS` - fd, errno := linux.open("/etc/os-release", {.RDONLY}, {}) - assert(errno == .NONE, "Failed to read /etc/os-release") - defer { - cerrno := linux.close(fd) - assert(cerrno == .NONE, "Failed to close the file descriptor") - } - os_release_buf: [2048]u8 - n, read_errno := linux.read(fd, os_release_buf[:]) - assert(read_errno == .NONE, "Failed to read data from /etc/os-release") - release := string(os_release_buf[:n]) - // Search the line in the file until we find "PRETTY_NAME=" - NEEDLE :: "PRETTY_NAME=\"" - pretty_start := strings.index(release, NEEDLE) + b := strings.builder_from_bytes(version_string_buf[:]) - if pretty_start > 0 { - for r, i in release[pretty_start + len(NEEDLE):] { - if r == '"' { - strings.write_string(&b, release[pretty_start + len(NEEDLE):][:i]) - break - } else if r == '\r' || r == '\n' { - strings.write_string(&b, "Unknown Linux Distro") - break + + // Try to parse `/etc/os-release` for `PRETTY_NAME="Ubuntu 20.04.3 LTS` + { + fd, errno := linux.open("/etc/os-release", {}) + assert(errno == .NONE, "Failed to read /etc/os-release") + defer { + cerrno := linux.close(fd) + assert(cerrno == .NONE, "Failed to close the file descriptor") + } + + os_release_buf: [2048]u8 + n, read_errno := linux.read(fd, os_release_buf[:]) + assert(read_errno == .NONE, "Failed to read data from /etc/os-release") + release := string(os_release_buf[:n]) + + // Search the line in the file until we find "PRETTY_NAME=" + NEEDLE :: "PRETTY_NAME=\"" + _, _, post := strings.partition(release, NEEDLE) + if len(post) > 0 { + end := strings.index_any(post, "\"\n") + if end > -1 && post[end] == '"' { + strings.write_string(&b, post[:end]) } } + if strings.builder_len(b) == 0 { + strings.write_string(&b, "Unknown Linux Distro") + } } + // Grab kernel info using `uname()` syscall, https://linux.die.net/man/2/uname uts: linux.UTS_Name uname_errno := linux.uname(&uts) @@ -48,31 +51,39 @@ init_os_version :: proc () { strings.write_string(&b, ", ") strings.write_string(&b, string(cstring(&uts.sysname[0]))) strings.write_rune(&b, ' ') - l := strings.builder_len(b) + + release_i := strings.builder_len(b) strings.write_string(&b, string(cstring(&uts.release[0]))) - // Parse kernel version, as substrings of the version info in `version_string_buf` - runtime.DEFAULT_TEMP_ALLOCATOR_TEMP_GUARD() - version_bits := strings.split_n(strings.to_string(b)[l:], "-", 2, context.temp_allocator) - if len(version_bits) > 1 { - os_version.version = version_bits[1] - } - // Parse major, minor, patch from release info - triplet := strings.split(version_bits[0], ".", context.temp_allocator) - if len(triplet) == 3 { - major, major_ok := strconv.parse_int(triplet[0]) - minor, minor_ok := strconv.parse_int(triplet[1]) - patch, patch_ok := strconv.parse_int(triplet[2]) - if major_ok && minor_ok && patch_ok { - os_version.major = major - os_version.minor = minor - os_version.patch = patch + release_str := string(b.buf[release_i:]) + + os_version.as_string = strings.to_string(b) + + // Parse the Linux version out of the release string + { + version_num, _, version_suffix := strings.partition(release_str, "-") + os_version.version = version_suffix + + i: int + for part in strings.split_iterator(&version_num, ".") { + defer i += 1 + + dst: ^int + switch i { + case 0: dst = &os_version.major + case 1: dst = &os_version.minor + case 2: dst = &os_version.patch + case: break + } + + num, ok := strconv.parse_int(part) + if !ok { break } + + dst^ = num } } - // Finish the string - os_version.as_string = strings.to_string(b) } -@(init) +@(init, private) init_ram :: proc() { // Retrieve RAM info using `sysinfo` sys_info: linux.Sys_Info @@ -84,4 +95,4 @@ init_ram :: proc() { total_swap = int(sys_info.totalswap) * int(sys_info.mem_unit), free_swap = int(sys_info.freeswap) * int(sys_info.mem_unit), } -} \ No newline at end of file +} diff --git a/core/sys/info/platform_windows.odin b/core/sys/info/platform_windows.odin index 250f938b1..4c00ddadf 100644 --- a/core/sys/info/platform_windows.odin +++ b/core/sys/info/platform_windows.odin @@ -1,4 +1,3 @@ -// +build windows package sysinfo import sys "core:sys/windows" @@ -259,7 +258,7 @@ init_os_version :: proc () { } } -@(init) +@(init, private) init_ram :: proc() { state: sys.MEMORYSTATUSEX diff --git a/core/sys/info/sysinfo.odin b/core/sys/info/sysinfo.odin index 69f9f1584..f0262f317 100644 --- a/core/sys/info/sysinfo.odin +++ b/core/sys/info/sysinfo.odin @@ -8,6 +8,9 @@ os_version: OS_Version ram: RAM gpus: []GPU +// Only on MacOS, contains the actual MacOS version, while the `os_version` contains the kernel version. +macos_version: Version + OS_Version_Platform :: enum { Unknown, Windows, @@ -19,12 +22,14 @@ OS_Version_Platform :: enum { NetBSD, } +Version :: struct { + major, minor, patch: int, +} + OS_Version :: struct { platform: OS_Version_Platform, - major: int, - minor: int, - patch: int, + using _: Version, build: [2]int, version: string, @@ -42,4 +47,4 @@ GPU :: struct { vendor_name: string, model_name: string, total_ram: int, -} \ No newline at end of file +} diff --git a/core/sys/linux/bits.odin b/core/sys/linux/bits.odin index ad519e1cd..4db689643 100644 --- a/core/sys/linux/bits.odin +++ b/core/sys/linux/bits.odin @@ -145,26 +145,29 @@ Errno :: enum i32 { } /* - Bits for Open_Flags + Bits for Open_Flags. + + RDONLY flag is not present, because it has the value of 0, i.e. it is the + default, unless WRONLY or RDWR is specified. */ Open_Flags_Bits :: enum { - RDONLY = 0, - WRONLY = 1, - RDWR = 2, - CREAT = 6, - EXCL = 7, - NOCTTY = 8, - TRUNC = 9, - APPEND = 10, - NONBLOCK = 11, - DSYNC = 12, - ASYNC = 13, - DIRECT = 14, - DIRECTORY = 16, - NOFOLLOW = 17, - NOATIME = 18, - CLOEXEC = 19, - PATH = 21, + WRONLY = 0, + RDWR = 1, + CREAT = 8, + EXCL = 9, + NOCTTY = 10, + TRUNC = 11, + APPEND = 12, + NONBLOCK = 14, + DSYNC = 16, + ASYNC = 17, + DIRECT = 18, + LARGEFILE = 20, + DIRECTORY = 21, + NOFOLLOW = 22, + NOATIME = 24, + CLOEXEC = 25, + PATH = 28, } /* diff --git a/core/sys/linux/sys.odin b/core/sys/linux/sys.odin index 869ce88e3..63fb3b776 100644 --- a/core/sys/linux/sys.odin +++ b/core/sys/linux/sys.odin @@ -40,10 +40,10 @@ write :: proc "contextless" (fd: Fd, buf: []u8) -> (int, Errno) { */ open :: proc "contextless" (name: cstring, flags: Open_Flags, mode: Mode = {}) -> (Fd, Errno) { when ODIN_ARCH == .arm64 { - ret := syscall(SYS_openat, AT_FDCWD, transmute(uintptr) name, transmute(u32) mode) + ret := syscall(SYS_openat, AT_FDCWD, transmute(uintptr) name, transmute(u32) flags, transmute(u32) mode) return errno_unwrap(ret, Fd) } else { - ret := syscall(SYS_open, transmute(uintptr) name, transmute(u32) mode) + ret := syscall(SYS_open, transmute(uintptr) name, transmute(u32) flags, transmute(u32) mode) return errno_unwrap(ret, Fd) } } @@ -91,10 +91,10 @@ stat :: proc "contextless" (filename: cstring, stat: ^Stat) -> (Errno) { */ fstat :: proc "contextless" (fd: Fd, stat: ^Stat) -> (Errno) { when size_of(int) == 8 { - ret := syscall(SYS_fstat, stat) + ret := syscall(SYS_fstat, cast(i32) fd, stat) return Errno(-ret) } else { - ret := syscall(SYS_fstat64, stat) + ret := syscall(SYS_fstat64, cast(i32) fd, stat) return Errno(-ret) } } @@ -787,8 +787,8 @@ exit :: proc "contextless" (code: i32) -> ! { Wait for the process to change state. Available since Linux 1.0. */ -wait4 :: proc "contextless" (pid: Pid, status: ^u32, options: Wait_Options) -> (Pid, Errno) { - ret := syscall(SYS_wait4, pid, status, transmute(u32) options) +wait4 :: proc "contextless" (pid: Pid, status: ^u32, options: Wait_Options, rusage: ^RUsage) -> (Pid, Errno) { + ret := syscall(SYS_wait4, pid, status, transmute(u32) options, rusage) return errno_unwrap(ret, Pid) } diff --git a/core/sys/unix/signal_netbsd.odin b/core/sys/unix/signal_netbsd.odin deleted file mode 100644 index c32f0bfbe..000000000 --- a/core/sys/unix/signal_netbsd.odin +++ /dev/null @@ -1,30 +0,0 @@ -package unix - -import "core:c" - -foreign import libc "system:c" - -ERROR_NONE :: 0 -EAGAIN :: 35 - -SIGCONT :: 19 - -SIG_BLOCK :: 1 -SIG_UNBLOCK :: 2 -SIG_SETMASK :: 3 - -siginfo_t :: struct { _: [128]u8 } -sigset_t :: struct { _: [4]u32 } - -foreign libc { - @(link_name="__sigemptyset14") sigemptyset :: proc(set: ^sigset_t) -> c.int --- - @(link_name="__sigaddset14") sigaddset :: proc(set: ^sigset_t, _signal: c.int) -> c.int --- - @(link_name="__sigtimedwait50") sigtimedwait :: proc(set: ^sigset_t, info: ^siginfo_t, timeout: ^timespec) -> c.int --- - @(link_name="sigwait") sigwait :: proc(set: ^sigset_t, _signal: ^c.int) -> c.int --- - - @(private="file", link_name="__errno") get_error_location :: proc() -> ^c.int --- -} - -errno :: #force_inline proc "contextless" () -> int { - return int(get_error_location()^) -} diff --git a/core/sys/unix/syscalls_netbsd.odin b/core/sys/unix/syscalls_netbsd.odin new file mode 100644 index 000000000..e92dc6d92 --- /dev/null +++ b/core/sys/unix/syscalls_netbsd.odin @@ -0,0 +1,3 @@ +package unix + +SYS___futex : uintptr : 166 diff --git a/core/sys/unix/sysctl_darwin.odin b/core/sys/unix/sysctl_darwin.odin index 76c72f478..6417961e5 100644 --- a/core/sys/unix/sysctl_darwin.odin +++ b/core/sys/unix/sysctl_darwin.odin @@ -1,20 +1,29 @@ //+build darwin package unix -import "core:sys/darwin" import "base:intrinsics" +import "core:c" +import "core:sys/darwin" + _ :: darwin -sysctl :: proc(mib: []i32, val: ^$T) -> (ok: bool) { - mib := mib - result_size := i64(size_of(T)) +sysctl :: proc "contextless" (mib: []i32, val: ^$T) -> (ok: bool) { + result_size := c.size_t(size_of(T)) + res := darwin.syscall_sysctl( + raw_data(mib), len(mib), + val, &result_size, + nil, 0, + ) + return res == 0 +} - res := intrinsics.syscall( - darwin.unix_offset_syscall(.sysctl), - uintptr(raw_data(mib)), uintptr(len(mib)), - uintptr(val), uintptr(&result_size), - uintptr(0), uintptr(0), +sysctlbyname :: proc "contextless" (name: string, val: ^$T) -> (ok: bool) { + result_size := c.size_t(size_of(T)) + res := darwin.syscall_sysctlbyname( + name, + val, &result_size, + nil, 0, ) return res == 0 } diff --git a/core/sys/unix/sysctl_netbsd.odin b/core/sys/unix/sysctl_netbsd.odin new file mode 100644 index 000000000..ad89b9ad4 --- /dev/null +++ b/core/sys/unix/sysctl_netbsd.odin @@ -0,0 +1,44 @@ +package unix + +import "core:c" +foreign import libc "system:c" + +@(default_calling_convention="c") +foreign libc { + @(link_name="sysctl") _unix_sysctl :: proc(name: [^]i32, namelen: u32, oldp: rawptr, oldlenp: ^c.size_t, newp: rawptr, newlen: c.size_t) -> i32 --- +} + +sysctl :: proc(mib: []i32, val: ^$T) -> (ok: bool) { + mib := mib + result_size := c.size_t(size_of(T)) + res := _unix_sysctl(raw_data(mib), u32(len(mib)), val, &result_size, nil, 0) + return res == 0 +} + +// See /usr/include/sys/sysctl.h for details +CTL_KERN :: 1 + KERN_OSTYPE :: 1 + KERN_OSRELEASE :: 2 + KERN_OSREV :: 3 + KERN_VERSION :: 4 +CTL_VM :: 2 +CTL_FS :: 3 +CTL_NET :: 4 +CTL_DEBUG :: 5 +CTL_HW :: 6 + HW_MACHINE :: 1 + HW_MODEL :: 2 + HW_NCPU :: 3 + HW_BYTEORDER :: 4 + HW_PHYSMEM :: 5 + HW_USERMEM :: 6 + HW_PAGESIZE :: 7 + HW_DISKNAMES :: 8 + HW_IOSTATS :: 9 + HW_MACHINE_ARCH :: 10 + HW_ALIGNBYTES :: 11 + HW_CNMAGIC :: 12 + HW_PHYSMEM64 :: 13 + HW_USERMEM64 :: 14 + HW_IOSTATNAMES :: 15 + HW_NCPUONLINE :: 16 diff --git a/core/sys/windows/dbghelp.odin b/core/sys/windows/dbghelp.odin index c2e506748..cb5458248 100644 --- a/core/sys/windows/dbghelp.odin +++ b/core/sys/windows/dbghelp.odin @@ -228,6 +228,38 @@ MINIDUMP_TYPE :: enum u32 { ValidTypeFlags = 0x01ffffff, } + +SYMBOL_INFOW :: struct { + SizeOfStruct: ULONG, + TypeIndex: ULONG, + Reserved: [2]ULONG64, + Index: ULONG, + Size: ULONG, + ModBase: ULONG64, + Flags: ULONG, + Value: ULONG64, + Address: ULONG64, + Register: ULONG, + Scope: ULONG, + Tag: ULONG, + NameLen: ULONG, + MaxNameLen: ULONG, + Name: [1]WCHAR, +} + +IMAGEHLP_LINE64 :: struct { + SizeOfStruct: DWORD, + Key: PVOID, + LineNumber: DWORD, + FileName: PWSTR, + Address: DWORD64, +} + +PSYMBOL_INFOW :: ^SYMBOL_INFOW +PIMAGEHLP_LINEW64 :: ^IMAGEHLP_LINE64 + +SYMOPT_LOAD_LINES :: 0x00000010 + @(default_calling_convention = "system") foreign Dbghelp { MiniDumpWriteDump :: proc( @@ -247,4 +279,10 @@ foreign Dbghelp { StreamPointer: ^PVOID, StreamSize: ^ULONG, ) -> BOOL --- + + SymInitialize :: proc(hProcess: HANDLE, UserSearchPath: PCSTR, fInvadeProcess: BOOL) -> BOOL --- + SymCleanup :: proc(hProcess: HANDLE) -> BOOL --- + SymSetOptions :: proc(SymOptions: DWORD) -> DWORD --- + SymFromAddrW :: proc(hProcess: HANDLE, Address: DWORD64, Displacement: PDWORD64, Symbol: PSYMBOL_INFOW) -> BOOL --- + SymGetLineFromAddrW64 :: proc(hProcess: HANDLE, dwAddr: DWORD64, pdwDisplacement: PDWORD, Line: PIMAGEHLP_LINEW64) -> BOOL --- } diff --git a/core/sys/windows/kernel32.odin b/core/sys/windows/kernel32.odin index 10cc80041..f998f33da 100644 --- a/core/sys/windows/kernel32.odin +++ b/core/sys/windows/kernel32.odin @@ -460,6 +460,8 @@ foreign kernel32 { PostQueuedCompletionStatus :: proc(CompletionPort: HANDLE, dwNumberOfBytesTransferred: DWORD, dwCompletionKey: c_ulong, lpOverlapped: ^OVERLAPPED) -> BOOL --- // [MS-Docs](https://learn.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-gethandleinformation) GetHandleInformation :: proc(hObject: HANDLE, lpdwFlags: ^DWORD) -> BOOL --- + + RtlCaptureStackBackTrace :: proc(FramesToSkip: ULONG, FramesToCapture: ULONG, BackTrace: [^]PVOID, BackTraceHash: PULONG) -> USHORT --- } DEBUG_PROCESS :: 0x00000001 diff --git a/core/sys/windows/types.odin b/core/sys/windows/types.odin index 4b54f0ed1..11d2774d6 100644 --- a/core/sys/windows/types.odin +++ b/core/sys/windows/types.odin @@ -93,10 +93,14 @@ LONG32 :: i32 ULONG64 :: u64 LONG64 :: i64 +DWORD64 :: u64 +PDWORD64 :: ^DWORD64 + PDWORD_PTR :: ^DWORD_PTR ATOM :: distinct WORD wstring :: [^]WCHAR +PWSTR :: [^]WCHAR PBYTE :: ^BYTE LPBYTE :: ^BYTE diff --git a/core/sys/windows/user32.odin b/core/sys/windows/user32.odin index a589c3ec9..cb4091058 100644 --- a/core/sys/windows/user32.odin +++ b/core/sys/windows/user32.odin @@ -126,6 +126,7 @@ foreign user32 { CreatePopupMenu :: proc() -> HMENU --- DestroyMenu :: proc(hMenu: HMENU) -> BOOL --- AppendMenuW :: proc(hMenu: HMENU, uFlags: UINT, uIDNewItem: UINT_PTR, lpNewItem: LPCWSTR) -> BOOL --- + SetMenu :: proc(hWnd: HWND, hMenu: HMENU) -> BOOL --- TrackPopupMenu :: proc(hMenu: HMENU, uFlags: UINT, x: int, y: int, nReserved: int, hWnd: HWND, prcRect: ^RECT) -> i32 --- RegisterWindowMessageW :: proc(lpString: LPCWSTR) -> UINT --- diff --git a/core/text/edit/text_edit.odin b/core/text/edit/text_edit.odin index caccb6be8..6f21c9860 100644 --- a/core/text/edit/text_edit.odin +++ b/core/text/edit/text_edit.odin @@ -137,6 +137,9 @@ clear_all :: proc(s: ^State) -> (cleared: bool) { // push current text state to the wanted undo|redo stack undo_state_push :: proc(s: ^State, undo: ^[dynamic]^Undo_State) -> mem.Allocator_Error { + if s.builder != nil { + return nil + } text := string(s.builder.buf[:]) item := (^Undo_State)(mem.alloc(size_of(Undo_State) + len(text), align_of(Undo_State), s.undo_text_allocator) or_return) item.selection = s.selection @@ -154,7 +157,7 @@ undo :: proc(s: ^State, undo, redo: ^[dynamic]^Undo_State) { undo_state_push(s, redo) item := pop(undo) s.selection = item.selection - #no_bounds_check { + #no_bounds_check if s.builder != nil { strings.builder_reset(s.builder) strings.write_string(s.builder, string(item.text[:item.len])) } @@ -224,13 +227,17 @@ input_rune :: proc(s: ^State, r: rune) { // insert a single rune into the edit state - deletes the current selection insert :: proc(s: ^State, at: int, text: string) { undo_check(s) - inject_at(&s.builder.buf, at, text) + if s.builder != nil { + inject_at(&s.builder.buf, at, text) + } } // remove the wanted range withing, usually the selection within byte indices remove :: proc(s: ^State, lo, hi: int) { undo_check(s) - remove_range(&s.builder.buf, lo, hi) + if s.builder != nil { + remove_range(&s.builder.buf, lo, hi) + } } // true if selection head and tail dont match and form a selection of multiple characters @@ -244,8 +251,8 @@ has_selection :: proc(s: ^State) -> bool { sorted_selection :: proc(s: ^State) -> (lo, hi: int) { lo = min(s.selection[0], s.selection[1]) hi = max(s.selection[0], s.selection[1]) - lo = clamp(lo, 0, len(s.builder.buf)) - hi = clamp(hi, 0, len(s.builder.buf)) + lo = clamp(lo, 0, len(s.builder.buf) if s.builder != nil else 0) + hi = clamp(hi, 0, len(s.builder.buf) if s.builder != nil else 0) return } @@ -265,7 +272,10 @@ translate_position :: proc(s: ^State, t: Translation) -> int { return b == ' ' || b == '\t' || b == '\n' } - buf := s.builder.buf[:] + buf: []byte + if s.builder != nil { + buf = s.builder.buf[:] + } pos := clamp(s.selection[0], 0, len(buf)) switch t { @@ -352,7 +362,10 @@ delete_to :: proc(s: ^State, t: Translation) { // return the currently selected text current_selected_text :: proc(s: ^State) -> string { lo, hi := sorted_selection(s) - return string(s.builder.buf[lo:hi]) + if s.builder != nil { + return string(s.builder.buf[lo:hi]) + } + return "" } // copy & delete the current selection when copy() succeeds @@ -431,7 +444,7 @@ perform_command :: proc(s: ^State, cmd: Command) { case .Cut: cut(s) case .Copy: copy(s) case .Paste: paste(s) - case .Select_All: s.selection = {len(s.builder.buf), 0} + case .Select_All: s.selection = {len(s.builder.buf) if s.builder != nil else 0, 0} case .Backspace: delete_to(s, .Left) case .Delete: delete_to(s, .Right) case .Delete_Word_Left: delete_to(s, .Word_Left) diff --git a/core/text/i18n/doc.odin b/core/text/i18n/doc.odin index ef619451e..54bf8b80f 100644 --- a/core/text/i18n/doc.odin +++ b/core/text/i18n/doc.odin @@ -1,111 +1,106 @@ -//+build ignore -package i18n /* - The i18n package is flexible and easy to use. +The `i18n` package is flexible and easy to use. - It has one call to get a translation: `get`, which the user can alias into something like `T`. +It has one call to get a translation: `get`, which the user can alias into something like `T`. - `get`, referred to as `T` here, has a few different signatures. - All of them will return the key if the entry can't be found in the active translation catalog. +`get`, referred to as `T` here, has a few different signatures. +All of them will return the key if the entry can't be found in the active translation catalog. - - `T(key)` returns the translation of `key`. - - `T(key, n)` returns a pluralized translation of `key` according to value `n`. +- `T(key)` returns the translation of `key`. +- `T(key, n)` returns a pluralized translation of `key` according to value `n`. - - `T(section, key)` returns the translation of `key` in `section`. - - `T(section, key, n)` returns a pluralized translation of `key` in `section` according to value `n`. +- `T(section, key)` returns the translation of `key` in `section`. +- `T(section, key, n)` returns a pluralized translation of `key` in `section` according to value `n`. - By default lookup take place in the global `i18n.ACTIVE` catalog for ease of use. - If you want to override which translation to use, for example in a language preview dialog, you can use the following: +By default lookup take place in the global `i18n.ACTIVE` catalog for ease of use. +If you want to override which translation to use, for example in a language preview dialog, you can use the following: - - `T(key, n, catalog)` returns the pluralized version of `key` from explictly supplied catalog. - - `T(section, key, n, catalog)` returns the pluralized version of `key` in `section` from explictly supplied catalog. +- `T(key, n, catalog)` returns the pluralized version of `key` from explictly supplied catalog. +- `T(section, key, n, catalog)` returns the pluralized version of `key` in `section` from explictly supplied catalog. - If a catalog has translation contexts or sections, then ommitting it in the above calls looks up in section "". +If a catalog has translation contexts or sections, then omitting it in the above calls looks up in section "". - The default pluralization rule is n != 1, which is to say that passing n == 1 (or not passing n) returns the singular form. - Passing n != 1 returns plural form 1. +The default pluralization rule is n != 1, which is to say that passing n == 1 (or not passing n) returns the singular form. +Passing n != 1 returns plural form 1. - Should a language not conform to this rule, you can pass a pluralizer procedure to the catalog parser. - This is a procedure that maps an integer to an integer, taking a value and returning which plural slot should be used. +Should a language not conform to this rule, you can pass a pluralizer procedure to the catalog parser. +This is a procedure that maps an integer to an integer, taking a value and returning which plural slot should be used. - You can also assign it to a loaded catalog after parsing, of course. +You can also assign it to a loaded catalog after parsing, of course. - Some code examples follow. -*/ +Example: -/* -```cpp -import "core:fmt" -import "core:text/i18n" + import "core:fmt" + import "core:text/i18n" -T :: i18n.get + T :: i18n.get -mo :: proc() { - using fmt + mo :: proc() { + using fmt - err: i18n.Error + err: i18n.Error - /* - Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter. - */ - i18n.ACTIVE, err = i18n.parse_mo(#load("translations/nl_NL.mo")) - defer i18n.destroy() + /* + Parse MO file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_mo(#load("translations/nl_NL.mo")) + defer i18n.destroy() - if err != .None { return } + if err != .None { return } - /* - These are in the .MO catalog. - */ - println("-----") - println(T("")) - println("-----") - println(T("There are 69,105 leaves here.")) - println("-----") - println(T("Hellope, World!")) - println("-----") - // We pass 1 into `T` to get the singular format string, then 1 again into printf. - printf(T("There is %d leaf.\n", 1), 1) - // We pass 42 into `T` to get the plural format string, then 42 again into printf. - printf(T("There is %d leaf.\n", 42), 42) + /* + These are in the .MO catalog. + */ + println("-----") + println(T("")) + println("-----") + println(T("There are 69,105 leaves here.")) + println("-----") + println(T("Hellope, World!")) + println("-----") + // We pass 1 into `T` to get the singular format string, then 1 again into printf. + printf(T("There is %d leaf.\n", 1), 1) + // We pass 42 into `T` to get the plural format string, then 42 again into printf. + printf(T("There is %d leaf.\n", 42), 42) - /* - This isn't in the translation catalog, so the key is passed back untranslated. - */ - println("-----") - println(T("Come visit us on Discord!")) -} - -qt :: proc() { - using fmt - - err: i18n.Error - - /* - Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. - */ - i18n.ACTIVE, err = i18n.parse_qt(#load("translations/nl_NL-qt-ts.ts")) - defer i18n.destroy() - - if err != .None { - return + /* + This isn't in the translation catalog, so the key is passed back untranslated. + */ + println("-----") + println(T("Come visit us on Discord!")) } - /* - These are in the .TS catalog. As you can see they have sections. - */ - println("--- Page section ---") - println("Page:Text for translation =", T("Page", "Text for translation")) - println("-----") - println("Page:Also text to translate =", T("Page", "Also text to translate")) - println("-----") - println("--- installscript section ---") - println("installscript:99 bottles of beer on the wall =", T("installscript", "99 bottles of beer on the wall")) - println("-----") - println("--- apple_count section ---") - println("apple_count:%d apple(s) =") - println("\t 1 =", T("apple_count", "%d apple(s)", 1)) - println("\t 42 =", T("apple_count", "%d apple(s)", 42)) -} -``` -*/ \ No newline at end of file + qt :: proc() { + using fmt + + err: i18n.Error + + /* + Parse QT file and set it as the active translation so we can omit `get`'s "catalog" parameter. + */ + i18n.ACTIVE, err = i18n.parse_qt(#load("translations/nl_NL-qt-ts.ts")) + defer i18n.destroy() + + if err != .None { + return + } + + /* + These are in the .TS catalog. As you can see they have sections. + */ + println("--- Page section ---") + println("Page:Text for translation =", T("Page", "Text for translation")) + println("-----") + println("Page:Also text to translate =", T("Page", "Also text to translate")) + println("-----") + println("--- installscript section ---") + println("installscript:99 bottles of beer on the wall =", T("installscript", "99 bottles of beer on the wall")) + println("-----") + println("--- apple_count section ---") + println("apple_count:%d apple(s) =") + println("\t 1 =", T("apple_count", "%d apple(s)", 1)) + println("\t 42 =", T("apple_count", "%d apple(s)", 42)) + } +*/ +package i18n diff --git a/core/text/i18n/i18n.odin b/core/text/i18n/i18n.odin index 151f9e129..64593c4e8 100644 --- a/core/text/i18n/i18n.odin +++ b/core/text/i18n/i18n.odin @@ -90,7 +90,7 @@ DEFAULT_PARSE_OPTIONS :: Parse_Options{ - get(key, number), which returns the appropriate plural from the active catalog, or - get(key, number, catalog) to grab text from a specific one. */ -get_single_section :: proc(key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) { +get_single_section :: proc(key: string, number := 1, catalog: ^Translation = ACTIVE) -> (value: string) { /* A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule. */ @@ -108,7 +108,7 @@ get_single_section :: proc(key: string, number := 0, catalog: ^Translation = ACT - get(section, key, number), which returns the appropriate plural from the active catalog, or - get(section, key, number, catalog) to grab text from a specific one. */ -get_by_section :: proc(section, key: string, number := 0, catalog: ^Translation = ACTIVE) -> (value: string) { +get_by_section :: proc(section, key: string, number := 1, catalog: ^Translation = ACTIVE) -> (value: string) { /* A lot of languages use singular for 1 item and plural for 0 or more than 1 items. This is our default pluralize rule. */ diff --git a/core/text/scanner/scanner.odin b/core/text/scanner/scanner.odin index 7c17a0ec0..d27c66f24 100644 --- a/core/text/scanner/scanner.odin +++ b/core/text/scanner/scanner.odin @@ -8,6 +8,7 @@ // A Scanner may be customized to recognize only a subset of those literals and to recognize different identifiers and white space characters. package text_scanner +import "base:runtime" import "core:fmt" import "core:strings" import "core:unicode" @@ -23,10 +24,12 @@ Position :: struct { } // position_is_valid reports where the position is valid +@(require_results) position_is_valid :: proc(pos: Position) -> bool { return pos.line > 0 } +@(require_results) position_to_string :: proc(pos: Position, allocator := context.temp_allocator) -> string { s := pos.filename if s == "" { @@ -140,7 +143,7 @@ init :: proc(s: ^Scanner, src: string, filename := "") -> ^Scanner { } -@(private) +@(private, require_results) advance :: proc(s: ^Scanner) -> rune { if s.src_pos >= len(s.src) { s.prev_char_len = 0 @@ -190,6 +193,7 @@ next :: proc(s: ^Scanner) -> rune { // peek returns the next Unicode character in the source without advancing the scanner // It returns EOF if the scanner's position is at least the last character of the source // if n > 0, it call next n times and return the nth Unicode character and then restore the Scanner's state +@(require_results) peek :: proc(s: ^Scanner, n := 0) -> (ch: rune) { if s.ch == -2 { s.ch = advance(s) @@ -211,6 +215,7 @@ peek :: proc(s: ^Scanner, n := 0) -> (ch: rune) { // peek returns the next token in the source // It returns EOF if the scanner's position is at least the last character of the source // if n > 0, it call next n times and return the nth token and then restore the Scanner's state +@(require_results) peek_token :: proc(s: ^Scanner, n := 0) -> (tok: rune) { assert(n >= 0) prev_s := s^ @@ -249,7 +254,7 @@ errorf :: proc(s: ^Scanner, format: string, args: ..any) { error(s, fmt.tprintf(format, ..args)) } -@(private) +@(private, require_results) is_ident_rune :: proc(s: ^Scanner, ch: rune, i: int) -> bool { if s.is_ident_rune != nil { return s.is_ident_rune(ch, i) @@ -257,7 +262,7 @@ is_ident_rune :: proc(s: ^Scanner, ch: rune, i: int) -> bool { return ch == '_' || unicode.is_letter(ch) || unicode.is_digit(ch) && i > 0 } -@(private) +@(private, require_results) scan_identifier :: proc(s: ^Scanner) -> rune { ch := advance(s) for i := 1; is_ident_rune(s, ch, i); i += 1 { @@ -266,13 +271,13 @@ scan_identifier :: proc(s: ^Scanner) -> rune { return ch } -@(private) lower :: proc(ch: rune) -> rune { return ('a' - 'A') | ch } -@(private) is_decimal :: proc(ch: rune) -> bool { return '0' <= ch && ch <= '9' } -@(private) is_hex :: proc(ch: rune) -> bool { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' } +@(private, require_results) lower :: proc(ch: rune) -> rune { return ('a' - 'A') | ch } +@(private, require_results) is_decimal :: proc(ch: rune) -> bool { return '0' <= ch && ch <= '9' } +@(private, require_results) is_hex :: proc(ch: rune) -> bool { return '0' <= ch && ch <= '9' || 'a' <= lower(ch) && lower(ch) <= 'f' } -@(private) +@(private, require_results) scan_number :: proc(s: ^Scanner, ch: rune, seen_dot: bool) -> (rune, rune) { lit_name :: proc(prefix: rune) -> string { switch prefix { @@ -417,7 +422,7 @@ scan_number :: proc(s: ^Scanner, ch: rune, seen_dot: bool) -> (rune, rune) { return tok, ch } -@(private) +@(private, require_results) scan_string :: proc(s: ^Scanner, quote: rune) -> (n: int) { digit_val :: proc(ch: rune) -> int { switch v := lower(ch); v { @@ -484,7 +489,7 @@ scan_char :: proc(s: ^Scanner) { } } -@(private) +@(private, require_results) scan_comment :: proc(s: ^Scanner, ch: rune) -> rune { ch := ch if ch == '/' { // line comment @@ -563,13 +568,13 @@ scan :: proc(s: ^Scanner) -> (tok: rune) { break case '"': if .Scan_Strings in s.flags { - scan_string(s, '"') + _ = scan_string(s, '"') tok = String } ch = advance(s) case '\'': if .Scan_Chars in s.flags { - scan_string(s, '\'') + _ = scan_string(s, '\'') tok = Char } ch = advance(s) @@ -611,6 +616,7 @@ scan :: proc(s: ^Scanner) -> (tok: rune) { // position returns the position of the character immediately after the character or token returns by the previous call to next or scan // Use the Scanner's position field for the most recently scanned token position +@(require_results) position :: proc(s: ^Scanner) -> Position { pos: Position pos.filename = s.pos.filename @@ -630,6 +636,7 @@ position :: proc(s: ^Scanner) -> Position { } // token_text returns the string of the most recently scanned token +@(require_results) token_text :: proc(s: ^Scanner) -> string { if s.tok_pos < 0 { return "" @@ -639,7 +646,8 @@ token_text :: proc(s: ^Scanner) -> string { // token_string returns a printable string for a token or Unicode character // By default, it uses the context.temp_allocator to produce the string -token_string :: proc(tok: rune, allocator := context.temp_allocator) -> string { +@(require_results) +token_string :: proc(tok: rune, allocator: runtime.Allocator) -> string { context.allocator = allocator switch tok { case EOF: return strings.clone("EOF") diff --git a/core/text/table/doc.odin b/core/text/table/doc.odin index 9b5c1f932..76886bdea 100644 --- a/core/text/table/doc.odin +++ b/core/text/table/doc.odin @@ -1,11 +1,8 @@ /* - package table implements ascii/markdown/html/custom rendering of tables. +The package `table` implements ASCII/markdown/HTML/custom rendering of tables. - --- +**Custom rendering example:** - Custom rendering example: - - ```odin tbl := init(&Table{}) padding(tbl, 0, 1) row(tbl, "A_LONG_ENUM", "= 54,", "// A comment about A_LONG_ENUM") @@ -17,19 +14,14 @@ } io.write_byte(stdio_writer(), '\n') } - ``` - This outputs: - ``` +This outputs: + A_LONG_ENUM = 54, // A comment about A_LONG_ENUM AN_EVEN_LONGER_ENUM = 1, // A comment about AN_EVEN_LONGER_ENUM - ``` - --- +**ASCII rendering example:** - ASCII rendering example: - - ```odin tbl := init(&Table{}) defer destroy(tbl) @@ -69,10 +61,9 @@ write_ascii_table(stdio_writer(), tbl) write_markdown_table(stdio_writer(), tbl) - ``` - This outputs: - ``` +This outputs: + +-----------------------------------------------+ | This is a table caption and it is very long | +------------------+-----------------+----------+ @@ -82,19 +73,15 @@ | 000000005 | 6.283185 | | | a | bbb | c | +------------------+-----------------+----------+ - ``` - and +and - ``` | AAAAAAAAA | B | C | |:-----------------|:---------------:|---------:| | 123 | foo | | | 000000005 | 6.283185 | | | a | bbb | c | - ``` - respectively. +respectively. */ - package text_table diff --git a/core/time/tsc_darwin.odin b/core/time/tsc_darwin.odin index 6688ae7d8..841c0b692 100644 --- a/core/time/tsc_darwin.odin +++ b/core/time/tsc_darwin.odin @@ -1,21 +1,10 @@ //+private -//+build darwin package time -import "core:c" +import "core:sys/unix" -foreign import libc "system:System.framework" -foreign libc { - @(link_name="sysctlbyname") _sysctlbyname :: proc(path: cstring, oldp: rawptr, oldlenp: rawptr, newp: rawptr, newlen: int) -> c.int --- -} - -_get_tsc_frequency :: proc "contextless" () -> (u64, bool) { - tmp_freq : u64 = 0 - tmp_size : i64 = size_of(tmp_freq) - ret := _sysctlbyname("machdep.tsc.frequency", &tmp_freq, &tmp_size, nil, 0) - if ret < 0 { - return 0, false - } - - return tmp_freq, true +_get_tsc_frequency :: proc "contextless" () -> (freq: u64, ok: bool) { + unix.sysctlbyname("machdep.tsc.frequency", &freq) or_return + ok = true + return } diff --git a/examples/all/all_main.odin b/examples/all/all_main.odin index f60088823..3eebdf0b0 100644 --- a/examples/all/all_main.odin +++ b/examples/all/all_main.odin @@ -59,6 +59,7 @@ import json "core:encoding/json" import varint "core:encoding/varint" import xml "core:encoding/xml" import endian "core:encoding/endian" +import cbor "core:encoding/cbor" import fmt "core:fmt" import hash "core:hash" @@ -180,6 +181,7 @@ _ :: json _ :: varint _ :: xml _ :: endian +_ :: cbor _ :: fmt _ :: hash _ :: xxhash diff --git a/examples/all/all_vendor.odin b/examples/all/all_vendor.odin index 9093e0f3e..1ab1debea 100644 --- a/examples/all/all_vendor.odin +++ b/examples/all/all_vendor.odin @@ -73,10 +73,12 @@ _ :: MTK _ :: CA +import DXC "vendor:directx/dxc" import D3D11 "vendor:directx/d3d11" import D3D12 "vendor:directx/d3d12" import DXGI "vendor:directx/dxgi" +_ :: DXC _ :: D3D11 _ :: D3D12 _ :: DXGI diff --git a/examples/demo/demo.odin b/examples/demo/demo.odin index b2736ffcd..a62c11310 100644 --- a/examples/demo/demo.odin +++ b/examples/demo/demo.odin @@ -7,7 +7,7 @@ import "core:os" import "core:thread" import "core:time" import "core:reflect" -import "core:runtime" +import "base:runtime" import "core:intrinsics" import "core:math/big" @@ -2385,7 +2385,7 @@ matrix_type :: proc() { c := a * b #assert(type_of(c) == matrix[2, 2]f32) - fmt.tprintln("c = a * b", c) + fmt.println("c = a * b", c) } { // Matrices support multiplication between matrices and arrays diff --git a/src/bug_report.cpp b/src/bug_report.cpp index 0ec383b44..c73595e99 100644 --- a/src/bug_report.cpp +++ b/src/bug_report.cpp @@ -204,14 +204,27 @@ gb_internal void report_cpu_info() { } #elif defined(GB_CPU_ARM) - /* - TODO(Jeroen): On *nix, perhaps query `/proc/cpuinfo`. - */ - #if defined(GB_ARCH_64_BIT) - gb_printf("ARM64\n"); - #else - gb_printf("ARM\n"); + bool generic = true; + + #if defined(GB_SYSTEM_OSX) + char cpu_name[128] = {}; + size_t cpu_name_size = 128; + if (sysctlbyname("machdep.cpu.brand_string", &cpu_name, &cpu_name_size, nullptr, 0) == 0) { + generic = false; + gb_printf("%s\n", (char *)&cpu_name[0]); + } #endif + + if (generic) { + /* + TODO(Jeroen): On *nix, perhaps query `/proc/cpuinfo`. + */ + #if defined(GB_ARCH_64_BIT) + gb_printf("ARM64\n"); + #else + gb_printf("ARM\n"); + #endif + } #else gb_printf("Unknown\n"); #endif diff --git a/src/build_settings.cpp b/src/build_settings.cpp index cd57816ab..b63f08980 100644 --- a/src/build_settings.cpp +++ b/src/build_settings.cpp @@ -274,13 +274,16 @@ enum BuildPath : u8 { }; enum VetFlags : u64 { - VetFlag_NONE = 0, - VetFlag_Unused = 1u<<0, // 1 - VetFlag_Shadowing = 1u<<1, // 2 - VetFlag_UsingStmt = 1u<<2, // 4 - VetFlag_UsingParam = 1u<<3, // 8 - VetFlag_Style = 1u<<4, // 16 - VetFlag_Semicolon = 1u<<5, // 32 + VetFlag_NONE = 0, + VetFlag_Shadowing = 1u<<0, + VetFlag_UsingStmt = 1u<<1, + VetFlag_UsingParam = 1u<<2, + VetFlag_Style = 1u<<3, + VetFlag_Semicolon = 1u<<4, + VetFlag_UnusedVariables = 1u<<5, + VetFlag_UnusedImports = 1u<<6, + + VetFlag_Unused = VetFlag_UnusedVariables|VetFlag_UnusedImports, VetFlag_All = VetFlag_Unused|VetFlag_Shadowing|VetFlag_UsingStmt, @@ -290,6 +293,10 @@ enum VetFlags : u64 { u64 get_vet_flag_from_name(String const &name) { if (name == "unused") { return VetFlag_Unused; + } else if (name == "unused-variables") { + return VetFlag_UnusedVariables; + } else if (name == "unused-imports") { + return VetFlag_UnusedImports; } else if (name == "shadowing") { return VetFlag_Shadowing; } else if (name == "using-stmt") { @@ -377,6 +384,7 @@ struct BuildContext { bool keep_temp_files; bool ignore_unknown_attributes; bool no_bounds_check; + bool no_type_assert; bool no_dynamic_literals; bool no_output_files; bool no_crt; @@ -634,6 +642,15 @@ gb_global TargetMetrics target_freestanding_amd64_sysv = { TargetABI_SysV, }; +gb_global TargetMetrics target_freestanding_amd64_win64 = { + TargetOs_freestanding, + TargetArch_amd64, + 8, 8, 8, 16, + str_lit("x86_64-pc-none-msvc"), + str_lit("e-m:w-i64:64-f80:128-n8:16:32:64-S128"), + TargetABI_Win64, +}; + gb_global TargetMetrics target_freestanding_arm64 = { TargetOs_freestanding, TargetArch_arm64, @@ -676,7 +693,9 @@ gb_global NamedTargetMetrics named_targets[] = { { str_lit("js_wasm64p32"), &target_js_wasm64p32 }, { str_lit("wasi_wasm64p32"), &target_wasi_wasm64p32 }, - { str_lit("freestanding_amd64_sysv"), &target_freestanding_amd64_sysv }, + { str_lit("freestanding_amd64_sysv"), &target_freestanding_amd64_sysv }, + { str_lit("freestanding_amd64_win64"), &target_freestanding_amd64_win64 }, + { str_lit("freestanding_arm64"), &target_freestanding_arm64 }, }; @@ -832,13 +851,11 @@ gb_internal String odin_root_dir(void) { char const *found = gb_get_env("ODIN_ROOT", a); if (found) { String path = path_to_full_path(a, make_string_c(found)); - if (path[path.len-1] != '/' && path[path.len-1] != '\\') { #if defined(GB_SYSTEM_WINDOWS) - path = concatenate_strings(a, path, WIN32_SEPARATOR_STRING); + path = normalize_path(a, path, WIN32_SEPARATOR_STRING); #else - path = concatenate_strings(a, path, NIX_SEPARATOR_STRING); + path = normalize_path(a, path, NIX_SEPARATOR_STRING); #endif - } global_module_path = path; global_module_path_set = true; @@ -1461,26 +1478,6 @@ gb_internal void init_build_context(TargetMetrics *cross_target, Subtarget subta } bc->metrics = *metrics; - if (metrics->os == TargetOs_darwin) { - if (!bc->minimum_os_version_string_given) { - bc->minimum_os_version_string = str_lit("11.0.0"); - } - - switch (subtarget) { - case Subtarget_Default: - bc->metrics.target_triplet = concatenate_strings(permanent_allocator(), bc->metrics.target_triplet, bc->minimum_os_version_string); - break; - case Subtarget_iOS: - if (metrics->arch == TargetArch_arm64) { - bc->metrics.target_triplet = str_lit("arm64-apple-ios"); - } else if (metrics->arch == TargetArch_amd64) { - bc->metrics.target_triplet = str_lit("x86_64-apple-ios"); - } else { - GB_PANIC("Unknown architecture for darwin"); - } - break; - } - } bc->ODIN_OS = target_os_names[metrics->os]; bc->ODIN_ARCH = target_arch_names[metrics->arch]; @@ -1516,65 +1513,26 @@ gb_internal void init_build_context(TargetMetrics *cross_target, Subtarget subta bc->ODIN_WINDOWS_SUBSYSTEM = windows_subsystem_names[Windows_Subsystem_CONSOLE]; } - // NOTE(zangent): The linker flags to set the build architecture are different - // across OSs. It doesn't make sense to allocate extra data on the heap - // here, so I just #defined the linker flags to keep things concise. - if (bc->metrics.arch == TargetArch_amd64) { - switch (bc->metrics.os) { - case TargetOs_windows: - bc->link_flags = str_lit("/machine:x64 "); + if (metrics->os == TargetOs_darwin && subtarget == Subtarget_iOS) { + switch (metrics->arch) { + case TargetArch_arm64: + bc->metrics.target_triplet = str_lit("arm64-apple-ios"); break; - case TargetOs_darwin: - bc->link_flags = str_lit("-arch x86_64 "); - break; - case TargetOs_linux: - bc->link_flags = str_lit("-arch x86-64 "); - break; - case TargetOs_freebsd: - bc->link_flags = str_lit("-arch x86-64 "); - break; - case TargetOs_openbsd: - bc->link_flags = str_lit("-arch x86-64 "); - break; - case TargetOs_netbsd: - bc->link_flags = str_lit("-arch x86-64 "); - break; - case TargetOs_haiku: - bc->link_flags = str_lit("-arch x86-64 "); - break; - } - } else if (bc->metrics.arch == TargetArch_i386) { - switch (bc->metrics.os) { - case TargetOs_windows: - bc->link_flags = str_lit("/machine:x86 "); - break; - case TargetOs_darwin: - gb_printf_err("Unsupported architecture\n"); - gb_exit(1); - break; - case TargetOs_linux: - bc->link_flags = str_lit("-arch x86 "); - break; - case TargetOs_freebsd: - bc->link_flags = str_lit("-arch x86 "); - break; - } - } else if (bc->metrics.arch == TargetArch_arm32) { - switch (bc->metrics.os) { - case TargetOs_linux: - bc->link_flags = str_lit("-arch arm "); + case TargetArch_amd64: + bc->metrics.target_triplet = str_lit("x86_64-apple-ios"); break; default: - gb_printf_err("Compiler Error: Unsupported architecture\n"); - gb_exit(1); + GB_PANIC("Unknown architecture for darwin"); } - } else if (bc->metrics.arch == TargetArch_arm64) { - switch (bc->metrics.os) { - case TargetOs_darwin: - bc->link_flags = str_lit("-arch arm64 "); + } + + if (bc->metrics.os == TargetOs_windows) { + switch (bc->metrics.arch) { + case TargetArch_amd64: + bc->link_flags = str_lit("/machine:x64 "); break; - case TargetOs_linux: - bc->link_flags = str_lit("-arch aarch64 "); + case TargetArch_i386: + bc->link_flags = str_lit("/machine:x86 "); break; } } else if (is_arch_wasm()) { @@ -1594,8 +1552,20 @@ gb_internal void init_build_context(TargetMetrics *cross_target, Subtarget subta // Disallow on wasm bc->use_separate_modules = false; } else { - gb_printf_err("Compiler Error: Unsupported architecture\n"); - gb_exit(1); + bc->link_flags = concatenate3_strings(permanent_allocator(), + str_lit("-target "), bc->metrics.target_triplet, str_lit(" ")); + } + + // NOTE: needs to be done after adding the -target flag to the linker flags so the linker + // does not annoy the user with version warnings. + if (metrics->os == TargetOs_darwin) { + if (!bc->minimum_os_version_string_given) { + bc->minimum_os_version_string = str_lit("11.0.0"); + } + + if (subtarget == Subtarget_Default) { + bc->metrics.target_triplet = concatenate_strings(permanent_allocator(), bc->metrics.target_triplet, bc->minimum_os_version_string); + } } if (bc->ODIN_DEBUG && !bc->custom_optimization_level) { @@ -1626,8 +1596,10 @@ gb_internal void init_build_context(TargetMetrics *cross_target, Subtarget subta if (bc->metrics.os == TargetOs_js || bc->metrics.os == TargetOs_wasi) { // TODO(bill): Should these even have a default "heap-like" allocator? } - bc->ODIN_DEFAULT_TO_PANIC_ALLOCATOR = true; - bc->ODIN_DEFAULT_TO_NIL_ALLOCATOR = !bc->ODIN_DEFAULT_TO_PANIC_ALLOCATOR; + + if (!bc->ODIN_DEFAULT_TO_NIL_ALLOCATOR && !bc->ODIN_DEFAULT_TO_PANIC_ALLOCATOR) { + bc->ODIN_DEFAULT_TO_PANIC_ALLOCATOR = true; + } } } diff --git a/src/check_expr.cpp b/src/check_expr.cpp index b893b3a00..06d0a8b12 100644 --- a/src/check_expr.cpp +++ b/src/check_expr.cpp @@ -8079,11 +8079,10 @@ gb_internal void add_constant_switch_case(CheckerContext *ctx, SeenMap *seen, Op } uintptr key = hash_exact_value(operand.value); - TypeAndToken *found = map_get(seen, key); - if (found != nullptr) { + GB_ASSERT(key != 0); + isize count = multi_map_count(seen, key); + if (count) { TEMPORARY_ALLOCATOR_GUARD(); - - isize count = multi_map_count(seen, key); TypeAndToken *taps = gb_alloc_array(temporary_allocator(), TypeAndToken, count); multi_map_get_all(seen, key, taps); @@ -9406,7 +9405,8 @@ gb_internal ExprKind check_compound_literal(CheckerContext *c, Operand *o, Ast * continue; } ExactValue v = f->Constant.value; - auto found = map_get(&seen, hash_exact_value(v)); + uintptr hash = hash_exact_value(v); + auto found = map_get(&seen, hash); if (!found) { array_add(&unhandled, f); } diff --git a/src/check_stmt.cpp b/src/check_stmt.cpp index fc3b9aa43..cccbab4f6 100644 --- a/src/check_stmt.cpp +++ b/src/check_stmt.cpp @@ -474,16 +474,59 @@ gb_internal Type *check_assignment_variable(CheckerContext *ctx, Operand *lhs, O } Entity *e = entity_of_node(lhs->expr); + Entity *original_e = e; + + Ast *name = unparen_expr(lhs->expr); + while (name->kind == Ast_SelectorExpr) { + name = name->SelectorExpr.expr; + e = entity_of_node(name); + } + if (e == nullptr) { + e = original_e; + } gbString str = expr_to_string(lhs->expr); if (e != nullptr && e->flags & EntityFlag_Param) { + ERROR_BLOCK(); if (e->flags & EntityFlag_Using) { error(lhs->expr, "Cannot assign to '%s' which is from a 'using' procedure parameter", str); } else { error(lhs->expr, "Cannot assign to '%s' which is a procedure parameter", str); } + error_line("\tSuggestion: Did you mean to pass '%.*s' by pointer?\n", LIT(e->token.string)); + show_error_on_line(e->token.pos, token_pos_end(e->token)); } else { + ERROR_BLOCK(); error(lhs->expr, "Cannot assign to '%s'", str); + + if (e && e->flags & EntityFlag_ForValue) { + isize offset = show_error_on_line(e->token.pos, token_pos_end(e->token), "Suggestion:"); + if (offset < 0) { + if (is_type_map(e->type)) { + error_line("\tSuggestion: Did you mean? 'for key, &%.*s in ...'\n", LIT(e->token.string)); + } else { + error_line("\tSuggestion: Did you mean? 'for &%.*s in ...'\n", LIT(e->token.string)); + } + } else { + error_line("\t"); + for (isize i = 0; i < offset-1; i++) { + error_line(" "); + } + error_line("'%.*s' is immutable, declare it as '&%.*s' to make it mutable\n", LIT(e->token.string), LIT(e->token.string)); + } + + } else if (e && e->flags & EntityFlag_SwitchValue) { + isize offset = show_error_on_line(e->token.pos, token_pos_end(e->token), "Suggestion:"); + if (offset < 0) { + error_line("\tSuggestion: Did you mean? 'switch &%.*s in ...'\n", LIT(e->token.string)); + } else { + error_line("\t"); + for (isize i = 0; i < offset-1; i++) { + error_line(" "); + } + error_line("'%.*s' is immutable, declare it as '&%.*s' to make it mutable\n", LIT(e->token.string), LIT(e->token.string)); + } + } } gb_string_free(str); @@ -1586,6 +1629,17 @@ gb_internal void check_range_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) if (build_context.no_rtti && is_type_enum(t->BitSet.elem)) { error(node, "Iteration over a bit_set of an enum is not allowed runtime type information (RTTI) has been disallowed"); } + if (rs->vals.count == 1 && rs->vals[0] && rs->vals[0]->kind == Ast_Ident) { + String name = rs->vals[0]->Ident.token.string; + Entity *found = scope_lookup(ctx->scope, name); + if (found && are_types_identical(found->type, t->BitSet.elem)) { + ERROR_BLOCK(); + gbString s = expr_to_string(expr); + error(rs->vals[0], "'%.*s' shadows a previous declaration which might be ambiguous with 'for (%.*s in %s)'", LIT(name), LIT(name), s); + error_line("\tSuggestion: Use a different identifier if iteration is wanted, or surround in parentheses if a normal for loop is wanted\n"); + gb_string_free(s); + } + } break; case Type_EnumeratedArray: @@ -1621,17 +1675,36 @@ gb_internal void check_range_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) if (is_reverse) { error(node, "#reverse for is not supported for map types, as maps are unordered"); } + if (rs->vals.count == 1 && rs->vals[0] && rs->vals[0]->kind == Ast_Ident) { + String name = rs->vals[0]->Ident.token.string; + Entity *found = scope_lookup(ctx->scope, name); + if (found && are_types_identical(found->type, t->Map.key)) { + ERROR_BLOCK(); + gbString s = expr_to_string(expr); + error(rs->vals[0], "'%.*s' shadows a previous declaration which might be ambiguous with 'for (%.*s in %s)'", LIT(name), LIT(name), s); + error_line("\tSuggestion: Use a different identifier if iteration is wanted, or surround in parentheses if a normal for loop is wanted\n"); + gb_string_free(s); + } + } break; case Type_Tuple: { isize count = t->Tuple.variables.count; - if (count < 1 || count > 3) { + if (count < 1) { ERROR_BLOCK(); check_not_tuple(ctx, &operand); - error_line("\tMultiple return valued parameters in a range statement are limited to a maximum of 2 usable values with a trailing boolean for the conditional\n"); + error_line("\tMultiple return valued parameters in a range statement are limited to a minimum of 1 usable values with a trailing boolean for the conditional, got %td\n", count); break; } + enum : isize {MAXIMUM_COUNT = 100}; + if (count > MAXIMUM_COUNT) { + ERROR_BLOCK(); + check_not_tuple(ctx, &operand); + error_line("\tMultiple return valued parameters in a range statement are limited to a maximum of %td usable values with a trailing boolean for the conditional, got %td\n", MAXIMUM_COUNT, count); + break; + } + Type *cond_type = t->Tuple.variables[count-1]->type; if (!is_type_boolean(cond_type)) { gbString s = type_to_string(cond_type); @@ -1640,24 +1713,23 @@ gb_internal void check_range_stmt(CheckerContext *ctx, Ast *node, u32 mod_flags) break; } + max_val_count = count; + for (Entity *e : t->Tuple.variables) { array_add(&vals, e->type); } is_possibly_addressable = false; - if (rs->vals.count > 1 && rs->vals[1] != nullptr && count < 3) { - gbString s = type_to_string(t); - error(operand.expr, "Expected a 3-valued expression on the rhs, got (%s)", s); - gb_string_free(s); - break; - } - - if (rs->vals.count > 0 && rs->vals[0] != nullptr && count < 2) { - gbString s = type_to_string(t); - error(operand.expr, "Expected at least a 2-valued expression on the rhs, got (%s)", s); - gb_string_free(s); - break; + bool do_break = false; + for (isize i = rs->vals.count-1; i >= 0; i--) { + if (rs->vals[i] != nullptr && count < i+2) { + gbString s = type_to_string(t); + error(operand.expr, "Expected a %td-valued expression on the rhs, got (%s)", i+2, s); + gb_string_free(s); + do_break = true; + break; + } } if (is_reverse) { @@ -2355,14 +2427,14 @@ gb_internal void check_return_stmt(CheckerContext *ctx, Ast *node) { unsafe_return_error(o, "the address of a compound literal"); } else if (x->kind == Ast_IndexExpr) { Entity *f = entity_of_node(x->IndexExpr.expr); - if (is_type_array_like(f->type) || is_type_matrix(f->type)) { + if (f && (is_type_array_like(f->type) || is_type_matrix(f->type))) { if (is_entity_local_variable(f)) { unsafe_return_error(o, "the address of an indexed variable", f->type); } } } else if (x->kind == Ast_MatrixIndexExpr) { Entity *f = entity_of_node(x->MatrixIndexExpr.expr); - if (is_type_matrix(f->type) && is_entity_local_variable(f)) { + if (f && (is_type_matrix(f->type) && is_entity_local_variable(f))) { unsafe_return_error(o, "the address of an indexed variable", f->type); } } diff --git a/src/check_type.cpp b/src/check_type.cpp index f1d991acb..ab8c0b057 100644 --- a/src/check_type.cpp +++ b/src/check_type.cpp @@ -29,10 +29,11 @@ gb_internal void populate_using_array_index(CheckerContext *ctx, Ast *node, AstF } } -gb_internal void populate_using_entity_scope(CheckerContext *ctx, Ast *node, AstField *field, Type *t) { +gb_internal void populate_using_entity_scope(CheckerContext *ctx, Ast *node, AstField *field, Type *t, isize level) { if (t == nullptr) { return; } + Type *original_type = t; t = base_type(type_deref(t)); gbString str = nullptr; defer (gb_string_free(str)); @@ -46,16 +47,18 @@ gb_internal void populate_using_entity_scope(CheckerContext *ctx, Ast *node, Ast String name = f->token.string; Entity *e = scope_lookup_current(ctx->scope, name); if (e != nullptr && name != "_") { + gbString ot = type_to_string(original_type); // TODO(bill): Better type error if (str != nullptr) { - error(e->token, "'%.*s' is already declared in '%s'", LIT(name), str); + error(e->token, "'%.*s' is already declared in '%s', through 'using' from '%s'", LIT(name), str, ot); } else { - error(e->token, "'%.*s' is already declared", LIT(name)); + error(e->token, "'%.*s' is already declared, through 'using' from '%s'", LIT(name), ot); } + gb_string_free(ot); } else { add_entity(ctx, ctx->scope, nullptr, f); if (f->flags & EntityFlag_Using) { - populate_using_entity_scope(ctx, node, field, f->type); + populate_using_entity_scope(ctx, node, field, f->type, level+1); } } } @@ -200,7 +203,7 @@ gb_internal void check_struct_fields(CheckerContext *ctx, Ast *node, Slicenames.count > 0) { @@ -952,13 +955,18 @@ gb_internal void check_bit_field_type(CheckerContext *ctx, Type *bit_field_type, GB_ASSERT(is_type_bit_field(bit_field_type)); Type *backing_type = check_type(ctx, bf->backing_type); - if (backing_type == nullptr || !is_valid_bit_field_backing_type(backing_type)) { - error(node, "Backing type for a bit_field must be an integer or an array of an integer"); + + bit_field_type->BitField.backing_type = backing_type ? backing_type : t_u8; + bit_field_type->BitField.scope = ctx->scope; + + if (backing_type == nullptr) { + error(bf->backing_type, "Backing type for a bit_field must be an integer or an array of an integer"); + return; + } + if (!is_valid_bit_field_backing_type(backing_type)) { + error(bf->backing_type, "Backing type for a bit_field must be an integer or an array of an integer"); return; } - - bit_field_type->BitField.backing_type = backing_type; - bit_field_type->BitField.scope = ctx->scope; auto fields = array_make(permanent_allocator(), 0, bf->fields.count); auto bit_sizes = array_make (permanent_allocator(), 0, bf->fields.count); @@ -1093,6 +1101,45 @@ gb_internal void check_bit_field_type(CheckerContext *ctx, Type *bit_field_type, gb_string_free(s); } + enum EndianKind { + Endian_Unknown, + Endian_Native, + Endian_Little, + Endian_Big, + }; + auto const &determine_endian_kind = [](Type *type) -> EndianKind { + if (is_type_boolean(type)) { + // NOTE(bill): it doesn't matter, and when it does, + // that api is absolutely stupid + return Endian_Unknown; + } else if (is_type_endian_specific(type)) { + if (is_type_endian_little(type)) { + return Endian_Little; + } else { + return Endian_Big; + } + } + return Endian_Native; + }; + + EndianKind backing_type_endian_kind = determine_endian_kind(core_array_type(backing_type)); + EndianKind endian_kind = Endian_Unknown; + for (Entity *f : fields) { + EndianKind field_kind = determine_endian_kind(f->type); + + if (field_kind && backing_type_endian_kind != field_kind) { + error(f->token, "All 'bit_field' field types must match the same endian kind as the backing type, i.e. all native, all little, or all big"); + } + + if (endian_kind == Endian_Unknown) { + endian_kind = field_kind; + } else if (field_kind && endian_kind != field_kind) { + error(f->token, "All 'bit_field' field types must be of the same endian variety, i.e. all native, all little, or all big"); + } + } + + + if (bit_sizes.count > 0 && is_type_integer(backing_type)) { bool all_booleans = is_type_boolean(fields[0]->type); bool all_ones = bit_sizes[0] == 1; @@ -2495,18 +2542,16 @@ gb_internal Type *get_map_cell_type(Type *type) { return s; } -gb_internal void init_map_internal_types(Type *type) { +gb_internal void init_map_internal_debug_types(Type *type) { GB_ASSERT(type->kind == Type_Map); GB_ASSERT(t_allocator != nullptr); - if (type->Map.lookup_result_type != nullptr) return; + if (type->Map.debug_metadata_type != nullptr) return; Type *key = type->Map.key; Type *value = type->Map.value; GB_ASSERT(key != nullptr); GB_ASSERT(value != nullptr); - - Type *key_cell = get_map_cell_type(key); Type *value_cell = get_map_cell_type(value); @@ -2541,6 +2586,18 @@ gb_internal void init_map_internal_types(Type *type) { gb_unused(type_size_of(debug_type)); type->Map.debug_metadata_type = debug_type; +} + + +gb_internal void init_map_internal_types(Type *type) { + GB_ASSERT(type->kind == Type_Map); + GB_ASSERT(t_allocator != nullptr); + if (type->Map.lookup_result_type != nullptr) return; + + Type *key = type->Map.key; + Type *value = type->Map.value; + GB_ASSERT(key != nullptr); + GB_ASSERT(value != nullptr); type->Map.lookup_result_type = make_optional_ok_type(value); } @@ -2613,8 +2670,6 @@ gb_internal void check_map_type(CheckerContext *ctx, Type *type, Ast *node) { init_core_map_type(ctx->checker); init_map_internal_types(type); - - // error(node, "'map' types are not yet implemented"); } gb_internal void check_matrix_type(CheckerContext *ctx, Type **type, Ast *node) { @@ -3233,6 +3288,11 @@ gb_internal bool check_type_internal(CheckerContext *ctx, Ast *e, Type **type, T Type *elem = t_invalid; Operand o = {}; + if (unparen_expr(pt->type) == nullptr) { + error(e, "Invalid pointer type"); + return false; + } + check_expr_or_type(&c, &o, pt->type); if (o.mode != Addressing_Invalid && o.mode != Addressing_Type) { if (o.mode == Addressing_Variable) { diff --git a/src/checker.cpp b/src/checker.cpp index 77d894624..12c8f5291 100644 --- a/src/checker.cpp +++ b/src/checker.cpp @@ -703,11 +703,11 @@ gb_internal void check_scope_usage(Checker *c, Scope *scope, u64 vet_flags) { array_add(&vetted_entities, ve_unused); } else if (is_shadowed) { array_add(&vetted_entities, ve_shadowed); - } else if (e->kind == Entity_Variable && (e->flags & (EntityFlag_Param|EntityFlag_Using)) == 0) { + } else if (e->kind == Entity_Variable && (e->flags & (EntityFlag_Param|EntityFlag_Using|EntityFlag_Static)) == 0 && !e->Variable.is_global) { i64 sz = type_size_of(e->type); // TODO(bill): When is a good size warn? - // Is 128 KiB good enough? - if (sz >= 1ll<<17) { + // Is >256 KiB good enough? + if (sz > 1ll<<18) { gbString type_str = type_to_string(e->type); warning(e->token, "Declaration of '%.*s' may cause a stack overflow due to its type '%s' having a size of %lld bytes", LIT(e->token.string), type_str, cast(long long)sz); gb_string_free(type_str); @@ -728,7 +728,10 @@ gb_internal void check_scope_usage(Checker *c, Scope *scope, u64 vet_flags) { } else if (vet_flags) { switch (ve.kind) { case VettedEntity_Unused: - if (vet_flags & VetFlag_Unused) { + if (e->kind == Entity_Variable && (vet_flags & VetFlag_UnusedVariables) != 0) { + error(e->token, "'%.*s' declared but not used", LIT(name)); + } + if ((e->kind == Entity_ImportName || e->kind == Entity_LibraryName) && (vet_flags & VetFlag_UnusedImports) != 0) { error(e->token, "'%.*s' declared but not used", LIT(name)); } break; @@ -1109,6 +1112,8 @@ gb_internal void init_universal(void) { add_global_bool_constant("ODIN_DEBUG", bc->ODIN_DEBUG); add_global_bool_constant("ODIN_DISABLE_ASSERT", bc->ODIN_DISABLE_ASSERT); add_global_bool_constant("ODIN_DEFAULT_TO_NIL_ALLOCATOR", bc->ODIN_DEFAULT_TO_NIL_ALLOCATOR); + add_global_bool_constant("ODIN_NO_BOUNDS_CHECK", build_context.no_bounds_check); + add_global_bool_constant("ODIN_NO_TYPE_ASSERT", build_context.no_type_assert); add_global_bool_constant("ODIN_DEFAULT_TO_PANIC_ALLOCATOR", bc->ODIN_DEFAULT_TO_PANIC_ALLOCATOR); add_global_bool_constant("ODIN_NO_DYNAMIC_LITERALS", bc->no_dynamic_literals); add_global_bool_constant("ODIN_NO_CRT", bc->no_crt); @@ -1893,8 +1898,7 @@ gb_internal void add_type_info_type_internal(CheckerContext *c, Type *t) { add_type_info_dependency(c->info, c->decl, t); MUTEX_GUARD_BLOCK(&c->info->type_info_mutex) { - MapFindResult fr; - auto found = map_try_get(&c->info->type_info_map, t, &fr); + auto found = map_get(&c->info->type_info_map, t); if (found != nullptr) { // Types have already been added return; @@ -1918,7 +1922,7 @@ gb_internal void add_type_info_type_internal(CheckerContext *c, Type *t) { ti_index = c->info->type_info_types.count; array_add(&c->info->type_info_types, t); } - map_set_internal_from_try_get(&c->checker->info.type_info_map, t, ti_index, fr); + map_set(&c->checker->info.type_info_map, t, ti_index); if (prev) { // NOTE(bill): If a previous one exists already, no need to continue @@ -2189,7 +2193,7 @@ gb_internal void add_min_dep_type_info(Checker *c, Type *t) { // IMPORTANT NOTE(bill): this must be copied as `map_set` takes a const ref // and effectively assigns the `+1` of the value isize const count = set->count; - if (map_set_if_not_previously_exists(set, ti_index, count)) { + if (map_set_if_not_previously_exists(set, ti_index+1, count)) { // Type already exists; return; } @@ -2532,6 +2536,11 @@ gb_internal void generate_minimum_dependency_set_internal(Checker *c, Entity *st is_init = false; } + if ((e->flags & EntityFlag_Disabled) != 0) { + warning(e->token, "This @(init) procedure is disabled; you must call it manually"); + is_init = false; + } + if (is_init) { add_dependency_to_set(c, e); array_add(&c->info.init_procedures, e); @@ -2922,6 +2931,8 @@ gb_internal void init_core_type_info(Checker *c) { return; } Entity *type_info_entity = find_core_entity(c, str_lit("Type_Info")); + GB_ASSERT(type_info_entity != nullptr); + GB_ASSERT(type_info_entity->type != nullptr); t_type_info = type_info_entity->type; t_type_info_ptr = alloc_type_pointer(t_type_info); @@ -4305,17 +4316,22 @@ gb_internal bool correct_single_type_alias(CheckerContext *c, Entity *e) { gb_internal bool correct_type_alias_in_scope_backwards(CheckerContext *c, Scope *s) { bool correction = false; - u32 n = s->elements.count; - for (u32 i = n-1; i < n; i--) { - correction |= correct_single_type_alias(c, s->elements.entries[i].value); + for (u32 n = s->elements.count, i = n-1; i < n; i--) { + auto const &entry = s->elements.entries[i]; + Entity *e = entry.value; + if (entry.hash && e != nullptr) { + correction |= correct_single_type_alias(c, e); + } } return correction; } gb_internal bool correct_type_alias_in_scope_forwards(CheckerContext *c, Scope *s) { bool correction = false; - u32 n = s->elements.count; - for (isize i = 0; i < n; i++) { - correction |= correct_single_type_alias(c, s->elements.entries[i].value); + for (auto const &entry : s->elements) { + Entity *e = entry.value; + if (e != nullptr) { + correction |= correct_single_type_alias(c, entry.value); + } } return correction; } diff --git a/src/checker.hpp b/src/checker.hpp index 1701da58d..2ade9312e 100644 --- a/src/checker.hpp +++ b/src/checker.hpp @@ -563,4 +563,7 @@ gb_internal void init_mem_allocator(Checker *c); gb_internal void add_untyped_expressions(CheckerInfo *cinfo, UntypedExprInfoMap *untyped); -gb_internal GenTypesData *ensure_polymorphic_record_entity_has_gen_types(CheckerContext *ctx, Type *original_type); \ No newline at end of file +gb_internal GenTypesData *ensure_polymorphic_record_entity_has_gen_types(CheckerContext *ctx, Type *original_type); + + +gb_internal void init_map_internal_types(Type *type); \ No newline at end of file diff --git a/src/common_memory.cpp b/src/common_memory.cpp index c6ee88f03..60e570eee 100644 --- a/src/common_memory.cpp +++ b/src/common_memory.cpp @@ -163,6 +163,10 @@ gb_internal void platform_virtual_memory_protect(void *memory, isize size); GB_ASSERT(is_protected); } #else + #if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) + #define MAP_ANONYMOUS MAP_ANON + #endif + gb_internal void platform_virtual_memory_init(void) { global_platform_memory_block_sentinel.prev = &global_platform_memory_block_sentinel; global_platform_memory_block_sentinel.next = &global_platform_memory_block_sentinel; diff --git a/src/docs_writer.cpp b/src/docs_writer.cpp index 9ced78d33..ba71eae4d 100644 --- a/src/docs_writer.cpp +++ b/src/docs_writer.cpp @@ -987,9 +987,8 @@ gb_internal void odin_doc_update_entities(OdinDocWriter *w) { auto entities = array_make(heap_allocator(), 0, w->entity_cache.count); defer (array_free(&entities)); - for (u32 i = 0; i < w->entity_cache.count; i++) { - Entity *e = w->entity_cache.entries[i].key; - array_add(&entities, e); + for (auto const &entry : w->entity_cache) { + array_add(&entities, entry.key); } for (Entity *e : entities) { GB_ASSERT(e != nullptr); @@ -998,9 +997,9 @@ gb_internal void odin_doc_update_entities(OdinDocWriter *w) { } } - for (u32 i = 0; i < w->entity_cache.count; i++) { - Entity *e = w->entity_cache.entries[i].key; - OdinDocEntityIndex entity_index = w->entity_cache.entries[i].value; + for (auto const &entry : w->entity_cache) { + Entity *e = entry.key; + OdinDocEntityIndex entity_index = entry.value; OdinDocTypeIndex type_index = odin_doc_type(w, e->type); OdinDocEntityIndex foreign_library = 0; diff --git a/src/entity.cpp b/src/entity.cpp index 6cea0930f..a12e1d0a6 100644 --- a/src/entity.cpp +++ b/src/entity.cpp @@ -496,7 +496,7 @@ gb_internal bool is_entity_local_variable(Entity *e) { if (e->scope == nullptr) { return true; } - if (e->flags & (EntityFlag_ForValue|EntityFlag_SwitchValue)) { + if (e->flags & (EntityFlag_ForValue|EntityFlag_SwitchValue|EntityFlag_Static)) { return false; } diff --git a/src/error.cpp b/src/error.cpp index eb167d4c3..1b091f88e 100644 --- a/src/error.cpp +++ b/src/error.cpp @@ -7,7 +7,8 @@ struct ErrorValue { ErrorValueKind kind; TokenPos pos; TokenPos end; - Array msgs; + Array msg; + bool seen_newline; }; struct ErrorCollector { @@ -30,19 +31,21 @@ gb_global ErrorCollector global_error_collector; gb_internal void push_error_value(TokenPos const &pos, ErrorValueKind kind = ErrorValue_Error) { GB_ASSERT_MSG(global_error_collector.curr_error_value_set.load() == false, "Possible race condition in error handling system, please report this with an issue"); ErrorValue ev = {kind, pos}; - ev.msgs.allocator = heap_allocator(); + ev.msg.allocator = heap_allocator(); global_error_collector.curr_error_value = ev; global_error_collector.curr_error_value_set.store(true); } gb_internal void pop_error_value(void) { + mutex_lock(&global_error_collector.mutex); if (global_error_collector.curr_error_value_set.load()) { array_add(&global_error_collector.error_values, global_error_collector.curr_error_value); global_error_collector.curr_error_value = {}; global_error_collector.curr_error_value_set.store(false); } + mutex_unlock(&global_error_collector.mutex); } @@ -84,6 +87,7 @@ gb_internal bool set_file_path_string(i32 index, String const &path) { bool ok = false; GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); if (index >= global_file_path_strings.count) { array_resize(&global_file_path_strings, index+1); @@ -94,6 +98,7 @@ gb_internal bool set_file_path_string(i32 index, String const &path) { ok = true; } + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return ok; } @@ -102,6 +107,7 @@ gb_internal bool thread_safe_set_ast_file_from_id(i32 index, AstFile *file) { bool ok = false; GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); if (index >= global_files.count) { array_resize(&global_files, index+1); @@ -111,7 +117,7 @@ gb_internal bool thread_safe_set_ast_file_from_id(i32 index, AstFile *file) { global_files[index] = file; ok = true; } - + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return ok; } @@ -119,12 +125,14 @@ gb_internal bool thread_safe_set_ast_file_from_id(i32 index, AstFile *file) { gb_internal String get_file_path_string(i32 index) { GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); String path = {}; if (index < global_file_path_strings.count) { path = global_file_path_strings[index]; } + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return path; } @@ -132,12 +140,14 @@ gb_internal String get_file_path_string(i32 index) { gb_internal AstFile *thread_safe_get_ast_file_from_id(i32 index) { GB_ASSERT(index >= 0); mutex_lock(&global_error_collector.path_mutex); + mutex_lock(&global_files_mutex); AstFile *file = nullptr; if (index < global_files.count) { file = global_files[index]; } + mutex_unlock(&global_files_mutex); mutex_unlock(&global_error_collector.path_mutex); return file; } @@ -173,9 +183,18 @@ gb_internal ERROR_OUT_PROC(default_error_out_va) { isize n = len-1; if (n > 0) { - String msg = copy_string(permanent_allocator(), {(u8 *)buf, n}); ErrorValue *ev = get_error_value(); - array_add(&ev->msgs, msg); + if (terse_errors()) { + for (isize i = 0; i < n && !ev->seen_newline; i++) { + u8 c = cast(u8)buf[i]; + if (c == '\n') { + ev->seen_newline = true; + } + array_add(&ev->msg, c); + } + } else { + array_add_elems(&ev->msg, (u8 *)buf, n); + } } } @@ -247,10 +266,10 @@ gb_internal void terminal_reset_colours(void) { } -gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { +gb_internal isize show_error_on_line(TokenPos const &pos, TokenPos end, char const *prefix=nullptr) { get_error_value()->end = end; if (!show_error_line()) { - return false; + return -1; } i32 offset = 0; @@ -270,6 +289,10 @@ gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { MAX_LINE_LENGTH_PADDED = MAX_LINE_LENGTH-MAX_TAB_WIDTH-ELLIPSIS_PADDING, }; + if (prefix) { + error_out("\t%s\n\n", prefix); + } + error_out("\t"); terminal_set_colours(TerminalStyle_Bold, TerminalColour_White); @@ -281,10 +304,11 @@ gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { if (line_len > MAX_LINE_LENGTH_PADDED) { i32 left = MAX_TAB_WIDTH; - if (offset > 0) { - line_text += offset-left; - line_len -= offset-left; - offset = left+MAX_TAB_WIDTH/2; + i32 diff = gb_max(offset-left, 0); + if (diff > 0) { + line_text += diff; + line_len -= diff; + offset = left + ELLIPSIS_PADDING/2; } if (line_len > MAX_LINE_LENGTH_PADDED) { line_len = MAX_LINE_LENGTH_PADDED; @@ -293,7 +317,7 @@ gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { squiggle_extra = 1; } } - if (offset > 0) { + if (diff > 0) { error_out("... %.*s ...", cast(i32)line_len, line_text); } else { error_out("%.*s ...", cast(i32)line_len, line_text); @@ -328,9 +352,9 @@ gb_internal bool show_error_on_line(TokenPos const &pos, TokenPos end) { terminal_reset_colours(); error_out("\n"); - return true; + return offset; } - return false; + return -1; } gb_internal void error_out_empty(void) { @@ -352,11 +376,11 @@ gb_internal void error_out_coloured(char const *str, TerminalStyle style, Termin gb_internal void error_va(TokenPos const &pos, TokenPos end, char const *fmt, va_list va) { global_error_collector.count.fetch_add(1); + mutex_lock(&global_error_collector.mutex); if (global_error_collector.count > MAX_ERROR_COLLECTOR_COUNT()) { print_all_errors(); gb_exit(1); } - mutex_lock(&global_error_collector.mutex); push_error_value(pos, ErrorValue_Error); // NOTE(bill): Duplicate error, skip it @@ -367,7 +391,11 @@ gb_internal void error_va(TokenPos const &pos, TokenPos end, char const *fmt, va error_out("\n"); } else if (global_error_collector.prev != pos) { global_error_collector.prev = pos; - error_out_pos(pos); + if (json_errors()) { + error_out_empty(); + } else { + error_out_pos(pos); + } if (has_ansi_terminal_colours()) { error_out_coloured("Error: ", TerminalStyle_Normal, TerminalColour_Red); } @@ -375,6 +403,8 @@ gb_internal void error_va(TokenPos const &pos, TokenPos end, char const *fmt, va error_out("\n"); show_error_on_line(pos, end); } else { + global_error_collector.curr_error_value = {}; + global_error_collector.curr_error_value_set.store(false); global_error_collector.count.fetch_sub(1); } try_pop_error_value(); @@ -400,7 +430,11 @@ gb_internal void warning_va(TokenPos const &pos, TokenPos end, char const *fmt, error_out("\n"); } else if (global_error_collector.prev != pos) { global_error_collector.prev = pos; - error_out_pos(pos); + if (json_errors()) { + error_out_empty(); + } else { + error_out_pos(pos); + } error_out_coloured("Warning: ", TerminalStyle_Normal, TerminalColour_Yellow); error_out_va(fmt, va); error_out("\n"); @@ -433,7 +467,11 @@ gb_internal void error_no_newline_va(TokenPos const &pos, char const *fmt, va_li error_out_va(fmt, va); } else if (global_error_collector.prev != pos) { global_error_collector.prev = pos; - error_out_pos(pos); + if (json_errors()) { + error_out_empty(); + } else { + error_out_pos(pos); + } if (has_ansi_terminal_colours()) { error_out_coloured("Error: ", TerminalStyle_Normal, TerminalColour_Red); } @@ -458,7 +496,11 @@ gb_internal void syntax_error_va(TokenPos const &pos, TokenPos end, char const * // NOTE(bill): Duplicate error, skip it if (global_error_collector.prev != pos) { global_error_collector.prev = pos; - error_out_pos(pos); + if (json_errors()) { + error_out_empty(); + } else { + error_out_pos(pos); + } error_out_coloured("Syntax Error: ", TerminalStyle_Normal, TerminalColour_Red); error_out_va(fmt, va); error_out("\n"); @@ -492,7 +534,11 @@ gb_internal void syntax_error_with_verbose_va(TokenPos const &pos, TokenPos end, error_out("\n"); } else if (global_error_collector.prev != pos) { global_error_collector.prev = pos; - error_out_pos(pos); + if (json_errors()) { + error_out_empty(); + } else { + error_out_pos(pos); + } if (has_ansi_terminal_colours()) { error_out_coloured("Syntax_Error: ", TerminalStyle_Normal, TerminalColour_Red); } @@ -521,7 +567,11 @@ gb_internal void syntax_warning_va(TokenPos const &pos, TokenPos end, char const // NOTE(bill): Duplicate error, skip it if (global_error_collector.prev != pos) { global_error_collector.prev = pos; - error_out_pos(pos); + if (json_errors()) { + error_out_empty(); + } else { + error_out_pos(pos); + } error_out_coloured("Syntax Warning: ", TerminalStyle_Normal, TerminalColour_Yellow); error_out_va(fmt, va); error_out("\n"); @@ -633,109 +683,119 @@ gb_internal int error_value_cmp(void const *a, void const *b) { } gb_internal void print_all_errors(void) { - auto const &escape_char = [](gbFile *f, u8 c) { + auto const &escape_char = [](gbString res, u8 c) -> gbString { switch (c) { - case '\n': gb_file_write(f, "\\n", 2); break; - case '"': gb_file_write(f, "\\\"", 2); break; - case '\\': gb_file_write(f, "\\\\", 2); break; - case '\b': gb_file_write(f, "\\b", 2); break; - case '\f': gb_file_write(f, "\\f", 2); break; - case '\r': gb_file_write(f, "\\r", 2); break; - case '\t': gb_file_write(f, "\\t", 2); break; + case '\n': res = gb_string_append_length(res, "\\n", 2); break; + case '"': res = gb_string_append_length(res, "\\\"", 2); break; + case '\\': res = gb_string_append_length(res, "\\\\", 2); break; + case '\b': res = gb_string_append_length(res, "\\b", 2); break; + case '\f': res = gb_string_append_length(res, "\\f", 2); break; + case '\r': res = gb_string_append_length(res, "\\r", 2); break; + case '\t': res = gb_string_append_length(res, "\\t", 2); break; default: if ('\x00' <= c && c <= '\x1f') { - gb_fprintf(f, "\\u%04x", c); + res = gb_string_append_fmt(res, "\\u%04x", c); } else { - gb_file_write(f, &c, 1); + res = gb_string_append_length(res, &c, 1); } break; } + return res; }; GB_ASSERT(any_errors() || any_warnings()); - gbFile *f = gb_file_get_standard(gbFileStandard_Error); + array_sort(global_error_collector.error_values, error_value_cmp); + gbString res = gb_string_make(heap_allocator(), ""); + defer (gb_string_free(res)); if (json_errors()) { - gb_fprintf(f, "{\n"); - gb_fprintf(f, "\t\"error_count\": %td,\n", global_error_collector.error_values.count); - gb_fprintf(f, "\t\"errors\": [\n"); + res = gb_string_append_fmt(res, "{\n"); + res = gb_string_append_fmt(res, "\t\"error_count\": %td,\n", global_error_collector.error_values.count); + res = gb_string_append_fmt(res, "\t\"errors\": [\n"); for_array(i, global_error_collector.error_values) { ErrorValue ev = global_error_collector.error_values[i]; - gb_fprintf(f, "\t\t{\n"); + res = gb_string_append_fmt(res, "\t\t{\n"); - gb_fprintf(f, "\t\t\t\"type\": \""); + res = gb_string_append_fmt(res, "\t\t\t\"type\": \""); if (ev.kind == ErrorValue_Warning) { - gb_fprintf(f, "warning"); + res = gb_string_append_fmt(res, "warning"); } else { - gb_fprintf(f, "error"); + res = gb_string_append_fmt(res, "error"); } - gb_fprintf(f, "\",\n"); + res = gb_string_append_fmt(res, "\",\n"); - gb_fprintf(f, "\t\t\t\"pos\": {\n"); + res = gb_string_append_fmt(res, "\t\t\t\"pos\": {\n"); if (ev.pos.file_id) { - gb_fprintf(f, "\t\t\t\t\"file\": \""); + res = gb_string_append_fmt(res, "\t\t\t\t\"file\": \""); String file = get_file_path_string(ev.pos.file_id); for (isize k = 0; k < file.len; k++) { - escape_char(f, file.text[k]); + res = escape_char(res, file.text[k]); } - gb_fprintf(f, "\",\n"); - gb_fprintf(f, "\t\t\t\t\"offset\": %d,\n", ev.pos.offset); - gb_fprintf(f, "\t\t\t\t\"line\": %d,\n", ev.pos.line); - gb_fprintf(f, "\t\t\t\t\"column\": %d,\n", ev.pos.column); + res = gb_string_append_fmt(res, "\",\n"); + res = gb_string_append_fmt(res, "\t\t\t\t\"offset\": %d,\n", ev.pos.offset); + res = gb_string_append_fmt(res, "\t\t\t\t\"line\": %d,\n", ev.pos.line); + res = gb_string_append_fmt(res, "\t\t\t\t\"column\": %d,\n", ev.pos.column); i32 end_column = gb_max(ev.end.column, ev.pos.column); - gb_fprintf(f, "\t\t\t\t\"end_column\": %d\n", end_column); - gb_fprintf(f, "\t\t\t},\n"); + res = gb_string_append_fmt(res, "\t\t\t\t\"end_column\": %d\n", end_column); + res = gb_string_append_fmt(res, "\t\t\t},\n"); } - gb_fprintf(f, "\t\t\t\"msgs\": [\n"); + res = gb_string_append_fmt(res, "\t\t\t\"msgs\": [\n"); - if (ev.msgs.count > 1) { - gb_fprintf(f, "\t\t\t\t\""); + auto lines = split_lines_from_array(ev.msg, heap_allocator()); + defer (array_free(&lines)); - for (isize j = 1; j < ev.msgs.count; j++) { - String msg = ev.msgs[j]; - for (isize k = 0; k < msg.len; k++) { - u8 c = msg.text[k]; - if (c == '\n') { - if (k+1 == msg.len && j+1 == ev.msgs.count) { - // don't do the last one - } else { - gb_fprintf(f, "\",\n"); - gb_fprintf(f, "\t\t\t\t\""); - } - } else { - escape_char(f, c); - } + if (lines.count > 0) { + res = gb_string_append_fmt(res, "\t\t\t\t\""); + + for (isize j = 0; j < lines.count; j++) { + String line = lines[j]; + for (isize k = 0; k < line.len; k++) { + u8 c = line.text[k]; + res = escape_char(res, c); + } + if (j+1 < lines.count) { + res = gb_string_append_fmt(res, "\",\n"); + res = gb_string_append_fmt(res, "\t\t\t\t\""); } } - gb_fprintf(f, "\"\n"); + res = gb_string_append_fmt(res, "\"\n"); } - gb_fprintf(f, "\t\t\t]\n"); - gb_fprintf(f, "\t\t}"); + res = gb_string_append_fmt(res, "\t\t\t]\n"); + res = gb_string_append_fmt(res, "\t\t}"); if (i+1 != global_error_collector.error_values.count) { - gb_fprintf(f, ","); + res = gb_string_append_fmt(res, ","); } - gb_fprintf(f, "\n"); + res = gb_string_append_fmt(res, "\n"); } - gb_fprintf(f, "\t]\n"); - gb_fprintf(f, "}\n"); + res = gb_string_append_fmt(res, "\t]\n"); + res = gb_string_append_fmt(res, "}\n"); } else { for_array(i, global_error_collector.error_values) { ErrorValue ev = global_error_collector.error_values[i]; - for (isize j = 0; j < ev.msgs.count; j++) { - String msg = ev.msgs[j]; - gb_file_write(f, msg.text, msg.len); - if (terse_errors() && string_contains_char(msg, '\n')) { + String_Iterator it = {{ev.msg.data, ev.msg.count}, 0}; + + for (isize line_idx = 0; /**/; line_idx++) { + String line = string_split_iterator(&it, '\n'); + if (line.len == 0) { + break; + } + line = string_trim_trailing_whitespace(line); + res = gb_string_append_length(res, line.text, line.len); + res = gb_string_append_length(res, " \n", 2); + if (line_idx == 0 && terse_errors()) { break; } } } } + gbFile *f = gb_file_get_standard(gbFileStandard_Error); + gb_file_write(f, res, gb_string_length(res)); } \ No newline at end of file diff --git a/src/exact_value.cpp b/src/exact_value.cpp index b744d2db0..83af82f55 100644 --- a/src/exact_value.cpp +++ b/src/exact_value.cpp @@ -54,37 +54,50 @@ gb_global ExactValue const empty_exact_value = {}; gb_internal uintptr hash_exact_value(ExactValue v) { mutex_lock(&hash_exact_value_mutex); defer (mutex_unlock(&hash_exact_value_mutex)); + + uintptr res = 0; switch (v.kind) { case ExactValue_Invalid: return 0; case ExactValue_Bool: - return gb_fnv32a(&v.value_bool, gb_size_of(v.value_bool)); + res = gb_fnv32a(&v.value_bool, gb_size_of(v.value_bool)); + break; case ExactValue_String: - return gb_fnv32a(v.value_string.text, v.value_string.len); + res = gb_fnv32a(v.value_string.text, v.value_string.len); + break; case ExactValue_Integer: { u32 key = gb_fnv32a(v.value_integer.dp, gb_size_of(*v.value_integer.dp) * v.value_integer.used); u8 last = (u8)v.value_integer.sign; - return (key ^ last) * 0x01000193; + res = (key ^ last) * 0x01000193; + break; } case ExactValue_Float: - return gb_fnv32a(&v.value_float, gb_size_of(v.value_float)); + res = gb_fnv32a(&v.value_float, gb_size_of(v.value_float)); + break; case ExactValue_Pointer: - return ptr_map_hash_key(v.value_pointer); + res = ptr_map_hash_key(v.value_pointer); + break; case ExactValue_Complex: - return gb_fnv32a(v.value_complex, gb_size_of(Complex128)); + res = gb_fnv32a(v.value_complex, gb_size_of(Complex128)); + break; case ExactValue_Quaternion: - return gb_fnv32a(v.value_quaternion, gb_size_of(Quaternion256)); + res = gb_fnv32a(v.value_quaternion, gb_size_of(Quaternion256)); + break; case ExactValue_Compound: - return ptr_map_hash_key(v.value_compound); + res = ptr_map_hash_key(v.value_compound); + break; case ExactValue_Procedure: - return ptr_map_hash_key(v.value_procedure); + res = ptr_map_hash_key(v.value_procedure); + break; case ExactValue_Typeid: - return ptr_map_hash_key(v.value_typeid); + res = ptr_map_hash_key(v.value_typeid); + break; + default: + res = gb_fnv32a(&v, gb_size_of(ExactValue)); } - return gb_fnv32a(&v, gb_size_of(ExactValue)); - + return res & 0x7fffffff; } diff --git a/src/linker.cpp b/src/linker.cpp index 498a96c5f..e694fd999 100644 --- a/src/linker.cpp +++ b/src/linker.cpp @@ -167,7 +167,7 @@ gb_internal i32 linker_stage(LinkerData *gen) { if (has_asm_extension(lib)) { if (!string_set_update(&asm_files, lib)) { - String asm_file = asm_files.entries[i].value; + String asm_file = lib; String obj_file = concatenate_strings(permanent_allocator(), asm_file, str_lit(".obj")); String obj_format = str_lit("win64"); #if defined(GB_ARCH_32_BIT) diff --git a/src/llvm_abi.cpp b/src/llvm_abi.cpp index 88bb58c55..85a16d321 100644 --- a/src/llvm_abi.cpp +++ b/src/llvm_abi.cpp @@ -1131,8 +1131,9 @@ namespace lbAbiArm64 { if (size <= 16) { LLVMTypeRef cast_type = nullptr; - GB_ASSERT(size > 0); - if (size <= 8) { + if (size == 0) { + cast_type = LLVMStructTypeInContext(c, nullptr, 0, false); + } else if (size <= 8) { cast_type = LLVMIntTypeInContext(c, cast(unsigned)(size*8)); } else { unsigned count = cast(unsigned)((size+7)/8); diff --git a/src/llvm_backend.cpp b/src/llvm_backend.cpp index 645a091b0..4b94cf020 100644 --- a/src/llvm_backend.cpp +++ b/src/llvm_backend.cpp @@ -2659,7 +2659,7 @@ gb_internal bool lb_generate_code(lbGenerator *gen) { LLVMSetInitializer(g, LLVMConstNull(internal_llvm_type)); LLVMSetLinkage(g, USE_SEPARATE_MODULES ? LLVMExternalLinkage : LLVMInternalLinkage); LLVMSetUnnamedAddress(g, LLVMGlobalUnnamedAddr); - LLVMSetGlobalConstant(g, /*true*/false); + LLVMSetGlobalConstant(g, true); lbValue value = {}; value.value = g; diff --git a/src/llvm_backend.hpp b/src/llvm_backend.hpp index c4bf2691d..7dc5f6b63 100644 --- a/src/llvm_backend.hpp +++ b/src/llvm_backend.hpp @@ -122,7 +122,6 @@ struct lbAddr { } swizzle_large; struct { Type *type; - i64 index; i64 bit_offset; i64 bit_size; } bitfield; diff --git a/src/llvm_backend_debug.cpp b/src/llvm_backend_debug.cpp index 511ff0475..2654a1d28 100644 --- a/src/llvm_backend_debug.cpp +++ b/src/llvm_backend_debug.cpp @@ -739,6 +739,7 @@ gb_internal LLVMMetadataRef lb_debug_type_internal(lbModule *m, Type *type) { } case Type_Map: { + init_map_internal_debug_types(type); Type *bt = base_type(type->Map.debug_metadata_type); GB_ASSERT(bt->kind == Type_Struct); @@ -945,6 +946,7 @@ gb_internal LLVMMetadataRef lb_debug_type(lbModule *m, Type *type) { } case Type_Map: { + init_map_internal_debug_types(bt); bt = base_type(bt->Map.debug_metadata_type); GB_ASSERT(bt->kind == Type_Struct); return lb_debug_struct(m, type, bt, name, scope, file, line); @@ -1025,7 +1027,7 @@ gb_internal void lb_add_debug_local_variable(lbProcedure *p, LLVMValueRef ptr, T LLVMDIBuilderInsertDeclareAtEnd(m->debug_builder, storage, var_info, llvm_expr, llvm_debug_loc, block); } -gb_internal void lb_add_debug_param_variable(lbProcedure *p, LLVMValueRef ptr, Type *type, Token const &token, unsigned arg_number, lbBlock *block, lbArgKind arg_kind) { +gb_internal void lb_add_debug_param_variable(lbProcedure *p, LLVMValueRef ptr, Type *type, Token const &token, unsigned arg_number, lbBlock *block) { if (p->debug_info == nullptr) { return; } @@ -1086,15 +1088,7 @@ gb_internal void lb_add_debug_param_variable(lbProcedure *p, LLVMValueRef ptr, T // NOTE(bill, 2022-02-01): For parameter values, you must insert them at the end of the decl block // The reason is that if the parameter is at index 0 and a pointer, there is not such things as an // instruction "before" it. - switch (arg_kind) { - case lbArg_Direct: - LLVMDIBuilderInsertDbgValueAtEnd(m->debug_builder, storage, var_info, llvm_expr, llvm_debug_loc, block->block); - break; - case lbArg_Indirect: - LLVMDIBuilderInsertDeclareAtEnd(m->debug_builder, storage, var_info, llvm_expr, llvm_debug_loc, block->block); - break; - } - + LLVMDIBuilderInsertDeclareAtEnd(m->debug_builder, storage, var_info, llvm_expr, llvm_debug_loc, block->block); } diff --git a/src/llvm_backend_expr.cpp b/src/llvm_backend_expr.cpp index fcec59968..ee1a384ae 100644 --- a/src/llvm_backend_expr.cpp +++ b/src/llvm_backend_expr.cpp @@ -2138,14 +2138,45 @@ gb_internal lbValue lb_emit_conv(lbProcedure *p, lbValue value, Type *t) { if (is_type_array_like(dst)) { Type *elem = base_array_type(dst); - lbValue e = lb_emit_conv(p, value, elem); - // NOTE(bill): Doesn't need to be zero because it will be initialized in the loops - lbAddr v = lb_add_local_generated(p, t, false); isize index_count = cast(isize)get_array_type_count(dst); - for (isize i = 0; i < index_count; i++) { - lbValue elem = lb_emit_array_epi(p, v.addr, i); + isize inlineable = type_size_of(dst) <= build_context.max_simd_align; + lbValue e = lb_emit_conv(p, value, elem); + if (inlineable && lb_is_const(e)) { + lbAddr v = {}; + if (e.value) { + TEMPORARY_ALLOCATOR_GUARD(); + LLVMValueRef *values = gb_alloc_array(temporary_allocator(), LLVMValueRef, index_count); + for (isize i = 0; i < index_count; i++) { + values[i] = e.value; + } + lbValue array_const_value = {}; + array_const_value.type = t; + array_const_value.value = LLVMConstArray(lb_type(m, elem), values, cast(unsigned)index_count); + v = lb_add_global_generated(m, t, array_const_value); + } else { + v = lb_add_global_generated(m, t); + } + + lb_make_global_private_const(v); + return lb_addr_load(p, v); + } + + // NOTE(bill): Doesn't need to be zero because it will be initialized in the loops + lbAddr v = lb_add_local_generated(p, t, false); + + if (!inlineable) { + auto loop_data = lb_loop_start(p, index_count, t_int); + + lbValue elem = lb_emit_array_ep(p, v.addr, loop_data.idx); lb_emit_store(p, elem, e); + + lb_loop_end(p, loop_data); + } else { + for (isize i = 0; i < index_count; i++) { + lbValue elem = lb_emit_array_epi(p, v.addr, i); + lb_emit_store(p, elem, e); + } } return lb_addr_load(p, v); } @@ -3085,7 +3116,7 @@ gb_internal lbValue lb_build_unary_and(lbProcedure *p, Ast *expr) { Type *dst_type = type; - if ((p->state_flags & StateFlag_no_type_assert) == 0) { + if (!build_context.no_type_assert && (p->state_flags & StateFlag_no_type_assert) == 0) { lbValue src_tag = {}; lbValue dst_tag = {}; if (is_type_union_maybe_pointer(src_type)) { @@ -3125,7 +3156,7 @@ gb_internal lbValue lb_build_unary_and(lbProcedure *p, Ast *expr) { v = lb_emit_load(p, v); } lbValue data_ptr = lb_emit_struct_ev(p, v, 0); - if ((p->state_flags & StateFlag_no_type_assert) == 0) { + if (!build_context.no_type_assert && (p->state_flags & StateFlag_no_type_assert) == 0) { GB_ASSERT(!build_context.no_rtti); lbValue any_id = lb_emit_struct_ev(p, v, 1); @@ -4265,7 +4296,19 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { switch (bt->kind) { default: GB_PANIC("Unknown CompoundLit type: %s", type_to_string(type)); break; - case Type_BitField: + case Type_BitField: { + TEMPORARY_ALLOCATOR_GUARD(); + + // Type *backing_type = core_type(bt->BitField.backing_type); + + struct FieldData { + Type *field_type; + u64 bit_offset; + u64 bit_size; + }; + auto values = array_make(temporary_allocator(), 0, cl->elems.count); + auto fields = array_make(temporary_allocator(), 0, cl->elems.count); + for (Ast *elem : cl->elems) { ast_node(fv, FieldValue, elem); String name = fv->field->Ident.token.string; @@ -4276,26 +4319,172 @@ gb_internal lbAddr lb_build_addr_compound_lit(lbProcedure *p, Ast *expr) { GB_ASSERT(sel.entity != nullptr); i64 index = sel.index[0]; - i64 bit_offset = 0; - i64 bit_size = -1; - for_array(i, bt->BitField.fields) { - Entity *f = bt->BitField.fields[i]; - if (f == sel.entity) { - bit_offset = bt->BitField.bit_offsets[i]; - bit_size = bt->BitField.bit_sizes[i]; - break; - } - } + Entity *f = bt->BitField.fields[index]; + GB_ASSERT(f == sel.entity); + i64 bit_offset = bt->BitField.bit_offsets[index]; + i64 bit_size = bt->BitField.bit_sizes[index]; GB_ASSERT(bit_size > 0); Type *field_type = sel.entity->type; lbValue field_expr = lb_build_expr(p, fv->value); field_expr = lb_emit_conv(p, field_expr, field_type); - - lbAddr field_addr = lb_addr_bit_field(v.addr, field_type, index, bit_offset, bit_size); - lb_addr_store(p, field_addr, field_expr); + array_add(&values, field_expr); + array_add(&fields, FieldData{field_type, cast(u64)bit_offset, cast(u64)bit_size}); } + + // NOTE(bill): inline insertion sort should be good enough, right? + for (isize i = 1; i < values.count; i++) { + for (isize j = i; + j > 0 && fields[i].bit_offset < fields[j].bit_offset; + j--) { + auto vtmp = values[j]; + values[j] = values[j-1]; + values[j-1] = vtmp; + + auto ftmp = fields[j]; + fields[j] = fields[j-1]; + fields[j-1] = ftmp; + } + } + + bool any_fields_different_endian = false; + for (auto const &f : fields) { + if (is_type_different_to_arch_endianness(f.field_type)) { + // NOTE(bill): Just be slow for this, to be correct + any_fields_different_endian = true; + break; + } + } + + if (!any_fields_different_endian && + fields.count == bt->BitField.fields.count) { + // SINGLE INTEGER BACKING ONLY + + Type *backing_type = core_type(bt->BitField.backing_type); + GB_ASSERT(is_type_integer(backing_type) || + (is_type_array(backing_type) && is_type_integer(backing_type->Array.elem))); + + // NOTE(bill): all fields are present + // this means no masking is necessary since on write, the bits will be overridden + + lbValue dst_byte_ptr = lb_emit_conv(p, v.addr, t_u8_ptr); + u64 total_bit_size = cast(u64)(8*type_size_of(bt)); + + if (is_type_integer(backing_type)) { + LLVMTypeRef lit = lb_type(p->module, backing_type); + + LLVMValueRef res = LLVMConstInt(lit, 0, false); + + for (isize i = 0; i < fields.count; i++) { + auto const &f = fields[i]; + + LLVMValueRef mask = LLVMConstInt(lit, 1, false); + mask = LLVMConstShl(mask, LLVMConstInt(lit, f.bit_size, false)); + mask = LLVMConstSub(mask, LLVMConstInt(lit, 1, false)); + + LLVMValueRef elem = values[i].value; + elem = LLVMBuildZExt(p->builder, elem, lit, ""); + elem = LLVMBuildAnd(p->builder, elem, mask, ""); + + elem = LLVMBuildShl(p->builder, elem, LLVMConstInt(lit, f.bit_offset, false), ""); + + res = LLVMBuildOr(p->builder, res, elem, ""); + } + + LLVMBuildStore(p->builder, res, v.addr.value); + } else if (is_type_array(backing_type)) { + // ARRAY OF INTEGER BACKING + + i64 array_count = backing_type->Array.count; + LLVMTypeRef lit = lb_type(p->module, core_type(backing_type->Array.elem)); + gb_unused(array_count); + gb_unused(lit); + + LLVMValueRef *elems = gb_alloc_array(temporary_allocator(), LLVMValueRef, array_count); + for (i64 i = 0; i < array_count; i++) { + elems[i] = LLVMConstInt(lit, 0, false); + } + + u64 elem_bit_size = cast(u64)(8*type_size_of(backing_type->Array.elem)); + u64 curr_bit_offset = 0; + for (isize i = 0; i < fields.count; i++) { + auto const &f = fields[i]; + + LLVMValueRef val = values[i].value; + LLVMTypeRef vt = lb_type(p->module, values[i].type); + for (u64 bits_to_set = f.bit_size; + bits_to_set > 0; + /**/) { + i64 elem_idx = curr_bit_offset/elem_bit_size; + u64 elem_bit_offset = curr_bit_offset%elem_bit_size; + + u64 mask_width = gb_min(bits_to_set, elem_bit_size-elem_bit_offset); + GB_ASSERT(mask_width > 0); + bits_to_set -= mask_width; + + LLVMValueRef mask = LLVMConstInt(vt, 1, false); + mask = LLVMConstShl(mask, LLVMConstInt(vt, mask_width, false)); + mask = LLVMConstSub(mask, LLVMConstInt(vt, 1, false)); + + LLVMValueRef to_set = LLVMBuildAnd(p->builder, val, mask, ""); + + if (elem_bit_offset != 0) { + to_set = LLVMBuildShl(p->builder, to_set, LLVMConstInt(vt, elem_bit_offset, false), ""); + } + to_set = LLVMBuildTrunc(p->builder, to_set, lit, ""); + + if (LLVMIsNull(elems[elem_idx])) { + elems[elem_idx] = to_set; // don't even bother doing `0 | to_set` + } else { + elems[elem_idx] = LLVMBuildOr(p->builder, elems[elem_idx], to_set, ""); + } + + if (mask_width != 0) { + val = LLVMBuildLShr(p->builder, val, LLVMConstInt(vt, mask_width, false), ""); + } + curr_bit_offset += mask_width; + } + + GB_ASSERT(curr_bit_offset == f.bit_offset + f.bit_size); + } + + for (i64 i = 0; i < array_count; i++) { + LLVMValueRef elem_ptr = LLVMBuildStructGEP2(p->builder, lb_type(p->module, backing_type), v.addr.value, cast(unsigned)i, ""); + LLVMBuildStore(p->builder, elems[i], elem_ptr); + } + } else { + // SLOW STORAGE + + for_array(i, fields) { + auto const &f = fields[i]; + + if ((f.bit_offset & 7) == 0) { + u64 unpacked_bit_size = cast(u64)(8*type_size_of(f.field_type)); + u64 byte_size = (f.bit_size+7)/8; + + if (f.bit_offset + unpacked_bit_size <= total_bit_size) { + byte_size = unpacked_bit_size/8; + } + lbValue dst = lb_emit_ptr_offset(p, dst_byte_ptr, lb_const_int(p->module, t_int, f.bit_offset/8)); + lbValue src = lb_address_from_load_or_generate_local(p, values[i]); + lb_mem_copy_non_overlapping(p, dst, src, lb_const_int(p->module, t_uintptr, byte_size)); + } else { + lbAddr dst = lb_addr_bit_field(v.addr, f.field_type, f.bit_offset, f.bit_size); + lb_addr_store(p, dst, values[i]); + } + } + } + } else { + // individual storing + for_array(i, values) { + auto const &f = fields[i]; + lbAddr dst = lb_addr_bit_field(v.addr, f.field_type, f.bit_offset, f.bit_size); + lb_addr_store(p, dst, values[i]); + } + } + return v; + } case Type_Struct: { // TODO(bill): "constant" '#raw_union's are not initialized constantly at the moment. @@ -4740,7 +4929,7 @@ gb_internal lbAddr lb_build_addr_internal(lbProcedure *p, Ast *expr) { u8 bit_size = bf_type->BitField.bit_sizes[index]; i64 bit_offset = bf_type->BitField.bit_offsets[index]; - return lb_addr_bit_field(ptr, f->type, index, bit_offset, bit_size); + return lb_addr_bit_field(ptr, f->type, bit_offset, bit_size); } { diff --git a/src/llvm_backend_general.cpp b/src/llvm_backend_general.cpp index 0d8d9258a..15cbb7c71 100644 --- a/src/llvm_backend_general.cpp +++ b/src/llvm_backend_general.cpp @@ -140,7 +140,7 @@ gb_internal bool lb_init_generator(lbGenerator *gen, Checker *c) { } gen->default_module.gen = gen; - map_set(&gen->modules, cast(void *)nullptr, &gen->default_module); + map_set(&gen->modules, cast(void *)1, &gen->default_module); lb_init_module(&gen->default_module, c); @@ -211,7 +211,7 @@ gb_internal void lb_loop_end(lbProcedure *p, lbLoopData const &data) { gb_internal void lb_make_global_private_const(LLVMValueRef global_data) { - LLVMSetLinkage(global_data, LLVMPrivateLinkage); + LLVMSetLinkage(global_data, LLVMLinkerPrivateLinkage); LLVMSetUnnamedAddress(global_data, LLVMGlobalUnnamedAddr); LLVMSetGlobalConstant(global_data, true); } @@ -450,14 +450,13 @@ gb_internal lbAddr lb_addr_swizzle_large(lbValue addr, Type *array_type, Slicebuilder, + shifted_value.value, + LLVMConstInt(LLVMTypeOf(shifted_value.value), shift_amount, false), ""); - auto args = array_make(temporary_allocator(), 4); - args[0] = dst; - args[1] = lb_address_from_load_or_generate_local(p, value); - args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); - args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); - lb_emit_runtime_call(p, "__write_bits", args); + lbValue src = lb_address_from_load_or_generate_local(p, shifted_value); + + auto args = array_make(temporary_allocator(), 4); + args[0] = dst; + args[1] = src; + args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); + args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); + lb_emit_runtime_call(p, "__write_bits", args); + } else if ((addr.bitfield.bit_offset % 8) == 0 && + (addr.bitfield.bit_size % 8) == 0) { + lbValue src = lb_address_from_load_or_generate_local(p, value); + + lbValue byte_offset = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset/8); + lbValue byte_size = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size/8); + lbValue dst_offset = lb_emit_conv(p, dst, t_u8_ptr); + dst_offset = lb_emit_ptr_offset(p, dst_offset, byte_offset); + lb_mem_copy_non_overlapping(p, dst_offset, src, byte_size); + } else { + lbValue src = lb_address_from_load_or_generate_local(p, value); + + auto args = array_make(temporary_allocator(), 4); + args[0] = dst; + args[1] = src; + args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); + args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); + lb_emit_runtime_call(p, "__write_bits", args); + } return; } else if (addr.kind == lbAddr_RelativePointer) { Type *rel_ptr = base_type(lb_addr_type(addr)); @@ -954,16 +980,6 @@ gb_internal void lb_addr_store(lbProcedure *p, lbAddr addr, lbValue value) { GB_ASSERT(value.value != nullptr); value = lb_emit_conv(p, value, lb_addr_type(addr)); - // if (lb_is_const_or_global(value)) { - // // NOTE(bill): Just bypass the actual storage and set the initializer - // if (LLVMGetValueKind(addr.addr.value) == LLVMGlobalVariableValueKind) { - // LLVMValueRef dst = addr.addr.value; - // LLVMValueRef src = value.value; - // LLVMSetInitializer(dst, src); - // return; - // } - // } - lb_emit_store(p, addr.addr, value); } @@ -1098,23 +1114,76 @@ gb_internal lbValue lb_addr_load(lbProcedure *p, lbAddr const &addr) { GB_ASSERT(addr.addr.value != nullptr); if (addr.kind == lbAddr_BitField) { + Type *ct = core_type(addr.bitfield.type); + bool do_mask = false; + if (is_type_unsigned(ct) || is_type_boolean(ct)) { + // Mask + if (addr.bitfield.bit_size != 8*type_size_of(ct)) { + do_mask = true; + } + } + + i64 total_bitfield_bit_size = 8*type_size_of(lb_addr_type(addr)); + i64 dst_byte_size = type_size_of(addr.bitfield.type); lbAddr dst = lb_add_local_generated(p, addr.bitfield.type, true); lbValue src = addr.addr; - auto args = array_make(temporary_allocator(), 4); - args[0] = dst.addr; - args[1] = src; - args[2] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); - args[3] = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); - lb_emit_runtime_call(p, "__read_bits", args); + lbValue bit_offset = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_offset); + lbValue bit_size = lb_const_int(p->module, t_uintptr, addr.bitfield.bit_size); + lbValue byte_offset = lb_const_int(p->module, t_uintptr, (addr.bitfield.bit_offset+7)/8); + lbValue byte_size = lb_const_int(p->module, t_uintptr, (addr.bitfield.bit_size+7)/8); - lbValue r = lb_addr_load(p, dst); + GB_ASSERT(type_size_of(addr.bitfield.type) >= ((addr.bitfield.bit_size+7)/8)); - if (!is_type_unsigned(core_type(addr.bitfield.type))) { + lbValue r = {}; + if (is_type_endian_big(addr.bitfield.type)) { + auto args = array_make(temporary_allocator(), 4); + args[0] = dst.addr; + args[1] = src; + args[2] = bit_offset; + args[3] = bit_size; + lb_emit_runtime_call(p, "__read_bits", args); + + LLVMValueRef shift_amount = LLVMConstInt( + lb_type(p->module, lb_addr_type(dst)), + 8*dst_byte_size - addr.bitfield.bit_size, + false + ); + r = lb_addr_load(p, dst); + r.value = LLVMBuildShl(p->builder, r.value, shift_amount, ""); + } else if ((addr.bitfield.bit_offset % 8) == 0) { + lbValue copy_size = byte_size; + lbValue src_offset = lb_emit_conv(p, src, t_u8_ptr); + src_offset = lb_emit_ptr_offset(p, src_offset, byte_offset); + if (addr.bitfield.bit_offset + dst_byte_size <= total_bitfield_bit_size) { + do_mask = true; + copy_size = lb_const_int(p->module, t_uintptr, dst_byte_size); + } + lb_mem_copy_non_overlapping(p, dst.addr, src_offset, copy_size, false); + r = lb_addr_load(p, dst); + } else { + auto args = array_make(temporary_allocator(), 4); + args[0] = dst.addr; + args[1] = src; + args[2] = bit_offset; + args[3] = bit_size; + lb_emit_runtime_call(p, "__read_bits", args); + r = lb_addr_load(p, dst); + } + + Type *t = addr.bitfield.type; + + if (do_mask) { + GB_ASSERT(addr.bitfield.bit_size < 8*type_size_of(ct)); + + lbValue mask = lb_const_int(p->module, t, (1ull<module, t, 1ull<<(addr.bitfield.bit_size-1)); r = lb_emit_arith(p, Token_Xor, r, m, t); r = lb_emit_arith(p, Token_Sub, r, m, t); @@ -2080,7 +2149,7 @@ gb_internal LLVMTypeRef lb_type_internal(lbModule *m, Type *type) { break; case Type_Map: - init_map_internal_types(type); + init_map_internal_debug_types(type); GB_ASSERT(t_raw_map != nullptr); return lb_type_internal(m, t_raw_map); diff --git a/src/llvm_backend_proc.cpp b/src/llvm_backend_proc.cpp index bb4aed3f1..f73698d34 100644 --- a/src/llvm_backend_proc.cpp +++ b/src/llvm_backend_proc.cpp @@ -597,16 +597,7 @@ gb_internal void lb_begin_procedure_body(lbProcedure *p) { lbValue ptr = lb_address_from_load_or_generate_local(p, param); GB_ASSERT(LLVMIsAAllocaInst(ptr.value)); lb_add_entity(p->module, e, ptr); - - lbBlock *block = p->decl_block; - if (original_value != value) { - block = p->curr_block; - } - LLVMValueRef debug_storage_value = value; - if (original_value != value && LLVMIsALoadInst(value)) { - debug_storage_value = LLVMGetOperand(value, 0); - } - lb_add_debug_param_variable(p, debug_storage_value, e->type, e->token, param_index+1, block, arg_type->kind); + lb_add_debug_param_variable(p, ptr.value, e->type, e->token, param_index+1, p->curr_block); } } else if (arg_type->kind == lbArg_Indirect) { if (e->token.string.len != 0 && !is_blank_ident(e->token.string)) { @@ -614,7 +605,7 @@ gb_internal void lb_begin_procedure_body(lbProcedure *p) { ptr.value = LLVMGetParam(p->value, param_offset+param_index); ptr.type = alloc_type_pointer(e->type); lb_add_entity(p->module, e, ptr); - lb_add_debug_param_variable(p, ptr.value, e->type, e->token, param_index+1, p->decl_block, arg_type->kind); + lb_add_debug_param_variable(p, ptr.value, e->type, e->token, param_index+1, p->decl_block); } } } diff --git a/src/llvm_backend_stmt.cpp b/src/llvm_backend_stmt.cpp index 24dd321f6..851433415 100644 --- a/src/llvm_backend_stmt.cpp +++ b/src/llvm_backend_stmt.cpp @@ -802,8 +802,19 @@ gb_internal void lb_build_range_enum(lbProcedure *p, Type *enum_type, Type *val_ if (done_) *done_ = done; } -gb_internal void lb_build_range_tuple(lbProcedure *p, Ast *expr, Type *val0_type, Type *val1_type, - lbValue *val0_, lbValue *val1_, lbBlock **loop_, lbBlock **done_) { +gb_internal void lb_build_range_tuple(lbProcedure *p, AstRangeStmt *rs, Scope *scope) { + Ast *expr = unparen_expr(rs->expr); + + Type *expr_type = type_of_expr(expr); + Type *et = base_type(type_deref(expr_type)); + GB_ASSERT(et->kind == Type_Tuple); + + i32 value_count = cast(i32)et->Tuple.variables.count; + + lbValue *values = gb_alloc_array(permanent_allocator(), lbValue, value_count); + + lb_open_scope(p, scope); + lbBlock *loop = lb_create_block(p, "for.tuple.loop"); lb_emit_jump(p, loop); lb_start_block(p, loop); @@ -821,11 +832,26 @@ gb_internal void lb_build_range_tuple(lbProcedure *p, Ast *expr, Type *val0_type lb_emit_if(p, cond, body, done); lb_start_block(p, body); + for (i32 i = 0; i < value_count; i++) { + values[i] = lb_emit_tuple_ev(p, tuple_value, i); + } - if (val0_) *val0_ = lb_emit_tuple_ev(p, tuple_value, 0); - if (val1_) *val1_ = lb_emit_tuple_ev(p, tuple_value, 1); - if (loop_) *loop_ = loop; - if (done_) *done_ = done; + GB_ASSERT(rs->vals.count <= value_count); + for (isize i = 0; i < rs->vals.count; i++) { + Ast *val = rs->vals[i]; + if (val != nullptr) { + lb_store_range_stmt_val(p, val, values[i]); + } + } + + lb_push_target_list(p, rs->label, done, loop, nullptr); + + lb_build_stmt(p, rs->body); + + lb_close_scope(p, lbDeferExit_Default, nullptr); + lb_pop_target_list(p); + lb_emit_jump(p, loop); + lb_start_block(p, done); } gb_internal void lb_build_range_stmt_struct_soa(lbProcedure *p, AstRangeStmt *rs, Scope *scope) { @@ -968,6 +994,17 @@ gb_internal void lb_build_range_stmt(lbProcedure *p, AstRangeStmt *rs, Scope *sc } } + TypeAndValue tav = type_and_value_of_expr(expr); + if (tav.mode != Addressing_Type) { + Type *expr_type = type_of_expr(expr); + Type *et = base_type(type_deref(expr_type)); + if (et->kind == Type_Tuple) { + lb_build_range_tuple(p, rs, scope); + return; + } + } + + lb_open_scope(p, scope); Ast *val0 = rs->vals.count > 0 ? lb_strip_and_prefix(rs->vals[0]) : nullptr; @@ -986,7 +1023,6 @@ gb_internal void lb_build_range_stmt(lbProcedure *p, AstRangeStmt *rs, Scope *sc lbBlock *loop = nullptr; lbBlock *done = nullptr; bool is_map = false; - TypeAndValue tav = type_and_value_of_expr(expr); if (tav.mode == Addressing_Type) { lb_build_range_enum(p, type_deref(tav.type), val0_type, &val, &key, &loop, &done); @@ -1062,8 +1098,7 @@ gb_internal void lb_build_range_stmt(lbProcedure *p, AstRangeStmt *rs, Scope *sc break; } case Type_Tuple: - lb_build_range_tuple(p, expr, val0_type, val1_type, &val, &key, &loop, &done); - break; + GB_PANIC("Should be handled already"); case Type_BitSet: { lbModule *m = p->module; diff --git a/src/llvm_backend_type.cpp b/src/llvm_backend_type.cpp index 93e2874a5..2c4abbb4d 100644 --- a/src/llvm_backend_type.cpp +++ b/src/llvm_backend_type.cpp @@ -2,7 +2,7 @@ gb_internal isize lb_type_info_index(CheckerInfo *info, Type *type, bool err_on_ auto *set = &info->minimum_dependency_type_info_set; isize index = type_info_index(info, type, err_on_not_found); if (index >= 0) { - auto *found = map_get(set, index); + auto *found = map_get(set, index+1); if (found) { GB_ASSERT(*found >= 0); return *found + 1; @@ -249,9 +249,7 @@ gb_internal void lb_setup_type_info_data_giant_array(lbModule *m, i64 global_typ char name[64] = {}; gb_snprintf(name, 63, "__$ti-%lld", cast(long long)index); LLVMValueRef g = LLVMAddGlobal(m->mod, type, name); - LLVMSetLinkage(g, LLVMInternalLinkage); - LLVMSetUnnamedAddress(g, LLVMGlobalUnnamedAddr); - LLVMSetGlobalConstant(g, true); + lb_make_global_private_const(g); return g; }; @@ -903,7 +901,7 @@ gb_internal void lb_setup_type_info_data_giant_array(lbModule *m, i64 global_typ case Type_Map: { tag_type = t_type_info_map; - init_map_internal_types(t); + init_map_internal_debug_types(t); LLVMValueRef vals[3] = { get_type_info_ptr(m, t->Map.key), @@ -1103,6 +1101,7 @@ gb_internal void lb_setup_type_info_data_giant_array(lbModule *m, i64 global_typ LLVMValueRef giant_const = LLVMConstArray(lb_type(m, t_type_info_ptr), giant_const_values, cast(unsigned)global_type_info_data_entity_count); LLVMValueRef giant_array = lb_global_type_info_data_ptr(m).value; LLVMSetInitializer(giant_array, giant_const); + lb_make_global_private_const(giant_array); } @@ -1132,4 +1131,7 @@ gb_internal void lb_setup_type_info_data(lbModule *m) { // NOTE(bill): Setup typ LLVMValueRef slice = llvm_const_slice_internal(m, data, len); LLVMSetInitializer(global_type_table.value, slice); + + // force it to be constant + LLVMSetGlobalConstant(global_type_table.value, true); } diff --git a/src/llvm_backend_utility.cpp b/src/llvm_backend_utility.cpp index 865c3f1ec..c01ab0692 100644 --- a/src/llvm_backend_utility.cpp +++ b/src/llvm_backend_utility.cpp @@ -97,15 +97,12 @@ gb_internal void lb_mem_zero_ptr(lbProcedure *p, LLVMValueRef ptr, Type *type, u LLVMTypeRef llvm_type = lb_type(p->module, type); LLVMTypeKind kind = LLVMGetTypeKind(llvm_type); - + i64 sz = type_size_of(type); switch (kind) { case LLVMStructTypeKind: case LLVMArrayTypeKind: - { - // NOTE(bill): Enforce zeroing through memset to make sure padding is zeroed too - i32 sz = cast(i32)type_size_of(type); - lb_mem_zero_ptr_internal(p, ptr, lb_const_int(p->module, t_int, sz).value, alignment, false); - } + // NOTE(bill): Enforce zeroing through memset to make sure padding is zeroed too + lb_mem_zero_ptr_internal(p, ptr, lb_const_int(p->module, t_int, sz).value, alignment, false); break; default: LLVMBuildStore(p->builder, LLVMConstNull(lb_type(p->module, type)), ptr); @@ -728,30 +725,32 @@ gb_internal lbValue lb_emit_union_cast(lbProcedure *p, lbValue value, Type *type lb_start_block(p, end_block); if (!is_tuple) { - GB_ASSERT((p->state_flags & StateFlag_no_type_assert) == 0); - // NOTE(bill): Panic on invalid conversion - Type *dst_type = tuple->Tuple.variables[0]->type; + if (!build_context.no_type_assert) { + GB_ASSERT((p->state_flags & StateFlag_no_type_assert) == 0); + // NOTE(bill): Panic on invalid conversion + Type *dst_type = tuple->Tuple.variables[0]->type; - isize arg_count = 7; - if (build_context.no_rtti) { - arg_count = 4; + isize arg_count = 7; + if (build_context.no_rtti) { + arg_count = 4; + } + + lbValue ok = lb_emit_load(p, lb_emit_struct_ep(p, v.addr, 1)); + auto args = array_make(permanent_allocator(), arg_count); + args[0] = ok; + + args[1] = lb_const_string(m, get_file_path_string(pos.file_id)); + args[2] = lb_const_int(m, t_i32, pos.line); + args[3] = lb_const_int(m, t_i32, pos.column); + + if (!build_context.no_rtti) { + args[4] = lb_typeid(m, src_type); + args[5] = lb_typeid(m, dst_type); + args[6] = lb_emit_conv(p, value_, t_rawptr); + } + lb_emit_runtime_call(p, "type_assertion_check2", args); } - lbValue ok = lb_emit_load(p, lb_emit_struct_ep(p, v.addr, 1)); - auto args = array_make(permanent_allocator(), arg_count); - args[0] = ok; - - args[1] = lb_const_string(m, get_file_path_string(pos.file_id)); - args[2] = lb_const_int(m, t_i32, pos.line); - args[3] = lb_const_int(m, t_i32, pos.column); - - if (!build_context.no_rtti) { - args[4] = lb_typeid(m, src_type); - args[5] = lb_typeid(m, dst_type); - args[6] = lb_emit_conv(p, value_, t_rawptr); - } - lb_emit_runtime_call(p, "type_assertion_check2", args); - return lb_emit_load(p, lb_emit_struct_ep(p, v.addr, 0)); } return lb_addr_load(p, v); @@ -806,25 +805,27 @@ gb_internal lbAddr lb_emit_any_cast_addr(lbProcedure *p, lbValue value, Type *ty if (!is_tuple) { // NOTE(bill): Panic on invalid conversion - lbValue ok = lb_emit_load(p, lb_emit_struct_ep(p, v.addr, 1)); + if (!build_context.no_type_assert) { + lbValue ok = lb_emit_load(p, lb_emit_struct_ep(p, v.addr, 1)); - isize arg_count = 7; - if (build_context.no_rtti) { - arg_count = 4; + isize arg_count = 7; + if (build_context.no_rtti) { + arg_count = 4; + } + auto args = array_make(permanent_allocator(), arg_count); + args[0] = ok; + + args[1] = lb_const_string(m, get_file_path_string(pos.file_id)); + args[2] = lb_const_int(m, t_i32, pos.line); + args[3] = lb_const_int(m, t_i32, pos.column); + + if (!build_context.no_rtti) { + args[4] = any_typeid; + args[5] = dst_typeid; + args[6] = lb_emit_struct_ev(p, value, 0); + } + lb_emit_runtime_call(p, "type_assertion_check2", args); } - auto args = array_make(permanent_allocator(), arg_count); - args[0] = ok; - - args[1] = lb_const_string(m, get_file_path_string(pos.file_id)); - args[2] = lb_const_int(m, t_i32, pos.line); - args[3] = lb_const_int(m, t_i32, pos.column); - - if (!build_context.no_rtti) { - args[4] = any_typeid; - args[5] = dst_typeid; - args[6] = lb_emit_struct_ev(p, value, 0); - } - lb_emit_runtime_call(p, "type_assertion_check2", args); return lb_addr(lb_emit_struct_ep(p, v.addr, 0)); } @@ -1125,7 +1126,7 @@ gb_internal lbValue lb_emit_struct_ep(lbProcedure *p, lbValue s, i32 index) { case 3: result_type = t_allocator; break; } } else if (is_type_map(t)) { - init_map_internal_types(t); + init_map_internal_debug_types(t); Type *itp = alloc_type_pointer(t_raw_map); s = lb_emit_transmute(p, s, itp); @@ -1264,7 +1265,7 @@ gb_internal lbValue lb_emit_struct_ev(lbProcedure *p, lbValue s, i32 index) { case Type_Map: { - init_map_internal_types(t); + init_map_internal_debug_types(t); switch (index) { case 0: result_type = get_struct_field_type(t_raw_map, 0); break; case 1: result_type = get_struct_field_type(t_raw_map, 1); break; diff --git a/src/main.cpp b/src/main.cpp index 2dbb72ca2..ee7de7f81 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -198,7 +198,12 @@ gb_internal void print_usage_line(i32 indent, char const *fmt, ...) { gb_printf("\n"); } -gb_internal void usage(String argv0) { +gb_internal void usage(String argv0, String argv1 = {}) { + if (argv1 == "run.") { + print_usage_line(0, "Did you mean 'odin run .'?"); + } else if (argv1 == "build.") { + print_usage_line(0, "Did you mean 'odin build .'?"); + } print_usage_line(0, "%.*s is a tool for managing Odin source code.", LIT(argv0)); print_usage_line(0, "Usage:"); print_usage_line(1, "%.*s command [arguments]", LIT(argv0)); @@ -212,6 +217,7 @@ gb_internal void usage(String argv0) { print_usage_line(1, "doc Generates documentation on a directory of .odin files."); print_usage_line(1, "version Prints version."); print_usage_line(1, "report Prints information useful to reporting a bug."); + print_usage_line(1, "root Prints the root path where Odin looks for the builtin collections."); print_usage_line(0, ""); print_usage_line(0, "For further details on a command, invoke command help:"); print_usage_line(1, "e.g. `odin build -help` or `odin help build`"); @@ -242,6 +248,7 @@ enum BuildFlagKind { BuildFlag_Debug, BuildFlag_DisableAssert, BuildFlag_NoBoundsCheck, + BuildFlag_NoTypeAssert, BuildFlag_NoDynamicLiterals, BuildFlag_NoCRT, BuildFlag_NoEntryPoint, @@ -253,6 +260,8 @@ enum BuildFlagKind { BuildFlag_Vet, BuildFlag_VetShadowing, BuildFlag_VetUnused, + BuildFlag_VetUnusedImports, + BuildFlag_VetUnusedVariables, BuildFlag_VetUsingStmt, BuildFlag_VetUsingParam, BuildFlag_VetStyle, @@ -333,12 +342,12 @@ struct BuildFlag { String name; BuildFlagParamKind param_kind; u32 command_support; - bool allow_mulitple; + bool allow_multiple; }; -gb_internal void add_flag(Array *build_flags, BuildFlagKind kind, String name, BuildFlagParamKind param_kind, u32 command_support, bool allow_mulitple=false) { - BuildFlag flag = {kind, name, param_kind, command_support, allow_mulitple}; +gb_internal void add_flag(Array *build_flags, BuildFlagKind kind, String name, BuildFlagParamKind param_kind, u32 command_support, bool allow_multiple=false) { + BuildFlag flag = {kind, name, param_kind, command_support, allow_multiple}; array_add(build_flags, flag); } @@ -433,6 +442,7 @@ gb_internal bool parse_build_flags(Array args) { add_flag(&build_flags, BuildFlag_Debug, str_lit("debug"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_DisableAssert, str_lit("disable-assert"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_NoBoundsCheck, str_lit("no-bounds-check"), BuildFlagParam_None, Command__does_check); + add_flag(&build_flags, BuildFlag_NoTypeAssert, str_lit("no-type-assert"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_NoThreadLocal, str_lit("no-thread-local"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_NoDynamicLiterals, str_lit("no-dynamic-literals"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_NoCRT, str_lit("no-crt"), BuildFlagParam_None, Command__does_build); @@ -444,6 +454,8 @@ gb_internal bool parse_build_flags(Array args) { add_flag(&build_flags, BuildFlag_Vet, str_lit("vet"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_VetUnused, str_lit("vet-unused"), BuildFlagParam_None, Command__does_check); + add_flag(&build_flags, BuildFlag_VetUnusedVariables, str_lit("vet-unused-variables"), BuildFlagParam_None, Command__does_check); + add_flag(&build_flags, BuildFlag_VetUnusedImports, str_lit("vet-unused-imports"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_VetShadowing, str_lit("vet-shadowing"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_VetUsingStmt, str_lit("vet-using-stmt"), BuildFlagParam_None, Command__does_check); add_flag(&build_flags, BuildFlag_VetUsingParam, str_lit("vet-using-param"), BuildFlagParam_None, Command__does_check); @@ -1008,6 +1020,9 @@ gb_internal bool parse_build_flags(Array args) { case BuildFlag_NoBoundsCheck: build_context.no_bounds_check = true; break; + case BuildFlag_NoTypeAssert: + build_context.no_type_assert = true; + break; case BuildFlag_NoDynamicLiterals: build_context.no_dynamic_literals = true; break; @@ -1026,10 +1041,9 @@ gb_internal bool parse_build_flags(Array args) { case BuildFlag_UseSeparateModules: build_context.use_separate_modules = true; break; - case BuildFlag_NoThreadedChecker: { + case BuildFlag_NoThreadedChecker: build_context.no_threaded_checker = true; break; - } case BuildFlag_ShowDebugMessages: build_context.show_debug_messages = true; break; @@ -1037,12 +1051,14 @@ gb_internal bool parse_build_flags(Array args) { build_context.vet_flags |= VetFlag_All; break; - case BuildFlag_VetUnused: build_context.vet_flags |= VetFlag_Unused; break; - case BuildFlag_VetShadowing: build_context.vet_flags |= VetFlag_Shadowing; break; - case BuildFlag_VetUsingStmt: build_context.vet_flags |= VetFlag_UsingStmt; break; - case BuildFlag_VetUsingParam: build_context.vet_flags |= VetFlag_UsingParam; break; - case BuildFlag_VetStyle: build_context.vet_flags |= VetFlag_Style; break; - case BuildFlag_VetSemicolon: build_context.vet_flags |= VetFlag_Semicolon; break; + case BuildFlag_VetUnusedVariables: build_context.vet_flags |= VetFlag_UnusedVariables; break; + case BuildFlag_VetUnusedImports: build_context.vet_flags |= VetFlag_UnusedImports; break; + case BuildFlag_VetUnused: build_context.vet_flags |= VetFlag_Unused; break; + case BuildFlag_VetShadowing: build_context.vet_flags |= VetFlag_Shadowing; break; + case BuildFlag_VetUsingStmt: build_context.vet_flags |= VetFlag_UsingStmt; break; + case BuildFlag_VetUsingParam: build_context.vet_flags |= VetFlag_UsingParam; break; + case BuildFlag_VetStyle: build_context.vet_flags |= VetFlag_Style; break; + case BuildFlag_VetSemicolon: build_context.vet_flags |= VetFlag_Semicolon; break; case BuildFlag_IgnoreUnknownAttributes: build_context.ignore_unknown_attributes = true; @@ -1347,7 +1363,7 @@ gb_internal bool parse_build_flags(Array args) { } } - if (!bf.allow_mulitple) { + if (!bf.allow_multiple) { set_flags[bf.kind] = ok; } } @@ -1844,6 +1860,10 @@ gb_internal void print_show_help(String const arg0, String const &command) { print_usage_line(2, "Disables bounds checking program wide."); print_usage_line(0, ""); + print_usage_line(1, "-no-type-assert"); + print_usage_line(2, "Disables type assertion checking program wide."); + print_usage_line(0, ""); + print_usage_line(1, "-no-crt"); print_usage_line(2, "Disables automatic linking with the C Run Time."); print_usage_line(0, ""); @@ -1875,6 +1895,8 @@ gb_internal void print_show_help(String const arg0, String const &command) { print_usage_line(2, "Does extra checks on the code."); print_usage_line(2, "Extra checks include:"); print_usage_line(3, "-vet-unused"); + print_usage_line(3, "-vet-unused-variables"); + print_usage_line(3, "-vet-unused-imports"); print_usage_line(3, "-vet-shadowing"); print_usage_line(3, "-vet-using-stmt"); print_usage_line(0, ""); @@ -1883,6 +1905,14 @@ gb_internal void print_show_help(String const arg0, String const &command) { print_usage_line(2, "Checks for unused declarations."); print_usage_line(0, ""); + print_usage_line(1, "-vet-unused-variables"); + print_usage_line(2, "Checks for unused variable declarations."); + print_usage_line(0, ""); + + print_usage_line(1, "-vet-unused-imports"); + print_usage_line(2, "Checks for unused import declarations."); + print_usage_line(0, ""); + print_usage_line(1, "-vet-shadowing"); print_usage_line(2, "Checks for variable shadowing within procedures."); print_usage_line(0, ""); @@ -2557,8 +2587,15 @@ int main(int arg_count, char const **arg_ptr) { print_show_help(args[0], args[2]); return 0; } + } else if (command == "root") { + gb_printf("%.*s", LIT(odin_root_dir())); + return 0; } else { - usage(args[0]); + String argv1 = {}; + if (args.count > 1) { + argv1 = args[1]; + } + usage(args[0], argv1); return 1; } diff --git a/src/parser.cpp b/src/parser.cpp index 01a3069ff..f4d3dc48d 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -6114,7 +6114,13 @@ gb_internal bool parse_file(Parser *p, AstFile *f) { CommentGroup *docs = f->lead_comment; if (f->curr_token.kind != Token_package) { + ERROR_BLOCK(); syntax_error(f->curr_token, "Expected a package declaration at the beginning of the file"); + // IMPORTANT NOTE(bill): this is technically a race condition with the suggestion, but it's ony a suggession + // so in practice is should be "fine" + if (f->pkg && f->pkg->name != "") { + error_line("\tSuggestion: Add 'package %.*s' to the top of the file\n", LIT(f->pkg->name)); + } return false; } diff --git a/src/ptr_map.cpp b/src/ptr_map.cpp index 23278014f..362e412ba 100644 --- a/src/ptr_map.cpp +++ b/src/ptr_map.cpp @@ -16,23 +16,21 @@ struct MapFindResult { }; enum : MapIndex { MAP_SENTINEL = ~(MapIndex)0 }; +static void *const MAP_TOMBSTONE = (void *)~(uintptr)0; template struct PtrMapEntry { static_assert(sizeof(K) == sizeof(void *), "Key size must be pointer size"); - K key; - V value; - MapIndex next; + K key; + V value; }; template struct PtrMap { - MapIndex * hashes; - usize hashes_count; PtrMapEntry *entries; u32 count; - u32 entries_capacity; + u32 capacity; }; @@ -69,7 +67,6 @@ template gb_internal void map_grow (PtrMap< template gb_internal void map_rehash (PtrMap *h, isize new_count); template gb_internal void map_reserve (PtrMap *h, isize cap); -#if PTR_MAP_ENABLE_MULTI_MAP // Mutlivalued map procedure template gb_internal PtrMapEntry * multi_map_find_first(PtrMap *h, K key); template gb_internal PtrMapEntry * multi_map_find_next (PtrMap *h, PtrMapEntry *e); @@ -79,7 +76,6 @@ template gb_internal void multi_map_get_all (PtrMap< template gb_internal void multi_map_insert (PtrMap *h, K key, V const &value); template gb_internal void multi_map_remove (PtrMap *h, K key, PtrMapEntry *e); template gb_internal void multi_map_remove_all(PtrMap *h, K key); -#endif gb_internal gbAllocator map_allocator(void) { return heap_allocator(); @@ -94,170 +90,141 @@ gb_internal gb_inline void map_init(PtrMap *h, isize capacity) { template gb_internal gb_inline void map_destroy(PtrMap *h) { gbAllocator a = map_allocator(); - gb_free(a, h->hashes); gb_free(a, h->entries); } -template -gb_internal void map__resize_hashes(PtrMap *h, usize count) { - h->hashes_count = cast(u32)resize_array_raw(&h->hashes, map_allocator(), h->hashes_count, count, MAP_CACHE_LINE_SIZE); -} template -gb_internal void map__reserve_entries(PtrMap *h, usize capacity) { - h->entries_capacity = cast(u32)resize_array_raw(&h->entries, map_allocator(), h->entries_capacity, capacity, MAP_CACHE_LINE_SIZE); -} - - -template -gb_internal MapIndex map__add_entry(PtrMap *h, K key) { - PtrMapEntry e = {}; - e.key = key; - e.next = MAP_SENTINEL; - if (h->count+1 >= h->entries_capacity) { - map__reserve_entries(h, gb_max(h->entries_capacity*2, 4)); - } - h->entries[h->count++] = e; - return cast(MapIndex)(h->count-1); -} - -template -gb_internal MapFindResult map__find(PtrMap *h, K key) { - MapFindResult fr = {MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL}; - if (h->hashes_count == 0) { - return fr; +gb_internal void map__insert(PtrMap *h, K key, V const &value) { + if (h->count+1 >= h->capacity) { + map_grow(h); } u32 hash = ptr_map_hash_key(key); - fr.hash_index = cast(MapIndex)(hash & (h->hashes_count-1)); - fr.entry_index = h->hashes[fr.hash_index]; - while (fr.entry_index != MAP_SENTINEL) { - auto *entry = &h->entries[fr.entry_index]; - if (entry->key == key) { - return fr; + u32 mask = h->capacity-1; + MapIndex index = hash & mask; + MapIndex original_index = index; + do { + auto *entry = h->entries+index; + if (!entry->key || entry->key == cast(K)MAP_TOMBSTONE) { + entry->key = key; + entry->value = value; + h->count += 1; + return; } - fr.entry_prev = fr.entry_index; - fr.entry_index = entry->next; - } - return fr; -} + index = (index+1)&mask; + } while (index != original_index); -template -gb_internal MapFindResult map__find_from_entry(PtrMap *h, PtrMapEntry *e) { - MapFindResult fr = {MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL}; - if (h->hashes_count == 0) { - return fr; - } - u32 hash = ptr_map_hash_key(e->key); - fr.hash_index = cast(MapIndex)(hash & (h->hashes_count-1)); - fr.entry_index = h->hashes[fr.hash_index]; - while (fr.entry_index != MAP_SENTINEL) { - if (&h->entries[fr.entry_index] == e) { - return fr; - } - fr.entry_prev = fr.entry_index; - fr.entry_index = h->entries[fr.entry_index].next; - } - return fr; + GB_PANIC("FAILED TO INSERT"); } template gb_internal b32 map__full(PtrMap *h) { - return 0.75f * h->hashes_count <= h->count; + return 0.75f * h->capacity <= h->count; } template gb_internal gb_inline void map_grow(PtrMap *h) { - isize new_count = gb_max(h->hashes_count<<1, 16); - map_rehash(h, new_count); + isize new_capacity = gb_max(h->capacity<<1, 16); + map_reserve(h, new_capacity); } template -gb_internal void map_reset_entries(PtrMap *h) { - for (usize i = 0; i < h->hashes_count; i++) { - h->hashes[i] = MAP_SENTINEL; - } - for (usize i = 0; i < h->count; i++) { - MapFindResult fr; - PtrMapEntry *e = &h->entries[i]; - e->next = MAP_SENTINEL; - fr = map__find_from_entry(h, e); - if (fr.entry_prev == MAP_SENTINEL) { - h->hashes[fr.hash_index] = cast(MapIndex)i; - } else { - h->entries[fr.entry_prev].next = cast(MapIndex)i; - } +gb_internal void try_map_grow(PtrMap *h) { + if (h->capacity == 0 || map__full(h)) { + map_grow(h); } } + template gb_internal void map_reserve(PtrMap *h, isize cap) { - if (h->count*2 < h->hashes_count) { + if (cap < h->capacity) { return; } - map__reserve_entries(h, cap); - map__resize_hashes(h, cap*2); - map_reset_entries(h); -} + cap = next_pow2_isize(cap); + typedef PtrMapEntry EntryType; + PtrMap new_h = {}; + new_h.count = 0; + new_h.capacity = cast(u32)cap; + new_h.entries = gb_alloc_array(map_allocator(), EntryType, new_h.capacity); -template -gb_internal void map_rehash(PtrMap *h, isize new_count) { - map_reserve(h, new_count); + if (h->count) { + for (u32 i = 0; i < h->capacity; i++) { + auto *entry = h->entries+i; + if (entry->key && + entry->key != cast(K)MAP_TOMBSTONE) { + map__insert(&new_h, entry->key, entry->value); + } + } + } + map_destroy(h); + *h = new_h; } template gb_internal V *map_get(PtrMap *h, K key) { - MapIndex hash_index = MAP_SENTINEL; - MapIndex entry_prev = MAP_SENTINEL; - MapIndex entry_index = MAP_SENTINEL; - if (h->hashes_count != 0) { - u32 hash = ptr_map_hash_key(key); - hash_index = cast(MapIndex)(hash & (h->hashes_count-1)); - entry_index = h->hashes[hash_index]; - while (entry_index != MAP_SENTINEL) { - auto *entry = &h->entries[entry_index]; - if (entry->key == key) { - return &entry->value; - } - entry_prev = entry_index; - entry_index = entry->next; - } + if (h->count == 0) { + return nullptr; } + if (key == 0) { + GB_PANIC("0 key"); + } + + u32 hash = ptr_map_hash_key(key); + u32 mask = (h->capacity-1); + u32 index = hash & mask; + u32 original_index = index; + do { + auto *entry = h->entries+index; + if (!entry->key) { + // NOTE(bill): no found, but there isn't any key removal for this hash map + return nullptr; + } else if (entry->key == key) { + return &entry->value; + } + index = (index+1) & mask; + } while (original_index != index); return nullptr; } template -gb_internal V *map_try_get(PtrMap *h, K key, MapFindResult *fr_) { - MapFindResult fr = {MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL}; - if (h->hashes_count != 0) { - u32 hash = ptr_map_hash_key(key); - fr.hash_index = cast(MapIndex)(hash & (h->hashes_count-1)); - fr.entry_index = h->hashes[fr.hash_index]; - while (fr.entry_index != MAP_SENTINEL) { - auto *entry = &h->entries[fr.entry_index]; - if (entry->key == key) { - return &entry->value; - } - fr.entry_prev = fr.entry_index; - fr.entry_index = entry->next; +gb_internal V *map_try_get(PtrMap *h, K key, MapIndex *found_index_) { + if (found_index_) *found_index_ = ~(MapIndex)0; + + if (h->count == 0) { + return nullptr; + } + if (key == 0) { + GB_PANIC("0 key"); + } + + u32 hash = ptr_map_hash_key(key); + u32 mask = (h->capacity-1); + u32 index = hash & mask; + u32 original_index = index; + do { + auto *entry = h->entries+index; + if (!entry->key) { + // NOTE(bill): no found, but there isn't any key removal for this hash map + return nullptr; + } else if (entry->key == key) { + if (found_index_) *found_index_ = index; + return &entry->value; } - } - if (h->hashes_count == 0 || map__full(h)) { - map_grow(h); - } - if (fr_) *fr_ = fr; + index = (index+1) & mask; + } while (original_index != index); return nullptr; } template -gb_internal void map_set_internal_from_try_get(PtrMap *h, K key, V const &value, MapFindResult const &fr) { - MapIndex index = map__add_entry(h, key); - if (fr.entry_prev != MAP_SENTINEL) { - h->entries[fr.entry_prev].next = index; +gb_internal void map_set_internal_from_try_get(PtrMap *h, K key, V const &value, MapIndex found_index) { + if (found_index != MAP_SENTINEL) { + GB_ASSERT(h->entries[found_index].key == key); + h->entries[found_index].value = value; } else { - h->hashes[fr.hash_index] = index; + map_set(h, key, value); } - h->entries[index].value = value; } template @@ -269,116 +236,83 @@ gb_internal V &map_must_get(PtrMap *h, K key) { template gb_internal void map_set(PtrMap *h, K key, V const &value) { - MapIndex index; - MapFindResult fr; - if (h->hashes_count == 0) { - map_grow(h); - } - fr = map__find(h, key); - if (fr.entry_index != MAP_SENTINEL) { - index = fr.entry_index; - } else { - index = map__add_entry(h, key); - if (fr.entry_prev != MAP_SENTINEL) { - h->entries[fr.entry_prev].next = index; - } else { - h->hashes[fr.hash_index] = index; - } - } - h->entries[index].value = value; - - if (map__full(h)) { - map_grow(h); + GB_ASSERT(key != 0); + try_map_grow(h); + auto *found = map_get(h, key); + if (found) { + *found = value; + return; } + map__insert(h, key, value); } // returns true if it previously existed template gb_internal bool map_set_if_not_previously_exists(PtrMap *h, K key, V const &value) { - MapIndex index; - MapFindResult fr; - if (h->hashes_count == 0) { - map_grow(h); - } - fr = map__find(h, key); - if (fr.entry_index != MAP_SENTINEL) { + try_map_grow(h); + auto *found = map_get(h, key); + if (found) { return true; - } else { - index = map__add_entry(h, key); - if (fr.entry_prev != MAP_SENTINEL) { - h->entries[fr.entry_prev].next = index; - } else { - h->hashes[fr.hash_index] = index; - } - } - h->entries[index].value = value; - - if (map__full(h)) { - map_grow(h); } + map__insert(h, key, value); return false; } -template -gb_internal void map__erase(PtrMap *h, MapFindResult const &fr) { - MapFindResult last; - if (fr.entry_prev == MAP_SENTINEL) { - h->hashes[fr.hash_index] = h->entries[fr.entry_index].next; - } else { - h->entries[fr.entry_prev].next = h->entries[fr.entry_index].next; - } - if (fr.entry_index == h->count-1) { - h->count--; - return; - } - h->entries[fr.entry_index] = h->entries[h->count-1]; - h->count--; - - last = map__find(h, h->entries[fr.entry_index].key); - if (last.entry_prev != MAP_SENTINEL) { - h->entries[last.entry_prev].next = fr.entry_index; - } else { - h->hashes[last.hash_index] = fr.entry_index; - } -} - template gb_internal void map_remove(PtrMap *h, K key) { - MapFindResult fr = map__find(h, key); - if (fr.entry_index != MAP_SENTINEL) { - map__erase(h, fr); + MapIndex found_index = 0; + if (map_try_get(h, key, &found_index)) { + h->entries[found_index].key = cast(K)MAP_TOMBSTONE; + h->count -= 1; } } template gb_internal gb_inline void map_clear(PtrMap *h) { h->count = 0; - for (usize i = 0; i < h->hashes_count; i++) { - h->hashes[i] = MAP_SENTINEL; - } + gb_zero_array(h->entries, h->capacity); } #if PTR_MAP_ENABLE_MULTI_MAP template gb_internal PtrMapEntry *multi_map_find_first(PtrMap *h, K key) { - MapIndex i = map__find(h, key).entry_index; - if (i == MAP_SENTINEL) { + if (h->count == 0) { return nullptr; } - return &h->entries[i]; + u32 hash = ptr_map_hash_key(key); + u32 mask = (h->capacity-1); + u32 index = hash & mask; + u32 original_index = index; + do { + auto *entry = h->entries+index; + if (!entry->key) { + // NOTE(bill): no found, but there isn't any key removal for this hash map + return nullptr; + } else if (entry->key == key) { + return entry; + } + index = (index+1) & mask; + } while (original_index != index); + return nullptr; } template gb_internal PtrMapEntry *multi_map_find_next(PtrMap *h, PtrMapEntry *e) { - MapIndex i = e->next; - while (i != MAP_SENTINEL) { - if (h->entries[i].key == e->key) { - return &h->entries[i]; + u32 mask = h->capacity-1; + MapIndex index = cast(MapIndex)(e - h->entries); + MapIndex original_index = index; + do { + index = (index+1)&mask; + auto *entry = h->entries+index; + if (!entry->key) { + return nullptr; } - i = h->entries[i].next; - } + if (entry->key == e->key) { + return entry; + } + } while (original_index != index); return nullptr; } @@ -405,34 +339,16 @@ gb_internal void multi_map_get_all(PtrMap *h, K key, V *items) { template gb_internal void multi_map_insert(PtrMap *h, K key, V const &value) { - MapFindResult fr; - MapIndex i; - if (h->hashes_count == 0) { - map_grow(h); - } - // Make - fr = map__find(h, key); - i = map__add_entry(h, key); - if (fr.entry_prev == MAP_SENTINEL) { - h->hashes[fr.hash_index] = i; - } else { - h->entries[fr.entry_prev].next = i; - } - h->entries[i].next = fr.entry_index; - h->entries[i].value = value; - // Grow if needed - if (map__full(h)) { - map_grow(h); - } + try_map_grow(h); + map__insert(h, key, value); } -template -gb_internal void multi_map_remove(PtrMap *h, K key, PtrMapEntry *e) { - MapFindResult fr = map__find_from_entry(h, e); - if (fr.entry_index != MAP_SENTINEL) { - map__erase(h, fr); - } -} +// template +// gb_internal void multi_map_remove(PtrMap *h, K key, PtrMapEntry *e) { +// if (fr.entry_index != MAP_SENTINEL) { +// map__erase(h, fr); +// } +// } template gb_internal void multi_map_remove_all(PtrMap *h, K key) { @@ -443,22 +359,77 @@ gb_internal void multi_map_remove_all(PtrMap *h, K key) { #endif -template -gb_internal PtrMapEntry *begin(PtrMap &m) { - return m.entries; -} -template -gb_internal PtrMapEntry const *begin(PtrMap const &m) { - return m.entries; -} template -gb_internal PtrMapEntry *end(PtrMap &m) { - return m.entries + m.count; +struct PtrMapIterator { + PtrMap *map; + MapIndex index; + + PtrMapIterator &operator++() noexcept { + for (;;) { + ++index; + if (map->capacity == index) { + return *this; + } + PtrMapEntry *entry = map->entries+index; + if (entry->key && entry->key != cast(K)MAP_TOMBSTONE) { + return *this; + } + } + } + + bool operator==(PtrMapIterator const &other) const noexcept { + return this->map == other->map && this->index == other->index; + } + + operator PtrMapEntry *() const { + return map->entries+index; + } +}; + + +template +gb_internal PtrMapIterator end(PtrMap &m) noexcept { + return PtrMapIterator{&m, m.capacity}; } template -gb_internal PtrMapEntry const *end(PtrMap const &m) { - return m.entries + m.count; +gb_internal PtrMapIterator const end(PtrMap const &m) noexcept { + return PtrMapIterator{&m, m.capacity}; +} + + + +template +gb_internal PtrMapIterator begin(PtrMap &m) noexcept { + if (m.count == 0) { + return end(m); + } + + MapIndex index = 0; + while (index < m.capacity) { + auto key = m.entries[index].key; + if (key && key != cast(K)MAP_TOMBSTONE) { + break; + } + index++; + } + return PtrMapIterator{&m, index}; +} +template +gb_internal PtrMapIterator const begin(PtrMap const &m) noexcept { + if (m.count == 0) { + return end(m); + } + + MapIndex index = 0; + while (index < m.capacity) { + auto key = m.entries[index].key; + if (key && key != cast(K)MAP_TOMBSTONE) { + break; + } + index++; + } + return PtrMapIterator{&m, index}; } diff --git a/src/string.cpp b/src/string.cpp index 7bfa52f33..b92dd589e 100644 --- a/src/string.cpp +++ b/src/string.cpp @@ -171,6 +171,9 @@ template gb_internal bool operator > (String const &a, char const (&b template gb_internal bool operator <= (String const &a, char const (&b)[N]) { return str_le(a, make_string(cast(u8 *)b, N-1)); } template gb_internal bool operator >= (String const &a, char const (&b)[N]) { return str_ge(a, make_string(cast(u8 *)b, N-1)); } +template <> bool operator == (String const &a, char const (&b)[1]) { return a.len == 0; } +template <> bool operator != (String const &a, char const (&b)[1]) { return a.len != 0; } + gb_internal gb_inline bool string_starts_with(String const &s, String const &prefix) { if (prefix.len > s.len) { return false; @@ -234,11 +237,16 @@ gb_internal String string_split_iterator(String_Iterator *it, const char sep) { return substring(it->str, start, end); } +gb_internal gb_inline bool is_separator(u8 const &ch) { + return (ch == '/' || ch == '\\'); +} + + gb_internal gb_inline isize string_extension_position(String const &str) { isize dot_pos = -1; isize i = str.len; while (i --> 0) { - if (str[i] == '\\' || str[i] == '/') + if (is_separator(str[i])) break; if (str[i] == '.') { dot_pos = i; @@ -273,6 +281,43 @@ gb_internal String string_trim_whitespace(String str) { return str; } +gb_internal String string_trim_trailing_whitespace(String str) { + while (str.len > 0) { + u8 c = str[str.len-1]; + if (rune_is_whitespace(c) || c == 0) { + str.len -= 1; + } else { + break; + } + } + return str; +} + +gb_internal String split_lines_first_line_from_array(Array const &array, gbAllocator allocator) { + String_Iterator it = {{array.data, array.count}, 0}; + + String line = string_split_iterator(&it, '\n'); + line = string_trim_trailing_whitespace(line); + return line; +} + +gb_internal Array split_lines_from_array(Array const &array, gbAllocator allocator) { + Array lines = {}; + lines.allocator = allocator; + + String_Iterator it = {{array.data, array.count}, 0}; + + for (;;) { + String line = string_split_iterator(&it, '\n'); + if (line.len == 0) { + break; + } + line = string_trim_trailing_whitespace(line); + array_add(&lines, line); + } + + return lines; +} gb_internal bool string_contains_char(String const &s, u8 c) { isize i; @@ -292,8 +337,7 @@ gb_internal String filename_from_path(String s) { if (i > 0) { isize j = 0; for (j = s.len-1; j >= 0; j--) { - if (s[j] == '/' || - s[j] == '\\') { + if (is_separator(s[j])) { break; } } @@ -306,8 +350,7 @@ gb_internal String filename_from_path(String s) { gb_internal String filename_without_directory(String s) { isize j = 0; for (j = s.len-1; j >= 0; j--) { - if (s[j] == '/' || - s[j] == '\\') { + if (is_separator(s[j])) { break; } } @@ -370,7 +413,26 @@ gb_internal String copy_string(gbAllocator a, String const &s) { return make_string(data, s.len); } - +gb_internal String normalize_path(gbAllocator a, String const &path, String const &sep) { + String s; + if (sep.len < 1) { + return path; + } + if (path.len < 1) { + s = STR_LIT(""); + } else if (is_separator(path[path.len-1])) { + s = copy_string(a, path); + } else { + s = concatenate_strings(a, path, sep); + } + isize i; + for (i = 0; i < s.len; i++) { + if (is_separator(s.text[i])) { + s.text[i] = sep.text[0]; + } + } + return s; +} #if defined(GB_SYSTEM_WINDOWS) diff --git a/src/string_map.cpp b/src/string_map.cpp index f8b86a950..4de88bbf9 100644 --- a/src/string_map.cpp +++ b/src/string_map.cpp @@ -2,8 +2,8 @@ GB_STATIC_ASSERT(sizeof(MapIndex) == sizeof(u32)); struct StringHashKey { - u32 hash; String string; + u32 hash; operator String() const noexcept { return this->string; @@ -13,7 +13,8 @@ struct StringHashKey { } }; gb_internal gb_inline u32 string_hash(String const &s) { - return fnv32a(s.text, s.len) & 0x7fffffff; + u32 res = fnv32a(s.text, s.len) & 0x7fffffff; + return res | (res == 0); } gb_internal gb_inline StringHashKey string_hash_string(String const &s) { @@ -25,19 +26,16 @@ gb_internal gb_inline StringHashKey string_hash_string(String const &s) { template struct StringMapEntry { - String key; - u32 hash; - MapIndex next; - T value; + String key; + u32 hash; + T value; }; template struct StringMap { - MapIndex * hashes; - usize hashes_count; StringMapEntry *entries; u32 count; - u32 entries_capacity; + u32 capacity; }; @@ -73,127 +71,91 @@ gb_internal gb_inline void string_map_init(StringMap *h, usize capacity) { template gb_internal gb_inline void string_map_destroy(StringMap *h) { - gb_free(string_map_allocator(), h->hashes); gb_free(string_map_allocator(), h->entries); } template -gb_internal void string_map__resize_hashes(StringMap *h, usize count) { - h->hashes_count = cast(u32)resize_array_raw(&h->hashes, string_map_allocator(), h->hashes_count, count, MAP_CACHE_LINE_SIZE); -} - - -template -gb_internal void string_map__reserve_entries(StringMap *h, usize capacity) { - h->entries_capacity = cast(u32)resize_array_raw(&h->entries, string_map_allocator(), h->entries_capacity, capacity, MAP_CACHE_LINE_SIZE); -} - - -template -gb_internal MapIndex string_map__add_entry(StringMap *h, u32 hash, String const &key) { - StringMapEntry e = {}; - e.key = key; - e.hash = hash; - e.next = MAP_SENTINEL; - if (h->count+1 >= h->entries_capacity) { - string_map__reserve_entries(h, gb_max(h->entries_capacity*2, 4)); +gb_internal void string_map__insert(StringMap *h, u32 hash, String const &key, T const &value) { + if (h->count+1 >= h->capacity) { + string_map_grow(h); } - h->entries[h->count++] = e; - return cast(MapIndex)(h->count-1); -} + GB_ASSERT(h->count+1 < h->capacity); -template -gb_internal MapFindResult string_map__find(StringMap *h, u32 hash, String const &key) { - MapFindResult fr = {MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL}; - if (h->hashes_count != 0) { - fr.hash_index = cast(MapIndex)(hash & (h->hashes_count-1)); - fr.entry_index = h->hashes[fr.hash_index]; - while (fr.entry_index != MAP_SENTINEL) { - auto *entry = &h->entries[fr.entry_index]; - if (entry->hash == hash && entry->key == key) { - return fr; - } - fr.entry_prev = fr.entry_index; - fr.entry_index = entry->next; + u32 mask = h->capacity-1; + MapIndex index = hash & mask; + MapIndex original_index = index; + do { + auto *entry = h->entries+index; + if (entry->hash == 0) { + entry->key = key; + entry->hash = hash; + entry->value = value; + + h->count += 1; + return; } - } - return fr; -} + index = (index+1)&mask; + } while (index != original_index); -template -gb_internal MapFindResult string_map__find_from_entry(StringMap *h, StringMapEntry *e) { - MapFindResult fr = {MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL}; - if (h->hashes_count != 0) { - fr.hash_index = cast(MapIndex)(e->hash & (h->hashes_count-1)); - fr.entry_index = h->hashes[fr.hash_index]; - while (fr.entry_index != MAP_SENTINEL) { - auto *entry = &h->entries[fr.entry_index]; - if (entry == e) { - return fr; - } - fr.entry_prev = fr.entry_index; - fr.entry_index = entry->next; - } - } - return fr; + GB_PANIC("Full map"); } template gb_internal b32 string_map__full(StringMap *h) { - return 0.75f * h->hashes_count <= h->count; + return 0.75f * h->count <= h->capacity; } template gb_inline void string_map_grow(StringMap *h) { - isize new_count = gb_max(h->hashes_count<<1, 16); - string_map_reserve(h, new_count); + isize new_capacity = gb_max(h->capacity<<1, 16); + string_map_reserve(h, new_capacity); } -template -gb_internal void string_map_reset_entries(StringMap *h) { - for (u32 i = 0; i < h->hashes_count; i++) { - h->hashes[i] = MAP_SENTINEL; - } - for (isize i = 0; i < h->count; i++) { - MapFindResult fr; - StringMapEntry *e = &h->entries[i]; - e->next = MAP_SENTINEL; - fr = string_map__find_from_entry(h, e); - if (fr.entry_prev == MAP_SENTINEL) { - h->hashes[fr.hash_index] = cast(MapIndex)i; - } else { - h->entries[fr.entry_prev].next = cast(MapIndex)i; - } - } -} - template gb_internal void string_map_reserve(StringMap *h, usize cap) { - if (h->count*2 < h->hashes_count) { + if (cap < h->capacity) { return; } - string_map__reserve_entries(h, cap); - string_map__resize_hashes(h, cap*2); - string_map_reset_entries(h); + cap = next_pow2_isize(cap); + + StringMap new_h = {}; + new_h.count = 0; + new_h.capacity = cast(u32)cap; + new_h.entries = gb_alloc_array(string_map_allocator(), StringMapEntry, new_h.capacity); + + if (h->count) { + for (u32 i = 0; i < h->capacity; i++) { + auto *entry = h->entries+i; + if (entry->hash) { + string_map__insert(&new_h, entry->hash, entry->key, entry->value); + } + } + } + string_map_destroy(h); + *h = new_h; } template gb_internal T *string_map_get(StringMap *h, u32 hash, String const &key) { - MapFindResult fr = {MAP_SENTINEL, MAP_SENTINEL, MAP_SENTINEL}; - if (h->hashes_count != 0) { - fr.hash_index = cast(MapIndex)(hash & (h->hashes_count-1)); - fr.entry_index = h->hashes[fr.hash_index]; - while (fr.entry_index != MAP_SENTINEL) { - auto *entry = &h->entries[fr.entry_index]; - if (entry->hash == hash && entry->key == key) { - return &entry->value; - } - fr.entry_prev = fr.entry_index; - fr.entry_index = entry->next; - } + if (h->count == 0) { + return nullptr; } + u32 mask = (h->capacity-1); + u32 index = hash & mask; + u32 original_index = index; + do { + auto *entry = h->entries+index; + u32 curr_hash = entry->hash; + if (curr_hash == 0) { + // NOTE(bill): no found, but there isn't any key removal for this hash map + return nullptr; + } else if (curr_hash == hash && entry->key == key) { + return &entry->value; + } + index = (index+1) & mask; + } while (original_index != index); return nullptr; } @@ -216,9 +178,9 @@ gb_internal gb_inline T *string_map_get(StringMap *h, char const *key) { template gb_internal T &string_map_must_get(StringMap *h, u32 hash, String const &key) { - isize index = string_map__find(h, hash, key).entry_index; - GB_ASSERT(index != MAP_SENTINEL); - return h->entries[index].value; + T *found = string_map_get(h, hash, key); + GB_ASSERT(found != nullptr); + return *found; } template @@ -239,27 +201,15 @@ gb_internal gb_inline T &string_map_must_get(StringMap *h, char const *key) { template gb_internal void string_map_set(StringMap *h, u32 hash, String const &key, T const &value) { - MapIndex index; - MapFindResult fr; - if (h->hashes_count == 0) { + if (h->count == 0) { string_map_grow(h); } - fr = string_map__find(h, hash, key); - if (fr.entry_index != MAP_SENTINEL) { - index = fr.entry_index; - } else { - index = string_map__add_entry(h, hash, key); - if (fr.entry_prev != MAP_SENTINEL) { - h->entries[fr.entry_prev].next = index; - } else { - h->hashes[fr.hash_index] = index; - } - } - h->entries[index].value = value; - - if (string_map__full(h)) { - string_map_grow(h); + auto *found = string_map_get(h, hash, key); + if (found) { + *found = value; + return; } + string_map__insert(h, hash, key, value); } template @@ -278,62 +228,80 @@ gb_internal gb_inline void string_map_set(StringMap *h, StringHashKey const & } - -// template -// gb_internal void string_map__erase(StringMap *h, MapFindResult const &fr) { -// MapFindResult last; -// if (fr.entry_prev == MAP_SENTINEL) { -// h->hashes[fr.hash_index] = h->entries[fr.entry_index].next; -// } else { -// h->entries[fr.entry_prev].next = h->entries[fr.entry_index].next; -// } -// if (fr.entry_index == h->count-1) { -// array_pop(&h->entries); -// return; -// } -// h->entries[fr.entry_index] = h->entries[h->count-1]; -// last = string_map__find(h, h->entries[fr.entry_index].key); -// if (last.entry_prev != MAP_SENTINEL) { -// h->entries[last.entry_prev].next = fr.entry_index; -// } else { -// h->hashes[last.hash_index] = fr.entry_index; -// } -// } - -// template -// gb_internal void string_map_remove(StringMap *h, StringHashKey const &key) { -// MapFindResult fr = string_map__find(h, key); -// if (fr.entry_index != MAP_SENTINEL) { -// string_map__erase(h, fr); -// } -// } - template gb_internal gb_inline void string_map_clear(StringMap *h) { h->count = 0; - for (u32 i = 0; i < h->hashes_count; i++) { - h->hashes[i] = MAP_SENTINEL; + gb_zero_array(h->entries, h->capacity); +} + + +template +struct StringMapIterator { + StringMap *map; + MapIndex index; + + StringMapIterator &operator++() noexcept { + for (;;) { + ++index; + if (map->capacity == index) { + return *this; + } + StringMapEntry *entry = map->entries+index; + if (entry->hash != 0) { + return *this; + } + } } + + bool operator==(StringMapIterator const &other) const noexcept { + return this->map == other->map && this->index == other->index; + } + + operator StringMapEntry *() const { + return map->entries+index; + } +}; + + +template +gb_internal StringMapIterator end(StringMap &m) noexcept { + return StringMapIterator{&m, m.capacity}; +} + +template +gb_internal StringMapIterator const end(StringMap const &m) noexcept { + return StringMapIterator{&m, m.capacity}; } template -gb_internal StringMapEntry *begin(StringMap &m) noexcept { - return m.entries; +gb_internal StringMapIterator begin(StringMap &m) noexcept { + if (m.count == 0) { + return end(m); + } + + MapIndex index = 0; + while (index < m.capacity) { + if (m.entries[index].hash) { + break; + } + index++; + } + return StringMapIterator{&m, index}; } template -gb_internal StringMapEntry const *begin(StringMap const &m) noexcept { - return m.entries; +gb_internal StringMapIterator const begin(StringMap const &m) noexcept { + if (m.count == 0) { + return end(m); + } + + MapIndex index = 0; + while (index < m.capacity) { + if (m.entries[index].hash) { + break; + } + index++; + } + return StringMapIterator{&m, index}; } - - -template -gb_internal StringMapEntry *end(StringMap &m) { - return m.entries + m.count; -} - -template -gb_internal StringMapEntry const *end(StringMap const &m) noexcept { - return m.entries + m.count; -} \ No newline at end of file diff --git a/src/string_set.cpp b/src/string_set.cpp index fb4640c20..a37d8ba80 100644 --- a/src/string_set.cpp +++ b/src/string_set.cpp @@ -208,7 +208,9 @@ gb_internal void string_set__erase(StringSet *s, MapFindResult fr) { } auto *entry = &s->entries[fr.entry_index]; *entry = s->entries[s->entries.count-1]; - StringHashKey key = {entry->hash, entry->value}; + StringHashKey key; + key.hash = entry->hash; + key.string = entry->value; last = string_set__find(s, key); if (last.entry_prev != MAP_SENTINEL) { s->entries[last.entry_prev].next = fr.entry_index; diff --git a/src/threading.cpp b/src/threading.cpp index 79ed8e8a4..b2cfa6d8e 100644 --- a/src/threading.cpp +++ b/src/threading.cpp @@ -634,9 +634,15 @@ gb_internal void thread_set_name(Thread *t, char const *name) { #endif } -#if defined(GB_SYSTEM_LINUX) -#include +#if defined(GB_SYSTEM_LINUX) || defined(GB_SYSTEM_NETBSD) + #include +#ifdef GB_SYSTEM_LINUX + #include +#else + #include + #define SYS_futex SYS___futex +#endif gb_internal void futex_signal(Futex *addr) { int ret = syscall(SYS_futex, addr, FUTEX_WAKE | FUTEX_PRIVATE_FLAG, 1, NULL, NULL, 0); @@ -903,11 +909,10 @@ gb_internal void futex_wait(Futex *f, Footex val) { } while (f->load() == val); } -#elif defined(GB_SYSTEM_HAIKU) || defined(GB_SYSTEM_NETBSD) +#elif defined(GB_SYSTEM_HAIKU) // Futex implementation taken from https://tavianator.com/2023/futex.html -#include #include #include diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 3d5348074..f7751d840 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -193,6 +193,7 @@ gb_internal void init_keyword_hash_table(void) { gb_global Array global_file_path_strings; // index is file id gb_global Array global_files; // index is file id +gb_global BlockingMutex global_files_mutex; gb_internal String get_file_path_string(i32 index); gb_internal struct AstFile *thread_safe_get_ast_file_from_id(i32 index); @@ -766,9 +767,8 @@ gb_internal void tokenizer_get_token(Tokenizer *t, Token *token, int repeat=0) { } } - // TODO(bill): Better Error Handling if (valid && n != 1) { - tokenizer_err(t, "Invalid rune literal"); + tokenizer_err(t, token->pos, "Invalid rune literal"); } token->string.len = t->curr - token->string.text; goto semicolon_check; diff --git a/src/types.cpp b/src/types.cpp index 97512d29b..18cb12ea1 100644 --- a/src/types.cpp +++ b/src/types.cpp @@ -769,7 +769,6 @@ gb_internal gbString type_to_string (Type *type, bool shorthand=true); gb_internal gbString type_to_string (Type *type, gbAllocator allocator, bool shorthand=true); gb_internal i64 type_size_of_internal(Type *t, TypePath *path); gb_internal i64 type_align_of_internal(Type *t, TypePath *path); -gb_internal void init_map_internal_types(Type *type); gb_internal Type * bit_set_to_int(Type *t); gb_internal bool are_types_identical(Type *x, Type *y); diff --git a/tests/core/Makefile b/tests/core/Makefile index d702c4e1d..2d5b74bad 100644 --- a/tests/core/Makefile +++ b/tests/core/Makefile @@ -58,6 +58,9 @@ encoding_test: $(ODIN) run encoding/json $(COMMON) -out:test_json $(ODIN) run encoding/varint $(COMMON) -out:test_varint $(ODIN) run encoding/xml $(COMMON) -out:test_xml + $(ODIN) run encoding/cbor $(COMMON) -out:test_cbor + $(ODIN) run encoding/hex $(COMMON) -out:test_hex + $(ODIN) run encoding/base64 $(COMMON) -out:test_base64 math_test: $(ODIN) run math $(COMMON) $(COLLECTION) -out:test_core_math diff --git a/tests/core/build.bat b/tests/core/build.bat index ac7137fa1..36d2bb9c4 100644 --- a/tests/core/build.bat +++ b/tests/core/build.bat @@ -40,6 +40,9 @@ rem %PATH_TO_ODIN% run encoding/hxa %COMMON% %COLLECTION% -out:test_hxa.exe | %PATH_TO_ODIN% run encoding/json %COMMON% -out:test_json.exe || exit /b %PATH_TO_ODIN% run encoding/varint %COMMON% -out:test_varint.exe || exit /b %PATH_TO_ODIN% run encoding/xml %COMMON% -out:test_xml.exe || exit /b +%PATH_TO_ODIN% test encoding/cbor %COMMON% -out:test_cbor.exe || exit /b +%PATH_TO_ODIN% run encoding/hex %COMMON% -out:test_hex.exe || exit /b +%PATH_TO_ODIN% run encoding/base64 %COMMON% -out:test_base64.exe || exit /b echo --- echo Running core:math/noise tests diff --git a/tests/core/crypto/test_core_crypto.odin b/tests/core/crypto/test_core_crypto.odin index 72d8e7c78..95db3f292 100644 --- a/tests/core/crypto/test_core_crypto.odin +++ b/tests/core/crypto/test_core_crypto.odin @@ -277,7 +277,7 @@ test_chacha20poly1305 :: proc(t: ^testing.T) { test_rand_bytes :: proc(t: ^testing.T) { tc.log(t, "Testing rand_bytes") - if !crypto.has_rand_bytes() { + if !crypto.HAS_RAND_BYTES { tc.log(t, "rand_bytes not supported - skipping") return } diff --git a/tests/core/encoding/base64/base64.odin b/tests/core/encoding/base64/base64.odin new file mode 100644 index 000000000..e48eea020 --- /dev/null +++ b/tests/core/encoding/base64/base64.odin @@ -0,0 +1,61 @@ +package test_encoding_base64 + +import "base:intrinsics" + +import "core:encoding/base64" +import "core:fmt" +import "core:os" +import "core:reflect" +import "core:testing" + +TEST_count := 0 +TEST_fail := 0 + +when ODIN_TEST { + expect_value :: testing.expect_value + +} else { + expect_value :: proc(t: ^testing.T, value, expected: $T, loc := #caller_location) -> bool where intrinsics.type_is_comparable(T) { + TEST_count += 1 + ok := value == expected || reflect.is_nil(value) && reflect.is_nil(expected) + if !ok { + TEST_fail += 1 + fmt.printf("[%v] expected %v, got %v\n", loc, expected, value) + } + return ok + } +} + +main :: proc() { + t := testing.T{} + + test_encoding(&t) + test_decoding(&t) + + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + if TEST_fail > 0 { + os.exit(1) + } +} + +@(test) +test_encoding :: proc(t: ^testing.T) { + expect_value(t, base64.encode(transmute([]byte)string("")), "") + expect_value(t, base64.encode(transmute([]byte)string("f")), "Zg==") + expect_value(t, base64.encode(transmute([]byte)string("fo")), "Zm8=") + expect_value(t, base64.encode(transmute([]byte)string("foo")), "Zm9v") + expect_value(t, base64.encode(transmute([]byte)string("foob")), "Zm9vYg==") + expect_value(t, base64.encode(transmute([]byte)string("fooba")), "Zm9vYmE=") + expect_value(t, base64.encode(transmute([]byte)string("foobar")), "Zm9vYmFy") +} + +@(test) +test_decoding :: proc(t: ^testing.T) { + expect_value(t, string(base64.decode("")), "") + expect_value(t, string(base64.decode("Zg==")), "f") + expect_value(t, string(base64.decode("Zm8=")), "fo") + expect_value(t, string(base64.decode("Zm9v")), "foo") + expect_value(t, string(base64.decode("Zm9vYg==")), "foob") + expect_value(t, string(base64.decode("Zm9vYmE=")), "fooba") + expect_value(t, string(base64.decode("Zm9vYmFy")), "foobar") +} diff --git a/tests/core/encoding/cbor/test_core_cbor.odin b/tests/core/encoding/cbor/test_core_cbor.odin new file mode 100644 index 000000000..72244e1d3 --- /dev/null +++ b/tests/core/encoding/cbor/test_core_cbor.odin @@ -0,0 +1,905 @@ +package test_encoding_cbor + +import "base:intrinsics" + +import "core:bytes" +import "core:encoding/cbor" +import "core:fmt" +import "core:io" +import "core:math/big" +import "core:mem" +import "core:os" +import "core:reflect" +import "core:testing" +import "core:time" + +TEST_count := 0 +TEST_fail := 0 + +when ODIN_TEST { + expect :: testing.expect + expect_value :: testing.expect_value + errorf :: testing.errorf + log :: testing.log + +} else { + expect :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) { + TEST_count += 1 + if !condition { + TEST_fail += 1 + fmt.printf("[%v] %v\n", loc, message) + return + } + } + + expect_value :: proc(t: ^testing.T, value, expected: $T, loc := #caller_location) -> bool where intrinsics.type_is_comparable(T) { + TEST_count += 1 + ok := value == expected || reflect.is_nil(value) && reflect.is_nil(expected) + if !ok { + TEST_fail += 1 + fmt.printf("[%v] expected %v, got %v\n", loc, expected, value) + } + return ok + } + + errorf :: proc(t: ^testing.T, fmts: string, args: ..any, loc := #caller_location) { + TEST_fail += 1 + fmt.printf("[%v] ERROR: ", loc) + fmt.printf(fmts, ..args) + fmt.println() + } + + log :: proc(t: ^testing.T, v: any, loc := #caller_location) { + fmt.printf("[%v] ", loc) + fmt.printf("log: %v\n", v) + } +} + +main :: proc() { + t := testing.T{} + + test_marshalling(&t) + + test_marshalling_maybe(&t) + test_marshalling_nil_maybe(&t) + + test_marshalling_union(&t) + + test_lying_length_array(&t) + + test_decode_unsigned(&t) + test_encode_unsigned(&t) + + test_decode_negative(&t) + test_encode_negative(&t) + + test_decode_simples(&t) + test_encode_simples(&t) + + test_decode_floats(&t) + test_encode_floats(&t) + + test_decode_bytes(&t) + test_encode_bytes(&t) + + test_decode_strings(&t) + test_encode_strings(&t) + + test_decode_lists(&t) + test_encode_lists(&t) + + test_decode_maps(&t) + test_encode_maps(&t) + + test_decode_tags(&t) + test_encode_tags(&t) + + fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count) + if TEST_fail > 0 { + os.exit(1) + } +} + +Foo :: struct { + str: string, + cstr: cstring, + value: cbor.Value, + neg: cbor.Negative_U16, + pos: u16, + iamint: int, + base64: string `cbor_tag:"base64"`, + renamed: f32 `cbor:"renamed :)"`, + now: time.Time `cbor_tag:"1"`, + nowie: time.Time, + child: struct{ + dyn: [dynamic]string, + mappy: map[string]int, + my_integers: [10]int, + }, + my_bytes: []byte, + ennie: FooBar, + ennieb: FooBars, + quat: quaternion64, + comp: complex128, + important: rune, + no: cbor.Nil, + nos: cbor.Undefined, + yes: b32, + biggie: u64, + smallie: cbor.Negative_U64, + onetwenty: i128, + small_onetwenty: i128, + biggest: big.Int, + smallest: big.Int, + ignore_this: ^Foo `cbor:"-"`, +} + +FooBar :: enum { + EFoo, + EBar, +} + +FooBars :: bit_set[FooBar; u16] + +@(test) +test_marshalling :: proc(t: ^testing.T) { + tracker: mem.Tracking_Allocator + mem.tracking_allocator_init(&tracker, context.allocator) + context.allocator = mem.tracking_allocator(&tracker) + context.temp_allocator = context.allocator + defer mem.tracking_allocator_destroy(&tracker) + + ev :: expect_value + + { + nice := "16 is a nice number" + now := time.Time{_nsec = 1701117968 * 1e9} + f: Foo = { + str = "Hellope", + cstr = "Hellnope", + value = &cbor.Map{{u8(16), &nice}, {u8(32), u8(69)}}, + neg = 68, + pos = 1212, + iamint = -256, + base64 = nice, + renamed = 123123.125, + + now = now, + nowie = now, + + child = { + dyn = [dynamic]string{"one", "two", "three", "four"}, + mappy = map[string]int{"one" = 1, "two" = 2, "three" = 3, "four" = 4}, + my_integers = [10]int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, + }, + + my_bytes = []byte{}, + + ennie = .EFoo, + ennieb = {.EBar}, + + quat = quaternion(w=16, x=17, y=18, z=19), + comp = complex(32, 33), + + important = '!', + + no = cbor.Nil(uintptr(3)), + + yes = true, + + biggie = max(u64), + smallie = cbor.Negative_U64(max(u64)), + onetwenty = i128(12345), + small_onetwenty = -i128(max(u64)), + ignore_this = &Foo{}, + } + + big.atoi(&f.biggest, "1234567891011121314151617181920") + big.atoi(&f.smallest, "-1234567891011121314151617181920") + + defer { + delete(f.child.dyn) + delete(f.child.mappy) + big.destroy(&f.biggest) + big.destroy(&f.smallest) + } + + data, err := cbor.marshal(f, cbor.ENCODE_FULLY_DETERMINISTIC) + ev(t, err, nil) + defer delete(data) + + decoded, derr := cbor.decode(string(data)) + ev(t, derr, nil) + defer cbor.destroy(decoded) + + diagnosis, eerr := cbor.to_diagnostic_format(decoded) + ev(t, eerr, nil) + defer delete(diagnosis) + + ev(t, diagnosis, `{ + "base64": 34("MTYgaXMgYSBuaWNlIG51bWJlcg=="), + "biggest": 2(h'f951a9fd3c158afdff08ab8e0'), + "biggie": 18446744073709551615, + "child": { + "dyn": [ + "one", + "two", + "three", + "four" + ], + "mappy": { + "one": 1, + "two": 2, + "four": 4, + "three": 3 + }, + "my_integers": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10 + ] + }, + "comp": [ + 32.0000, + 33.0000 + ], + "cstr": "Hellnope", + "ennie": 0, + "ennieb": 512, + "iamint": -256, + "important": "!", + "my_bytes": h'', + "neg": -69, + "no": nil, + "nos": undefined, + "now": 1(1701117968), + "nowie": { + "_nsec": 1701117968000000000 + }, + "onetwenty": 12345, + "pos": 1212, + "quat": [ + 17.0000, + 18.0000, + 19.0000, + 16.0000 + ], + "renamed :)": 123123.12500000, + "small_onetwenty": -18446744073709551615, + "smallest": 3(h'f951a9fd3c158afdff08ab8e0'), + "smallie": -18446744073709551616, + "str": "Hellope", + "value": { + 16: "16 is a nice number", + 32: 69 + }, + "yes": true +}`) + + backf: Foo + uerr := cbor.unmarshal(string(data), &backf) + ev(t, uerr, nil) + defer { + delete(backf.str) + delete(backf.cstr) + cbor.destroy(backf.value) + delete(backf.base64) + + for e in backf.child.dyn { delete(e) } + delete(backf.child.dyn) + + for k in backf.child.mappy { delete(k) } + delete(backf.child.mappy) + + delete(backf.my_bytes) + + big.destroy(&backf.biggest) + big.destroy(&backf.smallest) + } + + ev(t, backf.str, f.str) + ev(t, backf.cstr, f.cstr) + + #partial switch v in backf.value { + case ^cbor.Map: + for entry, i in v { + fm := f.value.(^cbor.Map) + ev(t, entry.key, fm[i].key) + + if str, is_str := entry.value.(^cbor.Text); is_str { + ev(t, str^, fm[i].value.(^cbor.Text)^) + } else { + ev(t, entry.value, fm[i].value) + } + } + + case: errorf(t, "wrong type %v", v) + } + + ev(t, backf.neg, f.neg) + ev(t, backf.iamint, f.iamint) + ev(t, backf.base64, f.base64) + ev(t, backf.renamed, f.renamed) + ev(t, backf.now, f.now) + ev(t, backf.nowie, f.nowie) + for e, i in f.child.dyn { ev(t, backf.child.dyn[i], e) } + for key, value in f.child.mappy { ev(t, backf.child.mappy[key], value) } + ev(t, backf.child.my_integers, f.child.my_integers) + ev(t, len(backf.my_bytes), 0) + ev(t, len(backf.my_bytes), len(f.my_bytes)) + ev(t, backf.ennie, f.ennie) + ev(t, backf.ennieb, f.ennieb) + ev(t, backf.quat, f.quat) + ev(t, backf.comp, f.comp) + ev(t, backf.important, f.important) + ev(t, backf.no, nil) + ev(t, backf.nos, nil) + ev(t, backf.yes, f.yes) + ev(t, backf.biggie, f.biggie) + ev(t, backf.smallie, f.smallie) + ev(t, backf.onetwenty, f.onetwenty) + ev(t, backf.small_onetwenty, f.small_onetwenty) + ev(t, backf.ignore_this, nil) + + s_equals, s_err := big.equals(&backf.smallest, &f.smallest) + ev(t, s_err, nil) + if !s_equals { + errorf(t, "smallest: %v does not equal %v", big.itoa(&backf.smallest), big.itoa(&f.smallest)) + } + + b_equals, b_err := big.equals(&backf.biggest, &f.biggest) + ev(t, b_err, nil) + if !b_equals { + errorf(t, "biggest: %v does not equal %v", big.itoa(&backf.biggest), big.itoa(&f.biggest)) + } + } + + for _, leak in tracker.allocation_map { + errorf(t, "%v leaked %m\n", leak.location, leak.size) + } + + for bad_free in tracker.bad_free_array { + errorf(t, "%v allocation %p was freed badly\n", bad_free.location, bad_free.memory) + } +} + +@(test) +test_marshalling_maybe :: proc(t: ^testing.T) { + maybe_test: Maybe(int) = 1 + data, err := cbor.marshal(maybe_test) + expect_value(t, err, nil) + + val, derr := cbor.decode(string(data)) + expect_value(t, derr, nil) + + expect_value(t, cbor.to_diagnostic_format(val), "1") + + maybe_dest: Maybe(int) + uerr := cbor.unmarshal(string(data), &maybe_dest) + expect_value(t, uerr, nil) + expect_value(t, maybe_dest, 1) +} + +@(test) +test_marshalling_nil_maybe :: proc(t: ^testing.T) { + maybe_test: Maybe(int) + data, err := cbor.marshal(maybe_test) + expect_value(t, err, nil) + + val, derr := cbor.decode(string(data)) + expect_value(t, derr, nil) + + expect_value(t, cbor.to_diagnostic_format(val), "nil") + + maybe_dest: Maybe(int) + uerr := cbor.unmarshal(string(data), &maybe_dest) + expect_value(t, uerr, nil) + expect_value(t, maybe_dest, nil) +} + +@(test) +test_marshalling_union :: proc(t: ^testing.T) { + My_Distinct :: distinct string + + My_Enum :: enum { + One, + Two, + } + + My_Struct :: struct { + my_enum: My_Enum, + } + + My_Union :: union { + string, + My_Distinct, + My_Struct, + int, + } + + { + test: My_Union = My_Distinct("Hello, World!") + data, err := cbor.marshal(test) + expect_value(t, err, nil) + + val, derr := cbor.decode(string(data)) + expect_value(t, derr, nil) + + expect_value(t, cbor.to_diagnostic_format(val, -1), `1010(["My_Distinct", "Hello, World!"])`) + + dest: My_Union + uerr := cbor.unmarshal(string(data), &dest) + expect_value(t, uerr, nil) + expect_value(t, dest, My_Distinct("Hello, World!")) + } + + My_Union_No_Nil :: union #no_nil { + string, + My_Distinct, + My_Struct, + int, + } + + { + test: My_Union_No_Nil = My_Struct{.Two} + data, err := cbor.marshal(test) + expect_value(t, err, nil) + + val, derr := cbor.decode(string(data)) + expect_value(t, derr, nil) + + expect_value(t, cbor.to_diagnostic_format(val, -1), `1010(["My_Struct", {"my_enum": 1}])`) + + dest: My_Union_No_Nil + uerr := cbor.unmarshal(string(data), &dest) + expect_value(t, uerr, nil) + expect_value(t, dest, My_Struct{.Two}) + } +} + +@(test) +test_lying_length_array :: proc(t: ^testing.T) { + // Input says this is an array of length max(u64), this should not allocate that amount. + input := []byte{0x9B, 0x00, 0x00, 0x42, 0xFA, 0x42, 0xFA, 0x42, 0xFA, 0x42} + _, err := cbor.decode(string(input)) + expect_value(t, err, io.Error.Unexpected_EOF) // .Out_Of_Memory would be bad. +} + +@(test) +test_decode_unsigned :: proc(t: ^testing.T) { + expect_decoding(t, "\x00", "0", u8) + expect_decoding(t, "\x01", "1", u8) + expect_decoding(t, "\x0a", "10", u8) + expect_decoding(t, "\x17", "23", u8) + expect_decoding(t, "\x18\x18", "24", u8) + expect_decoding(t, "\x18\x19", "25", u8) + expect_decoding(t, "\x18\x64", "100", u8) + expect_decoding(t, "\x19\x03\xe8", "1000", u16) + expect_decoding(t, "\x1a\x00\x0f\x42\x40", "1000000", u32) // Million. + expect_decoding(t, "\x1b\x00\x00\x00\xe8\xd4\xa5\x10\x00", "1000000000000", u64) // Trillion. + expect_decoding(t, "\x1b\xff\xff\xff\xff\xff\xff\xff\xff", "18446744073709551615", u64) // max(u64). +} + +@(test) +test_encode_unsigned :: proc(t: ^testing.T) { + expect_encoding(t, u8(0), "\x00") + expect_encoding(t, u8(1), "\x01") + expect_encoding(t, u8(10), "\x0a") + expect_encoding(t, u8(23), "\x17") + expect_encoding(t, u8(24), "\x18\x18") + expect_encoding(t, u8(25), "\x18\x19") + expect_encoding(t, u8(100), "\x18\x64") + expect_encoding(t, u16(1000), "\x19\x03\xe8") + expect_encoding(t, u32(1000000), "\x1a\x00\x0f\x42\x40") // Million. + expect_encoding(t, u64(1000000000000), "\x1b\x00\x00\x00\xe8\xd4\xa5\x10\x00") // Trillion. + expect_encoding(t, u64(18446744073709551615), "\x1b\xff\xff\xff\xff\xff\xff\xff\xff") // max(u64). +} + +@(test) +test_decode_negative :: proc(t: ^testing.T) { + expect_decoding(t, "\x20", "-1", cbor.Negative_U8) + expect_decoding(t, "\x29", "-10", cbor.Negative_U8) + expect_decoding(t, "\x38\x63", "-100", cbor.Negative_U8) + expect_decoding(t, "\x39\x03\xe7", "-1000", cbor.Negative_U16) + + // Negative max(u64). + expect_decoding(t, "\x3b\xff\xff\xff\xff\xff\xff\xff\xff", "-18446744073709551616", cbor.Negative_U64) +} + +@(test) +test_encode_negative :: proc(t: ^testing.T) { + expect_encoding(t, cbor.Negative_U8(0), "\x20") + expect_encoding(t, cbor.Negative_U8(9), "\x29") + expect_encoding(t, cbor.Negative_U8(99), "\x38\x63") + expect_encoding(t, cbor.Negative_U16(999), "\x39\x03\xe7") + + // Negative max(u64). + expect_encoding(t, cbor.Negative_U64(18446744073709551615), "\x3b\xff\xff\xff\xff\xff\xff\xff\xff") +} + +@(test) +test_decode_simples :: proc(t: ^testing.T) { + expect_decoding(t, "\xf4", "false", bool) + expect_decoding(t, "\xf5", "true", bool) + expect_decoding(t, "\xf6", "nil", cbor.Nil) + expect_decoding(t, "\xf7", "undefined", cbor.Undefined) + + expect_decoding(t, "\xf0", "simple(16)", cbor.Simple) + expect_decoding(t, "\xf8\xff", "simple(255)", cbor.Atom) +} + +@(test) +test_encode_simples :: proc(t: ^testing.T) { + expect_encoding(t, bool(false), "\xf4") + expect_encoding(t, bool(true), "\xf5") + expect_encoding(t, cbor.Nil{}, "\xf6") // default value for a distinct rawptr, in this case Nil. + expect_encoding(t, cbor.Undefined{}, "\xf7") // default value for a distinct rawptr, in this case Undefined. + + expect_encoding(t, cbor.Simple(16), "\xf0") // simple(16) + expect_encoding(t, cbor.Simple(255), "\xf8\xff") // simple(255) +} + +@(test) +test_decode_floats :: proc(t: ^testing.T) { + expect_float(t, "\xf9\x00\x00", f16(0.0)) + expect_float(t, "\xf9\x80\x00", f16(-0.0)) + expect_float(t, "\xf9\x3c\x00", f16(1.0)) + expect_float(t, "\xfb\x3f\xf1\x99\x99\x99\x99\x99\x9a", f64(1.1)) + expect_float(t, "\xf9\x3e\x00", f16(1.5)) + expect_float(t, "\xf9\x7b\xff", f16(65504.0)) + expect_float(t, "\xfa\x47\xc3\x50\x00", f32(100000.0)) + expect_float(t, "\xfa\x7f\x7f\xff\xff", f32(3.4028234663852886e+38)) + expect_float(t, "\xfb\x7e\x37\xe4\x3c\x88\x00\x75\x9c", f64(1.0e+300)) + expect_float(t, "\xf9\x00\x01", f16(5.960464477539063e-8)) + expect_float(t, "\xf9\x04\x00", f16(0.00006103515625)) + expect_float(t, "\xf9\xc4\x00", f16(-4.0)) + expect_float(t, "\xfb\xc0\x10\x66\x66\x66\x66\x66\x66", f64(-4.1)) + expect_decoding(t, "\xf9\x7c\x00", "+Inf", f16) + expect_decoding(t, "\xf9\x7e\x00", "NaN", f16) + expect_decoding(t, "\xf9\xfc\x00", "-Inf", f16) + expect_decoding(t, "\xfa\x7f\x80\x00\x00", "+Inf", f32) + expect_decoding(t, "\xfa\x7f\xc0\x00\x00", "NaN", f32) + expect_decoding(t, "\xfa\xff\x80\x00\x00", "-Inf", f32) + expect_decoding(t, "\xfb\x7f\xf0\x00\x00\x00\x00\x00\x00", "+Inf", f64) + expect_decoding(t, "\xfb\x7f\xf8\x00\x00\x00\x00\x00\x00", "NaN", f64) + expect_decoding(t, "\xfb\xff\xf0\x00\x00\x00\x00\x00\x00", "-Inf", f64) +} + +@(test) +test_encode_floats :: proc(t: ^testing.T) { + expect_encoding(t, f16(0.0), "\xf9\x00\x00") + expect_encoding(t, f16(-0.0), "\xf9\x80\x00") + expect_encoding(t, f16(1.0), "\xf9\x3c\x00") + expect_encoding(t, f64(1.1), "\xfb\x3f\xf1\x99\x99\x99\x99\x99\x9a") + expect_encoding(t, f16(1.5), "\xf9\x3e\x00") + expect_encoding(t, f16(65504.0), "\xf9\x7b\xff") + expect_encoding(t, f32(100000.0), "\xfa\x47\xc3\x50\x00") + expect_encoding(t, f32(3.4028234663852886e+38), "\xfa\x7f\x7f\xff\xff") + expect_encoding(t, f64(1.0e+300), "\xfb\x7e\x37\xe4\x3c\x88\x00\x75\x9c") + expect_encoding(t, f16(5.960464477539063e-8), "\xf9\x00\x01") + expect_encoding(t, f16(0.00006103515625), "\xf9\x04\x00") + expect_encoding(t, f16(-4.0), "\xf9\xc4\x00") + expect_encoding(t, f64(-4.1), "\xfb\xc0\x10\x66\x66\x66\x66\x66\x66") +} + +@(test) +test_decode_bytes :: proc(t: ^testing.T) { + expect_decoding(t, "\x40", "h''", ^cbor.Bytes) + expect_decoding(t, "\x44\x01\x02\x03\x04", "h'1234'", ^cbor.Bytes) + + // Indefinite lengths + + expect_decoding(t, "\x5f\x42\x01\x02\x43\x03\x04\x05\xff", "h'12345'", ^cbor.Bytes) +} + +@(test) +test_encode_bytes :: proc(t: ^testing.T) { + expect_encoding(t, &cbor.Bytes{}, "\x40") + expect_encoding(t, &cbor.Bytes{1, 2, 3, 4}, "\x44\x01\x02\x03\x04") + + // Indefinite lengths + + expect_streamed_encoding(t, "\x5f\x42\x01\x02\x43\x03\x04\x05\xff", &cbor.Bytes{1, 2}, &cbor.Bytes{3, 4, 5}) +} + +@(test) +test_decode_strings :: proc(t: ^testing.T) { + expect_decoding(t, "\x60", `""`, ^cbor.Text) + expect_decoding(t, "\x61\x61", `"a"`, ^cbor.Text) + expect_decoding(t, "\x64\x49\x45\x54\x46", `"IETF"`, ^cbor.Text) + expect_decoding(t, "\x62\x22\x5c", `""\"`, ^cbor.Text) + expect_decoding(t, "\x62\xc3\xbc", `"ü"`, ^cbor.Text) + expect_decoding(t, "\x63\xe6\xb0\xb4", `"水"`, ^cbor.Text) + expect_decoding(t, "\x64\xf0\x90\x85\x91", `"𐅑"`, ^cbor.Text) + + // Indefinite lengths + + expect_decoding(t, "\x7f\x65\x73\x74\x72\x65\x61\x64\x6d\x69\x6e\x67\xff", `"streaming"`, ^cbor.Text) +} + +@(test) +test_encode_strings :: proc(t: ^testing.T) { + expect_encoding(t, &cbor.Text{}, "\x60") + + a := "a" + expect_encoding(t, &a, "\x61\x61") + + b := "IETF" + expect_encoding(t, &b, "\x64\x49\x45\x54\x46") + + c := "\"\\" + expect_encoding(t, &c, "\x62\x22\x5c") + + d := "ü" + expect_encoding(t, &d, "\x62\xc3\xbc") + + e := "水" + expect_encoding(t, &e, "\x63\xe6\xb0\xb4") + + f := "𐅑" + expect_encoding(t, &f, "\x64\xf0\x90\x85\x91") + + // Indefinite lengths + + sa := "strea" + sb := "ming" + expect_streamed_encoding(t, "\x7f\x65\x73\x74\x72\x65\x61\x64\x6d\x69\x6e\x67\xff", &sa, &sb) +} + +@(test) +test_decode_lists :: proc(t: ^testing.T) { + expect_decoding(t, "\x80", "[]", ^cbor.Array) + expect_decoding(t, "\x83\x01\x02\x03", "[1, 2, 3]", ^cbor.Array) + expect_decoding(t, "\x83\x01\x82\x02\x03\x82\x04\x05", "[1, [2, 3], [4, 5]]", ^cbor.Array) + expect_decoding(t, "\x98\x19\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x18\x18\x19", "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]", ^cbor.Array) + expect_decoding(t, "\x82\x61\x61\xa1\x61\x62\x61\x63", `["a", {"b": "c"}]`, ^cbor.Array) + + // Indefinite lengths + + expect_decoding(t, "\x9f\xff", "[]", ^cbor.Array) + expect_decoding(t, "\x9f\x01\x82\x02\x03\x9f\x04\x05\xff\xff", "[1, [2, 3], [4, 5]]", ^cbor.Array) + expect_decoding(t, "\x9f\x01\x82\x02\x03\x82\x04\x05\xff", "[1, [2, 3], [4, 5]]", ^cbor.Array) + expect_decoding(t, "\x83\x01\x82\x02\x03\x9f\x04\x05\xff", "[1, [2, 3], [4, 5]]", ^cbor.Array) + expect_decoding(t, "\x83\x01\x9f\x02\x03\xff\x82\x04\x05", "[1, [2, 3], [4, 5]]", ^cbor.Array) + expect_decoding(t, "\x9f\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x18\x18\x19\xff", "[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25]", ^cbor.Array) + expect_decoding(t, "\x82\x61\x61\xbf\x61\x62\x61\x63\xff", `["a", {"b": "c"}]`, ^cbor.Array) +} + +@(test) +test_encode_lists :: proc(t: ^testing.T) { + expect_encoding(t, &cbor.Array{}, "\x80") + expect_encoding(t, &cbor.Array{u8(1), u8(2), u8(3)}, "\x83\x01\x02\x03") + expect_encoding(t, &cbor.Array{u8(1), &cbor.Array{u8(2), u8(3)}, &cbor.Array{u8(4), u8(5)}}, "\x83\x01\x82\x02\x03\x82\x04\x05") + expect_encoding(t, &cbor.Array{u8(1), u8(2), u8(3), u8(4), u8(5), u8(6), u8(7), u8(8), u8(9), u8(10), u8(11), u8(12), u8(13), u8(14), u8(15), u8(16), u8(17), u8(18), u8(19), u8(20), u8(21), u8(22), u8(23), u8(24), u8(25)}, "\x98\x19\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x18\x18\x19") + + { + a := "a" + b := "b" + c := "c" + expect_encoding(t, &cbor.Array{&a, &cbor.Map{{&b, &c}}}, "\x82\x61\x61\xa1\x61\x62\x61\x63") + } + + // Indefinite lengths + + expect_streamed_encoding(t, "\x9f\xff", &cbor.Array{}) + + { + bytes.buffer_reset(&buf) + + err: cbor.Encode_Error + err = cbor.encode_stream_begin(stream, .Array) + expect_value(t, err, nil) + + { + err = cbor.encode_stream_array_item(encoder, u8(1)) + expect_value(t, err, nil) + + err = cbor.encode_stream_array_item(encoder, &cbor.Array{u8(2), u8(3)}) + expect_value(t, err, nil) + + err = cbor.encode_stream_begin(stream, .Array) + expect_value(t, err, nil) + + { + err = cbor.encode_stream_array_item(encoder, u8(4)) + expect_value(t, err, nil) + + err = cbor.encode_stream_array_item(encoder, u8(5)) + expect_value(t, err, nil) + } + + err = cbor.encode_stream_end(stream) + expect_value(t, err, nil) + } + + err = cbor.encode_stream_end(stream) + expect_value(t, err, nil) + + expect_value(t, fmt.tprint(bytes.buffer_to_bytes(&buf)), fmt.tprint(transmute([]byte)string("\x9f\x01\x82\x02\x03\x9f\x04\x05\xff\xff"))) + } + + { + bytes.buffer_reset(&buf) + + err: cbor.Encode_Error + err = cbor._encode_u8(stream, 2, .Array) + expect_value(t, err, nil) + + a := "a" + err = cbor.encode(encoder, &a) + expect_value(t, err, nil) + + { + err = cbor.encode_stream_begin(stream, .Map) + expect_value(t, err, nil) + + b := "b" + c := "c" + err = cbor.encode_stream_map_entry(encoder, &b, &c) + expect_value(t, err, nil) + + err = cbor.encode_stream_end(stream) + expect_value(t, err, nil) + } + + expect_value(t, fmt.tprint(bytes.buffer_to_bytes(&buf)), fmt.tprint(transmute([]byte)string("\x82\x61\x61\xbf\x61\x62\x61\x63\xff"))) + } +} + +@(test) +test_decode_maps :: proc(t: ^testing.T) { + expect_decoding(t, "\xa0", "{}", ^cbor.Map) + expect_decoding(t, "\xa2\x01\x02\x03\x04", "{1: 2, 3: 4}", ^cbor.Map) + expect_decoding(t, "\xa2\x61\x61\x01\x61\x62\x82\x02\x03", `{"a": 1, "b": [2, 3]}`, ^cbor.Map) + expect_decoding(t, "\xa5\x61\x61\x61\x41\x61\x62\x61\x42\x61\x63\x61\x43\x61\x64\x61\x44\x61\x65\x61\x45", `{"a": "A", "b": "B", "c": "C", "d": "D", "e": "E"}`, ^cbor.Map) + + // Indefinite lengths + + expect_decoding(t, "\xbf\x61\x61\x01\x61\x62\x9f\x02\x03\xff\xff", `{"a": 1, "b": [2, 3]}`, ^cbor.Map) + expect_decoding(t, "\xbf\x63\x46\x75\x6e\xf5\x63\x41\x6d\x74\x21\xff", `{"Fun": true, "Amt": -2}`, ^cbor.Map) +} + +@(test) +test_encode_maps :: proc(t: ^testing.T) { + expect_encoding(t, &cbor.Map{}, "\xa0") + expect_encoding(t, &cbor.Map{{u8(1), u8(2)}, {u8(3), u8(4)}}, "\xa2\x01\x02\x03\x04") + + a := "a" + b := "b" + // NOTE: also tests the deterministic nature because it has to swap/sort the entries. + expect_encoding(t, &cbor.Map{{&b, &cbor.Array{u8(2), u8(3)}}, {&a, u8(1)}}, "\xa2\x61\x61\x01\x61\x62\x82\x02\x03") + + fun := "Fun" + amt := "Amt" + expect_streamed_encoding(t, "\xbf\x63\x46\x75\x6e\xf5\x63\x41\x6d\x74\x21\xff", &cbor.Map{{&fun, true}, {&amt, cbor.Negative_U8(1)}}) +} + +@(test) +test_decode_tags :: proc(t: ^testing.T) { + // Tag number 2 (unsigned bignumber), value bytes, max(u64) + 1. + expect_tag(t, "\xc2\x49\x01\x00\x00\x00\x00\x00\x00\x00\x00", cbor.TAG_UNSIGNED_BIG_NR, "2(h'100000000')") + + // Tag number 3 (negative bignumber), value bytes, negative max(u64) - 1. + expect_tag(t, "\xc3\x49\x01\x00\x00\x00\x00\x00\x00\x00\x00", cbor.TAG_NEGATIVE_BIG_NR, "3(h'100000000')") + + expect_tag(t, "\xc1\x1a\x51\x4b\x67\xb0", cbor.TAG_EPOCH_TIME_NR, "1(1363896240)") + expect_tag(t, "\xc1\xfb\x41\xd4\x52\xd9\xec\x20\x00\x00", cbor.TAG_EPOCH_TIME_NR, "1(1363896240.5000000000000000)") + expect_tag(t, "\xd8\x18\x45\x64\x49\x45\x54\x46", cbor.TAG_CBOR_NR, "24(h'6449455446')") +} + +@(test) +test_encode_tags :: proc(t: ^testing.T) { + expect_encoding(t, &cbor.Tag{cbor.TAG_UNSIGNED_BIG_NR, &cbor.Bytes{1, 0, 0, 0, 0, 0, 0, 0, 0}}, "\xc2\x49\x01\x00\x00\x00\x00\x00\x00\x00\x00") + expect_encoding(t, &cbor.Tag{cbor.TAG_EPOCH_TIME_NR, u32(1363896240)}, "\xc1\x1a\x51\x4b\x67\xb0") + expect_encoding(t, &cbor.Tag{cbor.TAG_EPOCH_TIME_NR, f64(1363896240.500)}, "\xc1\xfb\x41\xd4\x52\xd9\xec\x20\x00\x00") +} + +// Helpers + +expect_decoding :: proc(t: ^testing.T, encoded: string, decoded: string, type: typeid, loc := #caller_location) { + res, err := cbor.decode(encoded) + defer cbor.destroy(res) + + expect_value(t, reflect.union_variant_typeid(res), type, loc) + expect_value(t, err, nil, loc) + + str := cbor.to_diagnostic_format(res, padding=-1) + defer delete(str) + + expect_value(t, str, decoded, loc) +} + +expect_tag :: proc(t: ^testing.T, encoded: string, nr: cbor.Tag_Number, value_decoded: string, loc := #caller_location) { + res, err := cbor.decode(encoded) + defer cbor.destroy(res) + + expect_value(t, err, nil, loc) + + if tag, is_tag := res.(^cbor.Tag); is_tag { + expect_value(t, tag.number, nr, loc) + + str := cbor.to_diagnostic_format(tag, padding=-1) + defer delete(str) + + expect_value(t, str, value_decoded, loc) + } else { + errorf(t, "Value %#v is not a tag", res, loc) + } +} + +expect_float :: proc(t: ^testing.T, encoded: string, expected: $T, loc := #caller_location) where intrinsics.type_is_float(T) { + res, err := cbor.decode(encoded) + defer cbor.destroy(res) + + expect_value(t, reflect.union_variant_typeid(res), typeid_of(T), loc) + expect_value(t, err, nil, loc) + + #partial switch r in res { + case f16: + when T == f16 { expect_value(t, res, expected, loc) } else { unreachable() } + case f32: + when T == f32 { expect_value(t, res, expected, loc) } else { unreachable() } + case f64: + when T == f64 { expect_value(t, res, expected, loc) } else { unreachable() } + case: + unreachable() + } +} + +buf: bytes.Buffer +stream := bytes.buffer_to_stream(&buf) +encoder := cbor.Encoder{cbor.ENCODE_FULLY_DETERMINISTIC, stream, {}} + +expect_encoding :: proc(t: ^testing.T, val: cbor.Value, encoded: string, loc := #caller_location) { + bytes.buffer_reset(&buf) + + err := cbor.encode(encoder, val) + expect_value(t, err, nil, loc) + expect_value(t, fmt.tprint(bytes.buffer_to_bytes(&buf)), fmt.tprint(transmute([]byte)encoded), loc) +} + +expect_streamed_encoding :: proc(t: ^testing.T, encoded: string, values: ..cbor.Value, loc := #caller_location) { + bytes.buffer_reset(&buf) + + for value, i in values { + err: cbor.Encode_Error + err2: cbor.Encode_Error + #partial switch v in value { + case ^cbor.Bytes: + if i == 0 { err = cbor.encode_stream_begin(stream, .Bytes) } + err2 = cbor._encode_bytes(encoder, v^) + case ^cbor.Text: + if i == 0 { err = cbor.encode_stream_begin(stream, .Text) } + err2 = cbor._encode_text(encoder, v^) + case ^cbor.Array: + if i == 0 { err = cbor.encode_stream_begin(stream, .Array) } + for item in v { + err2 = cbor.encode_stream_array_item(encoder, item) + if err2 != nil { break } + } + case ^cbor.Map: + err = cbor.encode_stream_begin(stream, .Map) + for item in v { + err2 = cbor.encode_stream_map_entry(encoder, item.key, item.value) + if err2 != nil { break } + } + case: + errorf(t, "%v does not support streamed encoding", reflect.union_variant_typeid(value)) + } + + expect_value(t, err, nil, loc) + expect_value(t, err2, nil, loc) + } + + err := cbor.encode_stream_end(stream) + expect_value(t, err, nil, loc) + + expect_value(t, fmt.tprint(bytes.buffer_to_bytes(&buf)), fmt.tprint(transmute([]byte)encoded), loc) +} diff --git a/tests/core/encoding/hex/test_core_hex.odin b/tests/core/encoding/hex/test_core_hex.odin index a5daa206e..d928cd28e 100644 --- a/tests/core/encoding/hex/test_core_hex.odin +++ b/tests/core/encoding/hex/test_core_hex.odin @@ -4,7 +4,6 @@ import "core:encoding/hex" import "core:testing" import "core:fmt" import "core:os" -import "core:bytes" TEST_count := 0 TEST_fail := 0 diff --git a/tests/core/net/test_core_net.odin b/tests/core/net/test_core_net.odin index 579298904..9df03414c 100644 --- a/tests/core/net/test_core_net.odin +++ b/tests/core/net/test_core_net.odin @@ -473,6 +473,7 @@ client_sends_server_data :: proc(t: ^testing.T) { URL_Test :: struct { scheme, host, path: string, queries: map[string]string, + fragment: string, url: []string, } @@ -481,58 +482,78 @@ split_url_test :: proc(t: ^testing.T) { test_cases := []URL_Test{ { "http", "example.com", "/", - {}, + {}, "", {"http://example.com"}, }, { "https", "odin-lang.org", "/", - {}, + {}, "", {"https://odin-lang.org"}, }, { "https", "odin-lang.org", "/docs/", - {}, + {}, "", {"https://odin-lang.org/docs/"}, }, { "https", "odin-lang.org", "/docs/overview", - {}, + {}, "", {"https://odin-lang.org/docs/overview"}, }, { "http", "example.com", "/", - {"a" = "b"}, + {"a" = "b"}, "", {"http://example.com?a=b"}, }, { "http", "example.com", "/", - {"a" = ""}, + {"a" = ""}, "", {"http://example.com?a"}, }, { "http", "example.com", "/", - {"a" = "b", "c" = "d"}, + {"a" = "b", "c" = "d"}, "", {"http://example.com?a=b&c=d"}, }, { "http", "example.com", "/", - {"a" = "", "c" = "d"}, + {"a" = "", "c" = "d"}, "", {"http://example.com?a&c=d"}, }, { "http", "example.com", "/example", - {"a" = "", "b" = ""}, + {"a" = "", "b" = ""}, "", {"http://example.com/example?a&b"}, }, { "https", "example.com", "/callback", - {"redirect" = "https://other.com/login"}, + {"redirect" = "https://other.com/login"}, "", {"https://example.com/callback?redirect=https://other.com/login"}, }, + { + "http", "example.com", "/", + {}, "Hellope", + {"http://example.com#Hellope"}, + }, + { + "https", "odin-lang.org", "/", + {"a" = ""}, "Hellope", + {"https://odin-lang.org?a#Hellope"}, + }, + { + "http", "example.com", "/", + {"a" = "b"}, "BeesKnees", + {"http://example.com?a=b#BeesKnees"}, + }, + { + "https", "odin-lang.org", "/docs/overview/", + {}, "hellope", + {"https://odin-lang.org/docs/overview/#hellope"}, + }, } for test in test_cases { - scheme, host, path, queries := net.split_url(test.url[0]) + scheme, host, path, queries, fragment := net.split_url(test.url[0]) defer { delete(queries) delete(test.queries) @@ -551,6 +572,9 @@ split_url_test :: proc(t: ^testing.T) { msg = fmt.tprintf("Expected `net.split_url` to return %s, got %s", expected, v) expect(t, v == expected, msg) } + msg = fmt.tprintf("Expected `net.split_url` to return %s, got %s", test.fragment, fragment) + expect(t, fragment == test.fragment, msg) + } } @@ -560,53 +584,73 @@ join_url_test :: proc(t: ^testing.T) { test_cases := []URL_Test{ { "http", "example.com", "/", - {}, + {}, "", {"http://example.com/"}, }, { "https", "odin-lang.org", "/", - {}, + {}, "", {"https://odin-lang.org/"}, }, { "https", "odin-lang.org", "/docs/", - {}, + {}, "", {"https://odin-lang.org/docs/"}, }, { "https", "odin-lang.org", "/docs/overview", - {}, + {}, "", {"https://odin-lang.org/docs/overview"}, }, { "http", "example.com", "/", - {"a" = "b"}, + {"a" = "b"}, "", {"http://example.com/?a=b"}, }, { "http", "example.com", "/", - {"a" = ""}, + {"a" = ""}, "", {"http://example.com/?a"}, }, { "http", "example.com", "/", - {"a" = "b", "c" = "d"}, + {"a" = "b", "c" = "d"}, "", {"http://example.com/?a=b&c=d", "http://example.com/?c=d&a=b"}, }, { "http", "example.com", "/", - {"a" = "", "c" = "d"}, + {"a" = "", "c" = "d"}, "", {"http://example.com/?a&c=d", "http://example.com/?c=d&a"}, }, { "http", "example.com", "/example", - {"a" = "", "b" = ""}, + {"a" = "", "b" = ""}, "", {"http://example.com/example?a&b", "http://example.com/example?b&a"}, }, + { + "http", "example.com", "/", + {}, "Hellope", + {"http://example.com/#Hellope"}, + }, + { + "https", "odin-lang.org", "/", + {"a" = ""}, "Hellope", + {"https://odin-lang.org/?a#Hellope"}, + }, + { + "http", "example.com", "/", + {"a" = "b"}, "BeesKnees", + {"http://example.com/?a=b#BeesKnees"}, + }, + { + "https", "odin-lang.org", "/docs/overview/", + {}, "hellope", + {"https://odin-lang.org/docs/overview/#hellope"}, + }, } for test in test_cases { - url := net.join_url(test.scheme, test.host, test.path, test.queries) + url := net.join_url(test.scheme, test.host, test.path, test.queries, test.fragment) defer { delete(url) delete(test.queries) diff --git a/vendor/directx/d3d12/d3d12.odin b/vendor/directx/d3d12/d3d12.odin index c39c2c9c9..7c4065d8b 100644 --- a/vendor/directx/d3d12/d3d12.odin +++ b/vendor/directx/d3d12/d3d12.odin @@ -2283,7 +2283,7 @@ IHeap :: struct #raw_union { } IHeap_VTable :: struct { using id3d12devicechild_vtable: IDeviceChild_VTable, - GetDesc: proc "system" (this: ^IHeap) -> HEAP_DESC, + GetDesc: proc "system" (this: ^IHeap, pRetValue: ^HEAP_DESC) -> ^HEAP_DESC, } @@ -2297,7 +2297,7 @@ IResource_VTable :: struct { using id3d12devicechild_vtable: IDeviceChild_VTable, Map: proc "system" (this: ^IResource, Subresource: u32, pReadRange: ^RANGE, ppData: ^rawptr) -> HRESULT, Unmap: proc "system" (this: ^IResource, Subresource: u32, pWrittenRange: ^RANGE), - GetDesc: proc "system" (this: ^IResource) -> RESOURCE_DESC, + GetDesc: proc "system" (this: ^IResource, pRetValue: ^RESOURCE_DESC) -> ^RESOURCE_DESC, GetGPUVirtualAddress: proc "system" (this: ^IResource) -> GPU_VIRTUAL_ADDRESS, WriteToSubresource: proc "system" (this: ^IResource, DstSubresource: u32, pDstBox: ^BOX, pSrcData: rawptr, SrcRowPitch: u32, SrcDepthPitch: u32) -> HRESULT, ReadFromSubresource: proc "system" (this: ^IResource, pDstData: rawptr, DstRowPitch: u32, DstDepthPitch: u32, SrcSubresource: u32, pSrcBox: ^BOX) -> HRESULT, @@ -2514,7 +2514,7 @@ ICommandQueue_VTable :: struct { Wait: proc "system" (this: ^ICommandQueue, pFence: ^IFence, Value: u64) -> HRESULT, GetTimestampFrequency: proc "system" (this: ^ICommandQueue, pFrequency: ^u64) -> HRESULT, GetClockCalibration: proc "system" (this: ^ICommandQueue, pGpuTimestamp: ^u64, pCpuTimestamp: ^u64) -> HRESULT, - GetDesc: proc "system" (this: ^ICommandQueue) -> COMMAND_QUEUE_DESC, + GetDesc: proc "system" (this: ^ICommandQueue, pRetVal: ^COMMAND_QUEUE_DESC) -> ^COMMAND_QUEUE_DESC, } @@ -2712,7 +2712,7 @@ IProtectedResourceSession :: struct #raw_union { } IProtectedResourceSession_VTable :: struct { using id3d12protectedsession_vtable: IProtectedSession_VTable, - GetDesc: proc "system" (this: ^IProtectedResourceSession) -> PROTECTED_RESOURCE_SESSION_DESC, + GetDesc: proc "system" (this: ^IProtectedResourceSession, pRetVal: ^PROTECTED_RESOURCE_SESSION_DESC) -> ^PROTECTED_RESOURCE_SESSION_DESC, } @@ -3492,7 +3492,7 @@ IProtectedResourceSession1 :: struct #raw_union { } IProtectedResourceSession1_VTable :: struct { using id3d12protectedresourcesession_vtable: IProtectedResourceSession_VTable, - GetDesc1: proc "system" (this: ^IProtectedResourceSession1) -> PROTECTED_RESOURCE_SESSION_DESC1, + GetDesc1: proc "system" (this: ^IProtectedResourceSession1, pRetVal: ^PROTECTED_RESOURCE_SESSION_DESC1) -> ^PROTECTED_RESOURCE_SESSION_DESC1, } @@ -3545,7 +3545,7 @@ IResource2 :: struct #raw_union { } IResource2_VTable :: struct { using id3d12resource1_vtable: IResource1_VTable, - GetDesc1: proc "system" (this: ^IResource2) -> RESOURCE_DESC1, + GetDesc1: proc "system" (this: ^IResource2, pRetVal: ^RESOURCE_DESC1) -> ^RESOURCE_DESC1, } diff --git a/vendor/directx/dxc/dxcdef_unix.odin b/vendor/directx/dxc/dxcdef_unix.odin index 649bb8a13..12a682310 100644 --- a/vendor/directx/dxc/dxcdef_unix.odin +++ b/vendor/directx/dxc/dxcdef_unix.odin @@ -1,4 +1,4 @@ -//+build linux, darwin +//+build linux, darwin, freebsd, openbsd package directx_dxc import "core:c" diff --git a/vendor/wasm/js/runtime.js b/vendor/wasm/js/runtime.js index 2e69a28c7..320d74d68 100644 --- a/vendor/wasm/js/runtime.js +++ b/vendor/wasm/js/runtime.js @@ -104,9 +104,12 @@ class WasmMemoryInterface { storeInt(addr, value) { this.mem.setInt32 (addr, value, true); } storeUint(addr, value) { this.mem.setUint32 (addr, value, true); } + // Returned length might not be the same as `value.length` if non-ascii strings are given. storeString(addr, value) { - const bytes = this.loadBytes(addr, value.length); - new TextEncoder().encodeInto(value, bytes); + const src = new TextEncoder().encode(value); + const dst = new Uint8Array(this.memory.buffer, addr, src.length); + dst.set(src); + return src.length; } };