From 912f99abc8942dd4d1111ccd8f892e8c8e52b29b Mon Sep 17 00:00:00 2001 From: Laytan Laats Date: Fri, 9 Aug 2024 21:56:54 +0200 Subject: [PATCH] encoding/cbor: various fixes - "null" is the proper way to represent the nil value in the diagnostic format - hex encoding in diagnostic format was wrong - struct keys weren't sorted the right deterministic way --- core/encoding/cbor/cbor.odin | 5 +- core/encoding/cbor/marshal.odin | 18 +++-- core/encoding/hex/hex.odin | 7 ++ tests/core/encoding/cbor/test_core_cbor.odin | 83 ++++++++++---------- 4 files changed, 61 insertions(+), 52 deletions(-) diff --git a/core/encoding/cbor/cbor.odin b/core/encoding/cbor/cbor.odin index 7897b2a37..692be0020 100644 --- a/core/encoding/cbor/cbor.odin +++ b/core/encoding/cbor/cbor.odin @@ -3,6 +3,7 @@ package encoding_cbor import "base:intrinsics" import "core:encoding/json" +import "core:encoding/hex" import "core:io" import "core:mem" import "core:strconv" @@ -399,11 +400,11 @@ to_diagnostic_format_writer :: proc(w: io.Writer, val: Value, padding := 0) -> i io.write_string(w, str) or_return case bool: io.write_string(w, "true" if v else "false") or_return - case Nil: io.write_string(w, "nil") or_return + case Nil: io.write_string(w, "null") or_return case Undefined: io.write_string(w, "undefined") or_return case ^Bytes: io.write_string(w, "h'") or_return - for b in v { io.write_int(w, int(b), 16) or_return } + hex.encode_into_writer(w, v^) or_return io.write_string(w, "'") or_return case ^Text: io.write_string(w, `"`) or_return diff --git a/core/encoding/cbor/marshal.odin b/core/encoding/cbor/marshal.odin index 6657807f5..aca71deb2 100644 --- a/core/encoding/cbor/marshal.odin +++ b/core/encoding/cbor/marshal.odin @@ -481,9 +481,7 @@ _marshal_into_encoder :: proc(e: Encoder, v: any, ti: ^runtime.Type_Info) -> (er } } - marshal_entry :: #force_inline proc(e: Encoder, info: runtime.Type_Info_Struct, v: any, name: string, i: int) -> Marshal_Error { - err_conv(_encode_text(e, name)) or_return - + marshal_entry :: #force_inline proc(e: Encoder, info: runtime.Type_Info_Struct, v: any, i: int) -> Marshal_Error { id := info.types[i].id data := rawptr(uintptr(v.data) + info.offsets[i]) field_any := any{data, id} @@ -517,7 +515,7 @@ _marshal_into_encoder :: proc(e: Encoder, v: any, ti: ^runtime.Type_Info) -> (er if .Deterministic_Map_Sorting in e.flags { Name :: struct { - name: string, + name: []byte, field: int, } entries := make([dynamic]Name, 0, n, e.temp_allocator) or_return @@ -529,16 +527,19 @@ _marshal_into_encoder :: proc(e: Encoder, v: any, ti: ^runtime.Type_Info) -> (er continue } - append(&entries, Name{fname, i}) or_return + key_builder := strings.builder_make(e.temp_allocator) or_return + err_conv(_encode_text(Encoder{e.flags, strings.to_stream(&key_builder), e.temp_allocator}, fname)) or_return + append(&entries, Name{key_builder.buf[:], i}) or_return } // Sort lexicographic on the bytes of the key. slice.sort_by_cmp(entries[:], proc(a, b: Name) -> slice.Ordering { - return slice.Ordering(bytes.compare(transmute([]byte)a.name, transmute([]byte)b.name)) + return slice.Ordering(bytes.compare(a.name, b.name)) }) for entry in entries { - marshal_entry(e, info, v, entry.name, entry.field) or_return + io.write_full(e.writer, entry.name) or_return + marshal_entry(e, info, v, entry.field) or_return } } else { for _, i in info.names[:info.field_count] { @@ -547,7 +548,8 @@ _marshal_into_encoder :: proc(e: Encoder, v: any, ti: ^runtime.Type_Info) -> (er continue } - marshal_entry(e, info, v, fname, i) or_return + err_conv(_encode_text(e, fname)) or_return + marshal_entry(e, info, v, i) or_return } } return diff --git a/core/encoding/hex/hex.odin b/core/encoding/hex/hex.odin index c2cd89c5b..c1753003e 100644 --- a/core/encoding/hex/hex.odin +++ b/core/encoding/hex/hex.odin @@ -1,5 +1,6 @@ package encoding_hex +import "core:io" import "core:strings" encode :: proc(src: []byte, allocator := context.allocator, loc := #caller_location) -> []byte #no_bounds_check { @@ -14,6 +15,12 @@ encode :: proc(src: []byte, allocator := context.allocator, loc := #caller_locat return dst } +encode_into_writer :: proc(dst: io.Writer, src: []byte) -> io.Error { + for v in src { + io.write(dst, {HEXTABLE[v>>4], HEXTABLE[v&0x0f]}) or_return + } + return nil +} decode :: proc(src: []byte, allocator := context.allocator, loc := #caller_location) -> (dst: []byte, ok: bool) #no_bounds_check { if len(src) % 2 == 1 { diff --git a/tests/core/encoding/cbor/test_core_cbor.odin b/tests/core/encoding/cbor/test_core_cbor.odin index d069ef05b..45f47505a 100644 --- a/tests/core/encoding/cbor/test_core_cbor.odin +++ b/tests/core/encoding/cbor/test_core_cbor.odin @@ -117,11 +117,25 @@ test_marshalling :: proc(t: ^testing.T) { diagnosis, eerr := cbor.to_diagnostic_format(decoded) testing.expect_value(t, eerr, nil) defer delete(diagnosis) - testing.expect_value(t, diagnosis, `{ - "base64": 34("MTYgaXMgYSBuaWNlIG51bWJlcg=="), - "biggest": 2(h'f951a9fd3c158afdff08ab8e0'), - "biggie": 18446744073709551615, + "no": null, + "neg": -69, + "nos": undefined, + "now": 1(1701117968), + "pos": 1212, + "str": "Hellope", + "yes": true, + "comp": [ + 32.0000, + 33.0000 + ], + "cstr": "Hellnope", + "quat": [ + 17.0000, + 18.0000, + 19.0000, + 16.0000 + ], "child": { "dyn": [ "one", @@ -148,41 +162,26 @@ test_marshalling :: proc(t: ^testing.T) { 10 ] }, - "comp": [ - 32.0000, - 33.0000 - ], - "cstr": "Hellnope", "ennie": 0, - "ennieb": 512, - "iamint": -256, - "important": "!", - "my_bytes": h'', - "neg": -69, - "no": nil, - "nos": undefined, - "now": 1(1701117968), "nowie": { "_nsec": 1701117968000000000 }, - "onetwenty": 12345, - "pos": 1212, - "quat": [ - 17.0000, - 18.0000, - 19.0000, - 16.0000 - ], - "renamed :)": 123123.12500000, - "small_onetwenty": -18446744073709551615, - "smallest": 3(h'f951a9fd3c158afdff08ab8e0'), - "smallie": -18446744073709551616, - "str": "Hellope", "value": { 16: "16 is a nice number", 32: 69 }, - "yes": true + "base64": 34("MTYgaXMgYSBuaWNlIG51bWJlcg=="), + "biggie": 18446744073709551615, + "ennieb": 512, + "iamint": -256, + "biggest": 2(h'0f951a9fd3c158afdff08ab8e0'), + "smallie": -18446744073709551616, + "my_bytes": h'', + "smallest": 3(h'0f951a9fd3c158afdff08ab8e0'), + "important": "!", + "onetwenty": 12345, + "renamed :)": 123123.12500000, + "small_onetwenty": -18446744073709551615 }`) backf: Foo @@ -295,7 +294,7 @@ test_marshalling_nil_maybe :: proc(t: ^testing.T) { testing.expect_value(t, derr, nil) diag := cbor.to_diagnostic_format(val) - testing.expect_value(t, diag, "nil") + testing.expect_value(t, diag, "null") delete(diag) maybe_dest: Maybe(int) @@ -439,7 +438,7 @@ test_encode_negative :: proc(t: ^testing.T) { test_decode_simples :: proc(t: ^testing.T) { expect_decoding(t, "\xf4", "false", bool) expect_decoding(t, "\xf5", "true", bool) - expect_decoding(t, "\xf6", "nil", cbor.Nil) + expect_decoding(t, "\xf6", "null", cbor.Nil) expect_decoding(t, "\xf7", "undefined", cbor.Undefined) expect_decoding(t, "\xf0", "simple(16)", cbor.Simple) @@ -503,11 +502,11 @@ test_encode_floats :: proc(t: ^testing.T) { @(test) test_decode_bytes :: proc(t: ^testing.T) { expect_decoding(t, "\x40", "h''", ^cbor.Bytes) - expect_decoding(t, "\x44\x01\x02\x03\x04", "h'1234'", ^cbor.Bytes) + expect_decoding(t, "\x44\x01\x02\x03\x04", "h'01020304'", ^cbor.Bytes) // Indefinite lengths - expect_decoding(t, "\x5f\x42\x01\x02\x43\x03\x04\x05\xff", "h'12345'", ^cbor.Bytes) + expect_decoding(t, "\x5f\x42\x01\x02\x43\x03\x04\x05\xff", "h'0102030405'", ^cbor.Bytes) } @(test) @@ -703,10 +702,10 @@ test_encode_maps :: proc(t: ^testing.T) { @(test) test_decode_tags :: proc(t: ^testing.T) { // Tag number 2 (unsigned bignumber), value bytes, max(u64) + 1. - expect_tag(t, "\xc2\x49\x01\x00\x00\x00\x00\x00\x00\x00\x00", cbor.TAG_UNSIGNED_BIG_NR, "2(h'100000000')") + expect_tag(t, "\xc2\x49\x01\x00\x00\x00\x00\x00\x00\x00\x00", cbor.TAG_UNSIGNED_BIG_NR, "2(h'010000000000000000')") // Tag number 3 (negative bignumber), value bytes, negative max(u64) - 1. - expect_tag(t, "\xc3\x49\x01\x00\x00\x00\x00\x00\x00\x00\x00", cbor.TAG_NEGATIVE_BIG_NR, "3(h'100000000')") + expect_tag(t, "\xc3\x49\x01\x00\x00\x00\x00\x00\x00\x00\x00", cbor.TAG_NEGATIVE_BIG_NR, "3(h'010000000000000000')") expect_tag(t, "\xc1\x1a\x51\x4b\x67\xb0", cbor.TAG_EPOCH_TIME_NR, "1(1363896240)") expect_tag(t, "\xc1\xfb\x41\xd4\x52\xd9\xec\x20\x00\x00", cbor.TAG_EPOCH_TIME_NR, "1(1363896240.5000000000000000)") @@ -723,16 +722,16 @@ test_encode_tags :: proc(t: ^testing.T) { // Helpers expect_decoding :: proc(t: ^testing.T, encoded: string, decoded: string, type: typeid, loc := #caller_location) { - res, err := cbor.decode(encoded) + res, err := cbor.decode(encoded) defer cbor.destroy(res) testing.expect_value(t, reflect.union_variant_typeid(res), type, loc) - testing.expect_value(t, err, nil, loc) + testing.expect_value(t, err, nil, loc) str := cbor.to_diagnostic_format(res, padding=-1) defer delete(str) - testing.expect_value(t, str, decoded, loc) + testing.expect_value(t, str, decoded, loc) } expect_tag :: proc(t: ^testing.T, encoded: string, nr: cbor.Tag_Number, value_decoded: string, loc := #caller_location) { @@ -754,11 +753,11 @@ expect_tag :: proc(t: ^testing.T, encoded: string, nr: cbor.Tag_Number, value_de } expect_float :: proc(t: ^testing.T, encoded: string, expected: $T, loc := #caller_location) where intrinsics.type_is_float(T) { - res, err := cbor.decode(encoded) + res, err := cbor.decode(encoded) defer cbor.destroy(res) testing.expect_value(t, reflect.union_variant_typeid(res), typeid_of(T), loc) - testing.expect_value(t, err, nil, loc) + testing.expect_value(t, err, nil, loc) #partial switch r in res { case f16: