From a6fa41e290a1e0ddb20020db9135808849543d5b Mon Sep 17 00:00:00 2001 From: Michael Kutowski Date: Fri, 5 Aug 2022 18:53:29 +0200 Subject: [PATCH 1/6] marshal output options with pretty option and other config --- core/encoding/json/marshal.odin | 267 +++++++++++++++++++++++++------- 1 file changed, 215 insertions(+), 52 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 14df4c127..1e92bc435 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -1,41 +1,59 @@ -package json +package src -import "core:mem" import "core:math/bits" +import "core:mem" import "core:runtime" import "core:strconv" -import "core:strings" import "core:io" +import "core:strings" +import "core:encoding/json" -Marshal_Data_Error :: enum { - None, - Unsupported_Type, +// supports json specs +Marshal_Options :: struct { + // output based on spec + spec: json.Specification, + + // use line breaks & tab|spaces + pretty: bool, + + // spacing + use_spaces: bool, + spaces: int, + tabs: int, + + // state + indentation: int, + + // mjson output options + mjson_keys_use_quotes: bool, + mjson_keys_use_equal_sign: bool, + + // mjson state + mjson_skipped_first_braces_start: bool, + mjson_skipped_first_braces_end: bool, } -Marshal_Error :: union #shared_nil { - Marshal_Data_Error, - io.Error, -} - -marshal :: proc(v: any, allocator := context.allocator) -> (data: []byte, err: Marshal_Error) { - b := strings.make_builder(allocator) +marshal :: proc(v: any, opt: Marshal_Options = {}, allocator := context.allocator) -> (data: []byte, err: json.Marshal_Error) { + b := strings.builder_make(allocator) defer if err != nil { - strings.destroy_builder(&b) + strings.builder_destroy(&b) } - marshal_to_builder(&b, v) or_return + opt := opt + marshal_to_builder(&b, v, &opt) or_return if len(b.buf) != 0 { data = b.buf[:] } + return data, nil } -marshal_to_builder :: proc(b: ^strings.Builder, v: any) -> Marshal_Error { - return marshal_to_writer(strings.to_writer(b), v) +marshal_to_builder :: proc(b: ^strings.Builder, v: any, opt: ^Marshal_Options) -> json.Marshal_Error { + return marshal_to_writer(strings.to_writer(b), v, opt) } -marshal_to_writer :: proc(w: io.Writer, v: any) -> (err: Marshal_Error) { +marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: json.Marshal_Error) { if v == nil { io.write_string(w, "null") or_return return @@ -166,52 +184,48 @@ marshal_to_writer :: proc(w: io.Writer, v: any) -> (err: Marshal_Error) { return .Unsupported_Type case runtime.Type_Info_Array: - io.write_byte(w, '[') or_return + opt_write_start(w, opt, '[') or_return for i in 0.. 0 { io.write_string(w, ", ") or_return } - + opt_write_iteration(w, opt, i) or_return data := uintptr(v.data) + uintptr(i*info.elem_size) - marshal_to_writer(w, any{rawptr(data), info.elem.id}) or_return + marshal_to_writer(w, any{rawptr(data), info.elem.id}, opt) or_return } - io.write_byte(w, ']') or_return + opt_write_end(w, opt, ']') or_return case runtime.Type_Info_Enumerated_Array: index := runtime.type_info_base(info.index).variant.(runtime.Type_Info_Enum) - io.write_byte(w, '[') or_return + opt_write_start(w, opt, '[') or_return for i in 0.. 0 { io.write_string(w, ", ") or_return } - + opt_write_iteration(w, opt, i) or_return data := uintptr(v.data) + uintptr(i*info.elem_size) - marshal_to_writer(w, any{rawptr(data), info.elem.id}) or_return + marshal_to_writer(w, any{rawptr(data), info.elem.id}, opt) or_return } - io.write_byte(w, ']') or_return + opt_write_end(w, opt, ']') or_return case runtime.Type_Info_Dynamic_Array: - io.write_byte(w, '[') or_return + opt_write_start(w, opt, '[') or_return array := cast(^mem.Raw_Dynamic_Array)v.data for i in 0.. 0 { io.write_string(w, ", ") or_return } - + opt_write_iteration(w, opt, i) or_return data := uintptr(array.data) + uintptr(i*info.elem_size) - marshal_to_writer(w, any{rawptr(data), info.elem.id}) or_return + marshal_to_writer(w, any{rawptr(data), info.elem.id}, opt) or_return } - io.write_byte(w, ']') or_return + opt_write_end(w, opt, ']') or_return case runtime.Type_Info_Slice: - io.write_byte(w, '[') or_return + opt_write_start(w, opt, '[') or_return slice := cast(^mem.Raw_Slice)v.data for i in 0.. 0 { io.write_string(w, ", ") or_return } - + opt_write_iteration(w, opt, i) or_return data := uintptr(slice.data) + uintptr(i*info.elem_size) - marshal_to_writer(w, any{rawptr(data), info.elem.id}) or_return + marshal_to_writer(w, any{rawptr(data), info.elem.id}, opt) or_return } - io.write_byte(w, ']') or_return + opt_write_end(w, opt, ']') or_return case runtime.Type_Info_Map: m := (^mem.Raw_Map)(v.data) + opt_write_start(w, opt, '{') or_return - io.write_byte(w, '{') or_return if m != nil { if info.generated_struct == nil { return .Unsupported_Type @@ -223,31 +237,62 @@ marshal_to_writer :: proc(w: io.Writer, v: any) -> (err: Marshal_Error) { entry_size := ed.elem_size for i in 0.. 0 { io.write_string(w, ", ") or_return } + opt_write_iteration(w, opt, i) or_return data := uintptr(entries.data) + uintptr(i*entry_size) key := rawptr(data + entry_type.offsets[2]) value := rawptr(data + entry_type.offsets[3]) - marshal_to_writer(w, any{key, info.key.id}) or_return - io.write_string(w, ": ") or_return - marshal_to_writer(w, any{value, info.value.id}) or_return + // check for string type + { + v := any{key, info.key.id} + ti := runtime.type_info_base(type_info_of(v.id)) + a := any{v.data, ti.id} + name: string + + #partial switch info in ti.variant { + case runtime.Type_Info_String: { + // fmt.eprintln("WAS STRING") + + switch s in a { + case string: name = s + case cstring: name = string(s) + } + + // NOTE need to ensure that map keys are valid for mjson and contain no whitespace + if opt.spec == .MJSON && !opt.mjson_keys_use_quotes { + name, _ = strings.replace_all(name, " ", "_", context.temp_allocator) + } + + opt_write_key(w, opt, name) or_return + } + + case: { + // TODO better error output? + return .Unsupported_Type + } + } + } + + marshal_to_writer(w, any{value, info.value.id}, opt) or_return } } - io.write_byte(w, '}') or_return + + opt_write_end(w, opt, '}') or_return case runtime.Type_Info_Struct: - io.write_byte(w, '{') or_return + opt_write_start(w, opt, '{') or_return + for name, i in info.names { - if i > 0 { io.write_string(w, ", ") or_return } - io.write_quoted_string(w, name) or_return - io.write_string(w, ": ") or_return + opt_write_iteration(w, opt, i) or_return + opt_write_key(w, opt, name) or_return id := info.types[i].id data := rawptr(uintptr(v.data) + info.offsets[i]) - marshal_to_writer(w, any{data, id}) or_return + marshal_to_writer(w, any{data, id}, opt) or_return } - io.write_byte(w, '}') or_return + + opt_write_end(w, opt, '}') or_return case runtime.Type_Info_Union: tag_ptr := uintptr(v.data) + info.tag_offset @@ -270,11 +315,11 @@ marshal_to_writer :: proc(w: io.Writer, v: any) -> (err: Marshal_Error) { io.write_string(w, "null") or_return } else { id := info.variants[tag-1].id - return marshal_to_writer(w, any{v.data, id}) + return marshal_to_writer(w, any{v.data, id}, opt) } case runtime.Type_Info_Enum: - return marshal_to_writer(w, any{v.data, info.base.id}) + return marshal_to_writer(w, any{v.data, info.base.id}, opt) case runtime.Type_Info_Bit_Set: is_bit_set_different_endian_to_platform :: proc(ti: ^runtime.Type_Info) -> bool { @@ -330,3 +375,121 @@ marshal_to_writer :: proc(w: io.Writer, v: any) -> (err: Marshal_Error) { return } + +// write key as quoted string or with optional quotes in mjson +opt_write_key :: proc(w: io.Writer, opt: ^Marshal_Options, name: string) -> (err: io.Error) { + switch opt.spec { + case .JSON, .JSON5: { + io.write_quoted_string(w, name) or_return + io.write_string(w, ": ") or_return + } + + case .MJSON: { + if opt.mjson_keys_use_quotes { + io.write_quoted_string(w, name) or_return + } else { + io.write_string(w, name) or_return + } + + if opt.mjson_keys_use_equal_sign { + io.write_string(w, " = ") or_return + } else { + io.write_string(w, ": ") or_return + } + } + } + + return +} + +// insert start byte and increase indentation on pretty +opt_write_start :: proc(w: io.Writer, opt: ^Marshal_Options, c: byte) -> (err: io.Error) { + // skip mjson starting braces + if opt.spec == .MJSON && !opt.mjson_skipped_first_braces_start { + opt.mjson_skipped_first_braces_start = true + return + } + + io.write_byte(w, c) or_return + opt.indentation += 1 + + if opt.pretty { + io.write_byte(w, '\n') or_return + } + + return +} + +// insert comma seperation and write indentations +opt_write_iteration :: proc(w: io.Writer, opt: ^Marshal_Options, iteration: int) -> (err: io.Error) { + switch opt.spec { + case .JSON, .JSON5: { + if iteration > 0 { + io.write_string(w, ", ") or_return + + if opt.pretty { + io.write_byte(w, '\n') or_return + } + } + + opt_write_indentation(w, opt) or_return + } + + case .MJSON: { + if iteration > 0 { + // on pretty no commas necessary + if opt.pretty { + io.write_byte(w, '\n') or_return + } else { + // NOTE comma seperation necessary for non pretty output! + io.write_string(w, ", ") or_return + } + } + + opt_write_indentation(w, opt) or_return + } + } + + return +} + +// decrease indent, write spacing and insert end byte +opt_write_end :: proc(w: io.Writer, opt: ^Marshal_Options, c: byte) -> (err: io.Error) { + if opt.spec == .MJSON && opt.mjson_skipped_first_braces_start && !opt.mjson_skipped_first_braces_end { + if opt.indentation == 0 { + opt.mjson_skipped_first_braces_end = true + return + } + } + + opt.indentation -= 1 + + if opt.pretty { + io.write_byte(w, '\n') or_return + opt_write_indentation(w, opt) or_return + } + + io.write_byte(w, c) or_return + return +} + +// writes current indentation level based on options +opt_write_indentation :: proc(w: io.Writer, opt: ^Marshal_Options) -> (err: io.Error) { + if !opt.pretty { + return + } + + // TODO optimize? + if opt.use_spaces { + // NOTE maybe max(1, opt.spaces) + for _ in 0.. Date: Fri, 5 Aug 2022 18:56:14 +0200 Subject: [PATCH 2/6] ...bad paste --- core/encoding/json/marshal.odin | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 1e92bc435..095d290f5 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -1,17 +1,26 @@ -package src +package json -import "core:math/bits" import "core:mem" +import "core:math/bits" import "core:runtime" import "core:strconv" -import "core:io" import "core:strings" -import "core:encoding/json" +import "core:io" + +Marshal_Data_Error :: enum { + None, + Unsupported_Type, +} + +Marshal_Error :: union #shared_nil { + Marshal_Data_Error, + io.Error, +} // supports json specs Marshal_Options :: struct { // output based on spec - spec: json.Specification, + spec: Specification, // use line breaks & tab|spaces pretty: bool, @@ -33,7 +42,7 @@ Marshal_Options :: struct { mjson_skipped_first_braces_end: bool, } -marshal :: proc(v: any, opt: Marshal_Options = {}, allocator := context.allocator) -> (data: []byte, err: json.Marshal_Error) { +marshal :: proc(v: any, opt: Marshal_Options = {}, allocator := context.allocator) -> (data: []byte, err: Marshal_Error) { b := strings.builder_make(allocator) defer if err != nil { strings.builder_destroy(&b) @@ -49,11 +58,11 @@ marshal :: proc(v: any, opt: Marshal_Options = {}, allocator := context.allocato return data, nil } -marshal_to_builder :: proc(b: ^strings.Builder, v: any, opt: ^Marshal_Options) -> json.Marshal_Error { +marshal_to_builder :: proc(b: ^strings.Builder, v: any, opt: ^Marshal_Options) -> Marshal_Error { return marshal_to_writer(strings.to_writer(b), v, opt) } -marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: json.Marshal_Error) { +marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: Marshal_Error) { if v == nil { io.write_string(w, "null") or_return return From 425dec8bb8cc4fe2cf25a008de199d3084ecb510 Mon Sep 17 00:00:00 2001 From: Michael Kutowski Date: Mon, 8 Aug 2022 18:28:28 +0200 Subject: [PATCH 3/6] add uint as hex option --- core/encoding/json/marshal.odin | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 095d290f5..2a92cec28 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -17,7 +17,7 @@ Marshal_Error :: union #shared_nil { io.Error, } -// supports json specs +// NOTE careful with MJSON maps & non quotes usage as keys without whitespace will lead to bad results Marshal_Options :: struct { // output based on spec spec: Specification, @@ -28,11 +28,13 @@ Marshal_Options :: struct { // spacing use_spaces: bool, spaces: int, - tabs: int, // state indentation: int, + // option to output uint in JSON5 & MJSON + write_uint_as_hex: bool, + // mjson output options mjson_keys_use_quotes: bool, mjson_keys_use_equal_sign: bool, @@ -109,7 +111,23 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: case u128be: u = u128(i) } - s := strconv.append_bits_128(buf[:], u, 10, info.signed, 8*ti.size, "0123456789", nil) + s: string + + // allow uints to be printed as hex + if opt.write_uint_as_hex && (opt.spec == .JSON5 || opt.spec == .MJSON) { + switch i in a { + case u8, u16, u32, u64, u128: { + s = strconv.append_bits_128(buf[:], u, 16, info.signed, 8*ti.size, "0123456789abcdef", { .Prefix }) + } + + case: { + s = strconv.append_bits_128(buf[:], u, 10, info.signed, 8*ti.size, "0123456789", nil) + } + } + } else { + s = strconv.append_bits_128(buf[:], u, 10, info.signed, 8*ti.size, "0123456789", nil) + } + io.write_string(w, s) or_return @@ -261,18 +279,11 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: #partial switch info in ti.variant { case runtime.Type_Info_String: { - // fmt.eprintln("WAS STRING") - switch s in a { case string: name = s case cstring: name = string(s) } - // NOTE need to ensure that map keys are valid for mjson and contain no whitespace - if opt.spec == .MJSON && !opt.mjson_keys_use_quotes { - name, _ = strings.replace_all(name, " ", "_", context.temp_allocator) - } - opt_write_key(w, opt, name) or_return } From 107e016508189d366e6c628f93d0729cbd5873f3 Mon Sep 17 00:00:00 2001 From: Michael Kutowski Date: Fri, 12 Aug 2022 13:42:19 +0200 Subject: [PATCH 4/6] switch styling and opt.spaces max --- core/encoding/json/marshal.odin | 109 ++++++++++++++------------------ 1 file changed, 47 insertions(+), 62 deletions(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 9dbe31cb6..55a28f85a 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -17,7 +17,7 @@ Marshal_Error :: union #shared_nil { io.Error, } -// NOTE careful with MJSON maps & non quotes usage as keys without whitespace will lead to bad results +// careful with MJSON maps & non quotes usage as keys without whitespace will lead to bad results Marshal_Options :: struct { // output based on spec spec: Specification, @@ -44,7 +44,7 @@ Marshal_Options :: struct { mjson_skipped_first_braces_end: bool, } -marshal :: proc(v: any, opt := Marshal_Options{}, allocator := context.allocator) -> (data: []byte, err: Marshal_Error) { +marshal :: proc(v: any, opt: Marshal_Options = {}, allocator := context.allocator) -> (data: []byte, err: Marshal_Error) { b := strings.builder_make(allocator) defer if err != nil { strings.builder_destroy(&b) @@ -116,13 +116,11 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: // allow uints to be printed as hex if opt.write_uint_as_hex && (opt.spec == .JSON5 || opt.spec == .MJSON) { switch i in a { - case u8, u16, u32, u64, u128: { - s = strconv.append_bits_128(buf[:], u, 16, info.signed, 8*ti.size, "0123456789abcdef", { .Prefix }) - } + case u8, u16, u32, u64, u128: + s = strconv.append_bits_128(buf[:], u, 16, info.signed, 8*ti.size, "0123456789abcdef", { .Prefix }) - case: { - s = strconv.append_bits_128(buf[:], u, 10, info.signed, 8*ti.size, "0123456789", nil) - } + case: + s = strconv.append_bits_128(buf[:], u, 10, info.signed, 8*ti.size, "0123456789", nil) } } else { s = strconv.append_bits_128(buf[:], u, 10, info.signed, 8*ti.size, "0123456789", nil) @@ -192,9 +190,6 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: case runtime.Type_Info_Multi_Pointer: return .Unsupported_Type - case runtime.Type_Info_Soa_Pointer: - return .Unsupported_Type - case runtime.Type_Info_Procedure: return .Unsupported_Type @@ -281,19 +276,14 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: name: string #partial switch info in ti.variant { - case runtime.Type_Info_String: { - switch s in a { - case string: name = s - case cstring: name = string(s) - } - - opt_write_key(w, opt, name) or_return + case runtime.Type_Info_String: + switch s in a { + case string: name = s + case cstring: name = string(s) } + opt_write_key(w, opt, name) or_return - case: { - // TODO better error output? - return .Unsupported_Type - } + case: return .Unsupported_Type } } @@ -402,23 +392,21 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: // write key as quoted string or with optional quotes in mjson opt_write_key :: proc(w: io.Writer, opt: ^Marshal_Options, name: string) -> (err: io.Error) { switch opt.spec { - case .JSON, .JSON5: { - io.write_quoted_string(w, name) or_return - io.write_string(w, ": ") or_return - } + case .JSON, .JSON5: + io.write_quoted_string(w, name) or_return + io.write_string(w, ": ") or_return - case .MJSON: { - if opt.mjson_keys_use_quotes { - io.write_quoted_string(w, name) or_return - } else { - io.write_string(w, name) or_return - } - - if opt.mjson_keys_use_equal_sign { - io.write_string(w, " = ") or_return - } else { - io.write_string(w, ": ") or_return - } + case .MJSON: + if opt.mjson_keys_use_quotes { + io.write_quoted_string(w, name) or_return + } else { + io.write_string(w, name) or_return + } + + if opt.mjson_keys_use_equal_sign { + io.write_string(w, " = ") or_return + } else { + io.write_string(w, ": ") or_return } } @@ -446,31 +434,29 @@ opt_write_start :: proc(w: io.Writer, opt: ^Marshal_Options, c: byte) -> (err: i // insert comma seperation and write indentations opt_write_iteration :: proc(w: io.Writer, opt: ^Marshal_Options, iteration: int) -> (err: io.Error) { switch opt.spec { - case .JSON, .JSON5: { - if iteration > 0 { + case .JSON, .JSON5: + if iteration > 0 { + io.write_string(w, ", ") or_return + + if opt.pretty { + io.write_byte(w, '\n') or_return + } + } + + opt_write_indentation(w, opt) or_return + + case .MJSON: + if iteration > 0 { + // on pretty no commas necessary + if opt.pretty { + io.write_byte(w, '\n') or_return + } else { + // comma seperation necessary for non pretty output! io.write_string(w, ", ") or_return - - if opt.pretty { - io.write_byte(w, '\n') or_return - } } - - opt_write_indentation(w, opt) or_return } - case .MJSON: { - if iteration > 0 { - // on pretty no commas necessary - if opt.pretty { - io.write_byte(w, '\n') or_return - } else { - // NOTE comma seperation necessary for non pretty output! - io.write_string(w, ", ") or_return - } - } - - opt_write_indentation(w, opt) or_return - } + opt_write_indentation(w, opt) or_return } return @@ -502,10 +488,9 @@ opt_write_indentation :: proc(w: io.Writer, opt: ^Marshal_Options) -> (err: io.E return } - // TODO optimize? if opt.use_spaces { - // NOTE maybe max(1, opt.spaces) - for _ in 0.. Date: Fri, 12 Aug 2022 13:50:19 +0200 Subject: [PATCH 5/6] add new soa type, my bad wasnt on master --- core/encoding/json/marshal.odin | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 55a28f85a..37e2257f5 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -190,6 +190,9 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: case runtime.Type_Info_Multi_Pointer: return .Unsupported_Type + case Type_Info_Soa_Pointer: + return .Unsupported_Type + case runtime.Type_Info_Procedure: return .Unsupported_Type From b739044e69431e41c7fce0f40a62ebc6adf673ca Mon Sep 17 00:00:00 2001 From: Michael Kutowski Date: Fri, 12 Aug 2022 13:59:11 +0200 Subject: [PATCH 6/6] Update marshal.odin --- core/encoding/json/marshal.odin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/encoding/json/marshal.odin b/core/encoding/json/marshal.odin index 37e2257f5..8cc814fcf 100644 --- a/core/encoding/json/marshal.odin +++ b/core/encoding/json/marshal.odin @@ -190,7 +190,7 @@ marshal_to_writer :: proc(w: io.Writer, v: any, opt: ^Marshal_Options) -> (err: case runtime.Type_Info_Multi_Pointer: return .Unsupported_Type - case Type_Info_Soa_Pointer: + case runtime.Type_Info_Soa_Pointer: return .Unsupported_Type case runtime.Type_Info_Procedure: