From 0a8b266c7169dba1da9a4f316c4c54db2c1481c9 Mon Sep 17 00:00:00 2001 From: Rickard Andersson Date: Mon, 2 Oct 2023 11:50:16 +0300 Subject: [PATCH 1/6] fix(json): return `.Out_Of_Memory` when out of memory on parse Previously this would silently simply not do anything and the object would be empty/incomplete when parsed instead. --- core/encoding/json/parser.odin | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index bc381efee..a57f6bebd 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -263,6 +263,12 @@ parse_object_body :: proc(p: ^Parser, end_token: Token_Kind) -> (obj: Object, er return } + if len(obj) == cap(obj) { + reserve_error := reserve(&obj, max(1, cap(obj) * 2)) + if reserve_error != nil { + return nil, .Out_Of_Memory + } + } obj[key] = elem if parse_comma(p) { From 55a1ba710b637a676f6c6f9bb5e78e9453b959b5 Mon Sep 17 00:00:00 2001 From: Rickard Andersson Date: Mon, 2 Oct 2023 11:59:37 +0300 Subject: [PATCH 2/6] fix: use `runtime.map_insert` to not overallocate --- core/encoding/json/parser.odin | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index a57f6bebd..1ccb948bb 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -3,6 +3,7 @@ package json import "core:mem" import "core:unicode/utf8" import "core:unicode/utf16" +import "core:runtime" import "core:strconv" Parser :: struct { @@ -263,13 +264,10 @@ parse_object_body :: proc(p: ^Parser, end_token: Token_Kind) -> (obj: Object, er return } - if len(obj) == cap(obj) { - reserve_error := reserve(&obj, max(1, cap(obj) * 2)) - if reserve_error != nil { - return nil, .Out_Of_Memory - } + insert_success := runtime.map_insert(&obj, key, elem) + if insert_success == nil { + return nil, .Out_Of_Memory } - obj[key] = elem if parse_comma(p) { break From 11e884aec511d21798f7ecebec5c658a82ff590d Mon Sep 17 00:00:00 2001 From: Rickard Andersson Date: Mon, 2 Oct 2023 12:20:18 +0300 Subject: [PATCH 3/6] docs: add note about checking for alloc error --- core/encoding/json/parser.odin | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index 1ccb948bb..aa44d19e6 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -265,6 +265,8 @@ parse_object_body :: proc(p: ^Parser, end_token: Token_Kind) -> (obj: Object, er } insert_success := runtime.map_insert(&obj, key, elem) + // NOTE(gonz): we'd rather check specifically for an allocation error here but + // `map_insert` doesn't differentiate; we can only check for `nil` if insert_success == nil { return nil, .Out_Of_Memory } From cfa3765d50347647606be9468da549ce54347018 Mon Sep 17 00:00:00 2001 From: Rickard Andersson Date: Mon, 2 Oct 2023 15:10:12 +0300 Subject: [PATCH 4/6] fix: guard against empty key value in `parse_object_body` --- core/encoding/json/parser.odin | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index aa44d19e6..b40b92486 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -264,13 +264,17 @@ parse_object_body :: proc(p: ^Parser, end_token: Token_Kind) -> (obj: Object, er return } - insert_success := runtime.map_insert(&obj, key, elem) - // NOTE(gonz): we'd rather check specifically for an allocation error here but - // `map_insert` doesn't differentiate; we can only check for `nil` - if insert_success == nil { - return nil, .Out_Of_Memory + // NOTE(gonz): There are code paths for which this traversal ends up + // inserting empty key/values into the object and for those we do not + // want to allocate anything + if key != "" { + reserve_error := reserve(&obj, len(obj) + 1) + if reserve_error == mem.Allocator_Error.Out_Of_Memory { + return nil, .Out_Of_Memory + } + obj[key] = elem } - + if parse_comma(p) { break } From 2e3224a138832b786c4c2905b93208d27b37d358 Mon Sep 17 00:00:00 2001 From: Rickard Andersson Date: Mon, 2 Oct 2023 15:17:06 +0300 Subject: [PATCH 5/6] testing: add test for `Out_Of_Memory` return --- tests/core/encoding/json/test_core_json.odin | 44 ++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/core/encoding/json/test_core_json.odin b/tests/core/encoding/json/test_core_json.odin index 23361d694..813d11b2c 100644 --- a/tests/core/encoding/json/test_core_json.odin +++ b/tests/core/encoding/json/test_core_json.odin @@ -4,6 +4,7 @@ import "core:encoding/json" import "core:testing" import "core:fmt" import "core:os" +import "core:mem/virtual" TEST_count := 0 TEST_fail := 0 @@ -77,6 +78,49 @@ parse_json :: proc(t: ^testing.T) { expect(t, err == nil, msg) } +@test +out_of_memory_in_parse_json :: proc(t: ^testing.T) { + arena: virtual.Arena + arena_buffer: [256]byte + arena_init_error := virtual.arena_init_buffer(&arena, arena_buffer[:]) + testing.expect(t, arena_init_error == nil, fmt.tprintf("Expected arena initialization to not return error, got: %v\n", arena_init_error)) + + context.allocator = virtual.arena_allocator(&arena) + + json_data := ` + { + "firstName": "John", + "lastName": "Smith", + "isAlive": true, + "age": 27, + "address": { + "streetAddress": "21 2nd Street", + "city": "New York", + "state": "NY", + "postalCode": "10021-3100" + }, + "phoneNumbers": [ + { + "type": "home", + "number": "212 555-1234" + }, + { + "type": "office", + "number": "646 555-4567" + } + ], + "children": [], + "spouse": null + } + ` + + _, err := json.parse(transmute([]u8)json_data) + + expected_error := json.Error.Out_Of_Memory + msg := fmt.tprintf("Expected `json.parse` to fail with %v, got %v", expected_error, err) + expect(t, err == json.Error.Out_Of_Memory, msg) +} + @test marshal_json :: proc(t: ^testing.T) { From 931e0d4687da09bfa17f02b80dfef20ad95d4d6e Mon Sep 17 00:00:00 2001 From: Rickard Andersson Date: Mon, 2 Oct 2023 15:21:09 +0300 Subject: [PATCH 6/6] cleanup: remove unused import --- core/encoding/json/parser.odin | 1 - 1 file changed, 1 deletion(-) diff --git a/core/encoding/json/parser.odin b/core/encoding/json/parser.odin index b40b92486..6faaf3f32 100644 --- a/core/encoding/json/parser.odin +++ b/core/encoding/json/parser.odin @@ -3,7 +3,6 @@ package json import "core:mem" import "core:unicode/utf8" import "core:unicode/utf16" -import "core:runtime" import "core:strconv" Parser :: struct {