Merge pull request #1598 from Kelimion/varint

Add `core:encoding/varint` with LEB128 encoding, decoding and tests.

core/crypto/siphash/siphash.odin

@@ -14,7 +14,6 @@ package siphash
 
 import "core:crypto"
 import "core:crypto/util"
-import "core:mem"
 
 /*
     High level API

core/encoding/varint/doc.odin

@@ -0,0 +1,27 @@
+/*
+	Implementation of the LEB128 variable integer encoding as used by DWARF encoding and DEX files, among others.
+
+	Author of this Odin package: Jeroen van Rijn
+
+	Example:
+		```odin
+		import "core:encoding/varint"
+		import "core:fmt"
+
+		main :: proc() {
+			buf: [varint.LEB128_MAX_BYTES]u8
+
+			value := u128(42)
+
+			encode_size, encode_err := varint.encode_uleb128(buf[:], value)
+			assert(encode_size == 1 && encode_err == .None)
+
+			fmt.println(buf[:encode_size])
+
+			decoded_val, decode_size, decode_err := varint.decode_uleb128(buf[:encode_size])
+			assert(decoded_val == value && decode_size == encode_size && decode_err == .None)
+		}
+		```
+
+*/
+package varint

core/encoding/varint/leb128.odin

@@ -0,0 +1,139 @@
+/*
+	Copyright 2022 Jeroen van Rijn <nom@duclavier.com>.
+	Made available under Odin's BSD-3 license.
+
+	List of contributors:
+		Jeroen van Rijn: Initial implementation.
+*/
+
+// package varint implements variable length integer encoding and decoding using
+// the LEB128 format as used by DWARF debug info, Android .dex and other file formats.
+package varint
+
+import "core:fmt"
+
+// In theory we should use the bigint package. In practice, varints bigger than this indicate a corrupted file.
+// Instead we'll set limits on the values we'll encode/decode
+// 18 * 7 bits = 126, which means that a possible 19th byte may at most be `0b0000_0011`.
+LEB128_MAX_BYTES    :: 19
+
+Error :: enum {
+	None             = 0,
+	Buffer_Too_Small = 1,
+	Value_Too_Large  = 2,
+}
+
+// Decode a slice of bytes encoding an unsigned LEB128 integer into value and number of bytes used.
+// Returns `size` == 0 for an invalid value, empty slice, or a varint > 18 bytes.
+decode_uleb128 :: proc(buf: []u8) -> (val: u128, size: int, err: Error) {
+	more := true
+
+	for v, i in buf {
+		size = i + 1
+
+		// 18 * 7 bits = 126, which means that a possible 19th byte may at most be 0b0000_0011.
+		if size > LEB128_MAX_BYTES || size == LEB128_MAX_BYTES && v > 0b0000_0011 {
+			return 0, 0, .Value_Too_Large
+		}
+
+		val |= u128(v & 0x7f) << uint(i * 7)
+
+		if v < 128 {
+			more = false
+			break
+		}
+	}
+
+	// If the buffer runs out before the number ends, return an error.
+	if more {
+		return 0, 0, .Buffer_Too_Small
+	}
+	return
+}
+
+// Decode a slice of bytes encoding a signed LEB128 integer into value and number of bytes used.
+// Returns `size` == 0 for an invalid value, empty slice, or a varint > 18 bytes.
+decode_ileb128 :: proc(buf: []u8) -> (val: i128, size: int, err: Error) {
+	shift: uint
+
+	if len(buf) == 0 {
+		return 0, 0, .Buffer_Too_Small
+	}
+
+	for v in buf {
+		size += 1
+
+		// 18 * 7 bits = 126, which including sign means we can have a 19th byte.
+		if size > LEB128_MAX_BYTES || size == LEB128_MAX_BYTES && v > 0x7f {
+			return 0, 0, .Value_Too_Large
+		}
+
+		val |= i128(v & 0x7f) << shift
+		shift += 7
+
+		if v < 128 { break }
+	}
+
+	if buf[size - 1] & 0x40 == 0x40 {
+		val |= max(i128) << shift
+	}
+	return
+}
+
+// Encode `val` into `buf` as an unsigned LEB128 encoded series of bytes.
+// `buf` must be appropriately sized.
+encode_uleb128 :: proc(buf: []u8, val: u128) -> (size: int, err: Error) {
+	val := val
+
+	for {
+		size += 1
+
+		if size > len(buf) {
+			fmt.println(val, buf[:size - 1])
+			return 0, .Buffer_Too_Small
+		}
+
+		low := val & 0x7f
+		val >>= 7
+
+		if val > 0 {
+			low |= 0x80 // more bytes to follow
+		}
+		buf[size - 1] = u8(low)
+
+		if val == 0 { break }
+	}
+	return
+}
+
+@(private)
+SIGN_MASK :: (i128(1) << 121) // sign extend mask
+
+// Encode `val` into `buf` as a signed LEB128 encoded series of bytes.
+// `buf` must be appropriately sized.
+encode_ileb128 :: proc(buf: []u8, val: i128) -> (size: int, err: Error) {
+	val      := val
+	more     := true
+
+	for more {
+		size += 1
+
+		if size > len(buf) {
+			return 0, .Buffer_Too_Small
+		}
+
+		low := val & 0x7f
+		val >>= 7
+
+		low = (low ~ SIGN_MASK) - SIGN_MASK
+
+		if (val == 0 && low & 0x40 != 0x40) || (val == -1 && low & 0x40 == 0x40) {
+			more = false
+		} else {
+			low |= 0x80
+		}
+
+		buf[size - 1] = u8(low)
+	}
+	return
+}

tests/core/Makefile

@@ -1,7 +1,7 @@
 ODIN=../../odin
 PYTHON=$(shell which python3)
 
-all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test
+all: download_test_assets image_test compress_test strings_test hash_test crypto_test noise_test encoding_test
 
 download_test_assets:
 	$(PYTHON) download_assets.py
@@ -22,4 +22,8 @@ crypto_test:
 	$(ODIN) run crypto -out=crypto_hash -o:speed -no-bounds-check
 
 noise_test:
-	$(ODIN) run math/noise -out=test_noise
+	$(ODIN) run math/noise -out=test_noise
+
+encoding_test:
+	$(ODIN) run encoding/json -out=test_json
+	$(ODIN) run encoding/varint -out=test_varint

tests/core/build.bat

@@ -35,7 +35,8 @@ echo ---
 echo ---
 echo Running core:encoding tests
 echo ---
-%PATH_TO_ODIN% run encoding %COMMON%
+%PATH_TO_ODIN% run encoding/json %COMMON%
+%PATH_TO_ODIN% run encoding/varint %COMMON%
 
 echo ---
 echo Running core:math/noise tests

tests/core/compress/test_core_compress.odin

@@ -30,14 +30,12 @@ when ODIN_TEST {
 	log     :: testing.log
 } else {
 	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-		fmt.printf("[%v] ", loc)
 		TEST_count += 1
 		if !condition {
 			TEST_fail += 1
-			fmt.println(message)
+			fmt.printf("[%v] %v\n", loc, message)
 			return
 		}
-		fmt.println(" PASS")
 	}
 	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
 		fmt.printf("[%v] ", loc)

tests/core/encoding/json/test_core_json.odin

@@ -9,32 +9,30 @@ TEST_count := 0
 TEST_fail  := 0
 
 when ODIN_TEST {
-    expect  :: testing.expect
+	expect  :: testing.expect
-    log     :: testing.log
+	log     :: testing.log
 } else {
-    expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-        fmt.printf("[%v] ", loc)
+		TEST_count += 1
-        TEST_count += 1
+		if !condition {
-        if !condition {
+			TEST_fail += 1
-            TEST_fail += 1
+			fmt.printf("[%v] %v\n", loc, message)
-            fmt.println(message)
+			return
-            return
+		}
-        }
+	}
-        fmt.println(" PASS")
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
-    }
+		fmt.printf("[%v] ", loc)
-    log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("log: %v\n", v)
-        fmt.printf("[%v] ", loc)
+	}
-        fmt.printf("log: %v\n", v)
-    }
 }
 
 main :: proc() {
-    t := testing.T{}
+	t := testing.T{}
-	
+
 	parse_json(&t)
 	marshal_json(&t)
 
-    fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
+	fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
 	if TEST_fail > 0 {
 		os.exit(1)
 	}

tests/core/encoding/varint/test_core_varint.odin

@@ -0,0 +1,156 @@
+package test_core_varint
+
+import "core:encoding/varint"
+import "core:testing"
+import "core:fmt"
+import "core:os"
+import "core:slice"
+import "core:math/rand"
+
+TEST_count := 0
+TEST_fail  := 0
+
+RANDOM_TESTS :: 100
+
+when ODIN_TEST {
+	expect  :: testing.expect
+	log     :: testing.log
+} else {
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+		TEST_count += 1
+		if !condition {
+			TEST_fail += 1
+			fmt.printf("[%v] %v\n", loc, message)
+			return
+		}
+	}
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("[%v] ", loc)
+		fmt.printf("log: %v\n", v)
+	}
+}
+
+main :: proc() {
+	t := testing.T{}
+
+	test_leb128(&t)
+
+	fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
+	if TEST_fail > 0 {
+		os.exit(1)
+	}
+}
+
+@(test)
+test_leb128 :: proc(t: ^testing.T) {
+	buf: [varint.LEB128_MAX_BYTES]u8
+
+	for vector in ULEB_Vectors {
+		val, size, err := varint.decode_uleb128(vector.encoded)
+
+		msg := fmt.tprintf("Expected %02x to decode to %v consuming %v bytes, got %v and %v", vector.encoded, vector.value, vector.size, val, size)
+		expect(t, size == vector.size && val == vector.value, msg)
+
+		msg  = fmt.tprintf("Expected decoder to return error %v, got %v", vector.error, err)
+		expect(t, err == vector.error, msg)
+
+		if err == .None { // Try to roundtrip
+			size, err = varint.encode_uleb128(buf[:], vector.value)
+
+			msg = fmt.tprintf("Expected %v to encode to %02x, got %02x", vector.value, vector.encoded, buf[:size])
+			expect(t, size == vector.size && slice.simple_equal(vector.encoded, buf[:size]), msg)
+		}
+	}
+
+	for vector in ILEB_Vectors {
+		val, size, err := varint.decode_ileb128(vector.encoded)
+
+		msg := fmt.tprintf("Expected %02x to decode to %v consuming %v bytes, got %v and %v", vector.encoded, vector.value, vector.size, val, size)
+		expect(t, size == vector.size && val == vector.value, msg)
+
+		msg  = fmt.tprintf("Expected decoder to return error %v, got %v", vector.error, err)
+		expect(t, err == vector.error, msg)
+
+		if err == .None { // Try to roundtrip
+			size, err = varint.encode_ileb128(buf[:], vector.value)
+
+			msg = fmt.tprintf("Expected %v to encode to %02x, got %02x", vector.value, vector.encoded, buf[:size])
+			expect(t, size == vector.size && slice.simple_equal(vector.encoded, buf[:size]), msg)
+		}
+	}
+
+	for num_bytes in 1..uint(16) {
+		for _ in 0..RANDOM_TESTS {
+			unsigned, signed := get_random(num_bytes)
+
+			{
+				encode_size, encode_err := varint.encode_uleb128(buf[:], unsigned)
+				msg := fmt.tprintf("%v failed to encode as an unsigned LEB128 value, got %v", unsigned, encode_err)
+				expect(t, encode_err == .None, msg)
+
+				decoded, decode_size, decode_err := varint.decode_uleb128(buf[:])
+				msg = fmt.tprintf("Expected %02x to decode as %v, got %v", buf[:encode_size], unsigned, decoded)
+				expect(t, decode_err == .None && decode_size == encode_size && decoded == unsigned, msg)
+			}
+
+			{
+				encode_size, encode_err := varint.encode_ileb128(buf[:], signed)
+				msg := fmt.tprintf("%v failed to encode as a signed LEB128 value, got %v", signed, encode_err)
+				expect(t, encode_err == .None, msg)
+
+				decoded, decode_size, decode_err := varint.decode_ileb128(buf[:])
+				msg = fmt.tprintf("Expected %02x to decode as %v, got %v, err: %v", buf[:encode_size], signed, decoded, decode_err)
+				expect(t, decode_err == .None && decode_size == encode_size && decoded == signed, msg)
+			}
+		}
+	}
+}
+
+get_random :: proc(byte_count: uint) -> (u: u128, i: i128) {
+	assert(byte_count >= 0 && byte_count <= size_of(u128))
+
+	for _ in 1..byte_count {
+		u <<= 8
+		u |= u128(rand.uint32() & 0xff)
+	}
+
+	bias := i128(1 << (byte_count * 7)) - 1
+	i     = i128(u) - bias
+
+	return
+}
+
+ULEB_Test_Vector :: struct {
+	encoded: []u8,
+	value:   u128,
+	size:    int,
+	error:   varint.Error,
+}
+
+ULEB_Vectors :: []ULEB_Test_Vector{
+	{ []u8{0x00},             0,         1, .None },
+	{ []u8{0x7f},             127,       1, .None },
+	{ []u8{0xE5, 0x8E, 0x26}, 624485,    3, .None },
+	{ []u8{0x80},             0,         0, .Buffer_Too_Small },
+	{ []u8{},                 0,         0, .Buffer_Too_Small },
+
+	{ []u8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x03}, max(u128), 19, .None },
+}
+
+ILEB_Test_Vector :: struct {
+	encoded: []u8,
+	value:   i128,
+	size:    int,
+	error:   varint.Error,
+}
+
+ILEB_Vectors :: []ILEB_Test_Vector{
+	{ []u8{0x00},             0,       1, .None },
+	{ []u8{0x3f},             63,      1, .None },
+	{ []u8{0x40},             -64,     1, .None },
+	{ []u8{0xC0, 0xBB, 0x78}, -123456, 3, .None },
+	{ []u8{},                 0,       0, .Buffer_Too_Small },
+
+	{ []u8{0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x7e}, min(i128), 19, .None },
+	{ []u8{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x01}, max(i128), 19, .None },
+}

tests/core/hash/test_core_hash.odin

@@ -15,14 +15,12 @@ when ODIN_TEST {
 	log     :: testing.log
 } else {
 	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-		fmt.printf("[%v] ", loc)
 		TEST_count += 1
 		if !condition {
 			TEST_fail += 1
-			fmt.println(" FAIL:", message)
+			fmt.printf("[%v] %v\n", loc, message)
 			return
 		}
-		fmt.println(" PASS")
 	}
 	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
 		fmt.printf("[%v] ", loc)

tests/core/image/test_core_image.odin

@@ -32,23 +32,21 @@ TEST_count := 0
 TEST_fail  := 0
 
 when ODIN_TEST {
-    expect  :: testing.expect
+	expect  :: testing.expect
-    log     :: testing.log
+	log     :: testing.log
 } else {
-    expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-        fmt.printf("[%v] ", loc)
+		TEST_count += 1
-        TEST_count += 1
+		if !condition {
-        if !condition {
+			TEST_fail += 1
-            TEST_fail += 1
+			fmt.printf("[%v] %v\n", loc, message)
-            fmt.println(message)
+			return
-            return
+		}
-        }
+	}
-        fmt.println(" PASS")
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
-    }
+		fmt.printf("[%v] ", loc)
-    log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("log: %v\n", v)
-        fmt.printf("[%v] ", loc)
+	}
-        fmt.printf("log: %v\n", v)
-    }
 }
 I_Error :: image.Error
 

tests/core/math/noise/test_core_math_noise.odin

@@ -13,23 +13,21 @@ V3 :: noise.Vec3
 V4 :: noise.Vec4
 
 when ODIN_TEST {
-    expect  :: testing.expect
+	expect  :: testing.expect
-    log     :: testing.log
+	log     :: testing.log
 } else {
-    expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-        fmt.printf("[%v] ", loc)
+		TEST_count += 1
-        TEST_count += 1
+		if !condition {
-        if !condition {
+			TEST_fail += 1
-            TEST_fail += 1
+			fmt.printf("[%v] %v\n", loc, message)
-            fmt.println(message)
+			return
-            return
+		}
-        }
+	}
-        fmt.println(" PASS")
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
-    }
+		fmt.printf("[%v] ", loc)
-    log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("log: %v\n", v)
-        fmt.printf("[%v] ", loc)
+	}
-        fmt.printf("log: %v\n", v)
-    }
 }
 
 main :: proc() {

tests/core/odin/test_parser.odin

@@ -10,34 +10,31 @@ TEST_count := 0
 TEST_fail  := 0
 
 when ODIN_TEST {
-    expect  :: testing.expect
+	expect  :: testing.expect
-    log     :: testing.log
+	log     :: testing.log
 } else {
-    expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-        fmt.printf("[%v] ", loc)
+		TEST_count += 1
-        TEST_count += 1
+		if !condition {
-        if !condition {
+			TEST_fail += 1
-            TEST_fail += 1
+			fmt.printf("[%v] %v\n", loc, message)
-            fmt.println(message)
+			return
-            return
+		}
-        }
+	}
-        fmt.println(" PASS")
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
-    }
+		fmt.printf("[%v] ", loc)
-    log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("log: %v\n", v)
-        fmt.printf("[%v] ", loc)
+	}
-        fmt.printf("log: %v\n", v)
-    }
 }
 
-
 main :: proc() {
-    t := testing.T{}
+	t := testing.T{}
-    test_parse_demo(&t)
+	test_parse_demo(&t)
 
-    fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
+	fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
-    if TEST_fail > 0 {
+	if TEST_fail > 0 {
-        os.exit(1)
+		os.exit(1)
-    }
+	}
 }
 
 
@@ -50,4 +47,4 @@ test_parse_demo :: proc(t: ^testing.T) {
 	for key, value in pkg.files {
 		expect(t, value.syntax_error_count == 0, fmt.tprintf("%v should contain zero errors", key))
 	}
-}
+}

tests/core/strings/test_core_strings.odin

@@ -9,59 +9,57 @@ TEST_count := 0
 TEST_fail  := 0
 
 when ODIN_TEST {
-    expect  :: testing.expect
+	expect  :: testing.expect
-    log     :: testing.log
+	log     :: testing.log
 } else {
-    expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
+	expect  :: proc(t: ^testing.T, condition: bool, message: string, loc := #caller_location) {
-        fmt.printf("[%v] ", loc)
+		TEST_count += 1
-        TEST_count += 1
+		if !condition {
-        if !condition {
+			TEST_fail += 1
-            TEST_fail += 1
+			fmt.printf("[%v] %v\n", loc, message)
-            fmt.println(message)
+			return
-            return
+		}
-        }
+	}
-        fmt.println(" PASS")
+	log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
-    }
+		fmt.printf("[%v] ", loc)
-    log     :: proc(t: ^testing.T, v: any, loc := #caller_location) {
+		fmt.printf("log: %v\n", v)
-        fmt.printf("[%v] ", loc)
+	}
-        fmt.printf("log: %v\n", v)
-    }
 }
 
 main :: proc() {
-    t := testing.T{}
+	t := testing.T{}
-    test_index_any_small_string_not_found(&t)
+	test_index_any_small_string_not_found(&t)
-    test_index_any_larger_string_not_found(&t)
+	test_index_any_larger_string_not_found(&t)
-    test_index_any_small_string_found(&t)
+	test_index_any_small_string_found(&t)
-    test_index_any_larger_string_found(&t)
+	test_index_any_larger_string_found(&t)
 
-    fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
+	fmt.printf("%v/%v tests successful.\n", TEST_count - TEST_fail, TEST_count)
-    if TEST_fail > 0 {
+	if TEST_fail > 0 {
-        os.exit(1)
+		os.exit(1)
-    }
+	}
 }
 
 @test
 test_index_any_small_string_not_found :: proc(t: ^testing.T) {
-    index := strings.index_any(".", "/:\"")
+	index := strings.index_any(".", "/:\"")
-    log(t, index)
+	log(t, index)
-    expect(t, index == -1, "index_any should be negative")
+	expect(t, index == -1, "index_any should be negative")
 }
 
 @test
 test_index_any_larger_string_not_found :: proc(t: ^testing.T) {
-    index := strings.index_any("aaaaaaaa.aaaaaaaa", "/:\"")
+	index := strings.index_any("aaaaaaaa.aaaaaaaa", "/:\"")
-    expect(t, index == -1, "index_any should be negative")
+	expect(t, index == -1, "index_any should be negative")
 }
 
 @test
 test_index_any_small_string_found :: proc(t: ^testing.T) {
-    index := strings.index_any(".", "/:.\"")
+	index := strings.index_any(".", "/:.\"")
-    expect(t, index == 0, "index_any should be 0")
+	expect(t, index == 0, "index_any should be 0")
 }
 
 @test
 test_index_any_larger_string_found :: proc(t: ^testing.T) {
-    index := strings.index_any("aaaaaaaa:aaaaaaaa", "/:\"")
+	index := strings.index_any("aaaaaaaa:aaaaaaaa", "/:\"")
-    expect(t, index == 8, "index_any should be 8")
+	expect(t, index == 8, "index_any should be 8")
 }