diff --git a/core/hash/murmur3.odin b/core/hash/murmur3.odin new file mode 100644 index 000000000..a6fc42087 --- /dev/null +++ b/core/hash/murmur3.odin @@ -0,0 +1,293 @@ +package hash + +rotl32 :: #force_inline proc "contextless" (x: u32, r: u8) -> u32 { + return (x << r) | (x >> (32 - r)) +} + +rotl64 :: #force_inline proc "contextless" (x: u64, r: u8) -> u64 { + return (x << r) | (x >> (64 - r)) +} + +fmix32 :: #force_inline proc "contextless" (h: u32) -> u32 { + h := h + h ~= (h >> 16) + h *= 0x85ebca6b + h ~= (h >> 13) + h *= 0xc2b2ae35 + h ~= (h >> 16) + return h +} + +fmix64 :: #force_inline proc "contextless" (h: u64) -> u64 { + h := h + h ~= h >> 33 + h *= 0xff51afd7ed558ccd + h ~= h >> 33 + h *= 0xc4ceb9fe1a85ec53 + h ~= h >> 33 + return h +} + +// See https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp#L94 +@(optimization_mode="favor_size") +murmur3_x86_32 :: proc "contextless" (data: []u8, seed: u32 = 0) -> u32 #no_bounds_check { + len : uint = len(data) + nblocks : uint = len / 4 + h1 : u32 = seed + + c1 :: 0xcc9e2d51 + c2 :: 0x1b873593 + + // BODY + for i : uint = 0; i < nblocks; i += 1 { + k1 : u32 = (transmute([]u32)data)[i] + + k1 *= c1 + k1 = rotl32(k1, 15) + k1 *= c2 + + h1 ~= k1 + h1 = rotl32(h1, 13) + h1 = h1 * 5 + 0xe6546b64 + } + + // TAIL + k1 : u32 = 0 + switch(len & 3) { + case 3: k1 ~= u32(data[nblocks*4+2]) << 16; fallthrough + case 2: k1 ~= u32(data[nblocks*4+1]) << 8; fallthrough + case 1: + k1 ~= u32(data[nblocks*4]) + k1 *= c1 + k1 = rotl32(k1, 15) + k1 *= c2 + h1 ~= k1 + } + + // END + h1 ~= u32(len) + + h1 = fmix32(h1) + return h1 +} + +// See https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp#L150 +@(optimization_mode="favor_size") +murmur3_x86_128 :: proc "contextless" (data: []u8, seed: u32 = 0) -> u128 #no_bounds_check { + len : uint = len(data) + nblocks : uint = len / 16 + + h1 : u32 = seed + h2 : u32 = seed + h3 : u32 = seed + h4 : u32 = seed + + c1 :: 0x239b961b + c2 :: 0xab0e9789 + c3 :: 0x38b34ae5 + c4 :: 0xa1e38b93 + + // BODY + for i : uint = 0; i < nblocks; i += 1 { + k1 : u32 = (transmute([]u32)data)[4*i+0] + k2 : u32 = (transmute([]u32)data)[4*i+1] + k3 : u32 = (transmute([]u32)data)[4*i+2] + k4 : u32 = (transmute([]u32)data)[4*i+3] + + k1 *= c1 + k1 = rotl32(k1, 15) + k1 *= c2 + h1 ~= k1 + + h1 = rotl32(h1, 19) + h1 += h2 + h1 = h1 * 5 + 0x561ccd1b + + + k2 *= c2 + k2 = rotl32(k2, 16) + k2 *= c3 + h2 ~= k2 + + h2 = rotl32(h2, 17) + h2 += h3 + h2 = h2 * 5 + 0x0bcaa747 + + + k3 *= c3 + k3 = rotl32(k3, 17) + k3 *= c4 + h3 ~= k3 + + h3 = rotl32(h3, 15) + h3 += h4 + h3 = h3 * 5 + 0x96cd1c35 + + + k4 *= c4 + k4 = rotl32(k4, 18) + k4 *= c1 + h4 ~= k4 + + h4 = rotl32(h4, 13) + h4 += h1 + h4 = h4 * 5 + 0x32ac3b17 + } + + // TAIL + k1 : u32 = 0 + k2 : u32 = 0 + k3 : u32 = 0 + k4 : u32 = 0 + switch(len & 15) { + case 15: k4 ~= u32(data[nblocks*16+14]) << 16; fallthrough + case 14: k4 ~= u32(data[nblocks*16+13]) << 8; fallthrough + case 13: + k4 ~= u32(data[nblocks*16+12]) + k4 *= c4 + k4 = rotl32(k4, 18) + k4 *= c1 + h4 ~= k4 + fallthrough + case 12: k3 ~= u32(data[nblocks*16+11]) << 24; fallthrough + case 11: k3 ~= u32(data[nblocks*16+10]) << 16; fallthrough + case 10: k3 ~= u32(data[nblocks*16+9]) << 8; fallthrough + case 9: + k3 ~= u32(data[nblocks*16+8]) + k3 *= c3 + k3 = rotl32(k3, 17) + k3 *= c4 + h3 ~= k3 + fallthrough + case 8: k2 ~= u32(data[nblocks*16+7]) << 24; fallthrough + case 7: k2 ~= u32(data[nblocks*16+6]) << 16; fallthrough + case 6: k2 ~= u32(data[nblocks*16+5]) << 8; fallthrough + case 5: + k2 ~= u32(data[nblocks*16+4]) + k2 *= c2 + k2 = rotl32(k2, 16) + k2 *= c3 + h2 ~= k2 + fallthrough + case 4: k1 ~= u32(data[nblocks*16+3]) << 24; fallthrough + case 3: k1 ~= u32(data[nblocks*16+2]) << 16; fallthrough + case 2: k1 ~= u32(data[nblocks*16+1]) << 8; fallthrough + case 1: + k1 ~= u32(data[nblocks*16+0]) + k1 *= c1 + k1 = rotl32(k1, 15) + k1 *= c2 + h1 ~= k1 + } + + // END + h1 ~= u32(len) + h2 ~= u32(len) + h3 ~= u32(len) + h4 ~= u32(len) + + h1 += h2 + h1 += h3 + h1 += h4 + h2 += h1 + h3 += h1 + h4 += h1 + + h1 = fmix32(h1) + h2 = fmix32(h2) + h3 = fmix32(h3) + h4 = fmix32(h4) + + h1 += h2 + h1 += h3 + h1 += h4 + h2 += h1 + h3 += h1 + h4 += h1 + + return u128(h1) << 96 | u128(h2) << 64 | u128(h3) << 32 | u128(h4) +} + +// See https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp#L255 +@(optimization_mode="favor_size") +murmur3_x64_128 :: proc "contextless" (data: []u8, seed: u32 = 0) -> u128 #no_bounds_check { + len : uint = len(data) + nblocks : uint = len / 16 + + h1 : u64 = u64(seed) + h2 : u64 = u64(seed) + + c1 :: 0x87c37b91114253d5 + c2 :: 0x4cf5ad432745937f + + // BODY + for i : uint = 0; i < nblocks; i += 1 { + k1 : u64 = (transmute([]u64)data)[2*i+0] + k2 : u64 = (transmute([]u64)data)[2*i+1] + + k1 *= c1 + k1 = rotl64(k1, 31) + k1 *= c2 + h1 ~= k1 + + h1 = rotl64(h1, 27) + h1 += h2 + h1 = h1 * 5 + 0x52dce729 + + + k2 *= c2 + k2 = rotl64(k2, 33) + k2 *= c1 + h2 ~= k2 + + h2 = rotl64(h2, 31) + h2 += h1 + h2 = h2 * 5 + 0x38495ab5 + } + + // TAIL + k1 : u64 = 0 + k2 : u64 = 0 + switch(len & 15) { + case 15: k2 ~= u64(data[nblocks*16+14]) << 48; fallthrough + case 14: k2 ~= u64(data[nblocks*16+13]) << 40; fallthrough + case 13: k2 ~= u64(data[nblocks*16+12]) << 32; fallthrough + case 12: k2 ~= u64(data[nblocks*16+11]) << 24; fallthrough + case 11: k2 ~= u64(data[nblocks*16+10]) << 16; fallthrough + case 10: k2 ~= u64(data[nblocks*16+9]) << 8; fallthrough + case 9: + k2 ~= u64(data[nblocks*16+8]) + k2 *= c2 + k2 = rotl64(k2, 33) + k2 *= c1 + h2 ~= k2 + fallthrough + case 8: k1 ~= u64(data[nblocks*16+7]) << 56; fallthrough + case 7: k1 ~= u64(data[nblocks*16+6]) << 48; fallthrough + case 6: k1 ~= u64(data[nblocks*16+5]) << 40; fallthrough + case 5: k1 ~= u64(data[nblocks*16+4]) << 32; fallthrough + case 4: k1 ~= u64(data[nblocks*16+3]) << 24; fallthrough + case 3: k1 ~= u64(data[nblocks*16+2]) << 16; fallthrough + case 2: k1 ~= u64(data[nblocks*16+1]) << 8; fallthrough + case 1: + k1 ~= u64(data[nblocks*16+0]) + k1 *= c1 + k1 = rotl64(k1, 31) + k1 *= c2 + h1 ~= k1 + } + // END + h1 ~= u64(len) + h2 ~= u64(len) + + h1 += h2 + h2 += h1 + + h1 = fmix64(h1) + h2 = fmix64(h2) + + h1 += h2 + h2 += h1 + + return u128(h1) << 64 | u128(h2) +} diff --git a/tests/core/hash/test_core_hash.odin b/tests/core/hash/test_core_hash.odin index 30ee911a8..e1e0b4738 100644 --- a/tests/core/hash/test_core_hash.odin +++ b/tests/core/hash/test_core_hash.odin @@ -282,4 +282,66 @@ test_crc16_ccitt_0x1021_vectors :: proc(t: ^testing.T) { crc16 := hash.crc16_ccitt_0x1021(b) testing.expectf(t, crc16 == vector.h, "\n\t[CCITT CRC-16({0:q})] Expected: 0x{1:4x}, got: 0x{2:4x}", vector.s, vector.h, crc16) } -} \ No newline at end of file +} + +@test +test_murmur3_x86_32 :: proc(t: ^testing.T) { + vectors :: [?]struct{s: string, seed, h: u32}{ + {"", 0x0, 0x00000000}, + {"", 0x1, 0x514e28b7}, + {"", 0xffffffff, 0x81f16f39}, + {"test", 0x0, 0xba6bd213}, + {"test", 0x9747b28c, 0x704b81dc}, + {"Hello, world!", 0x0, 0xc0363e43}, + {"Hello, world!", 0x9747b28c, 0x24884cba}, + {"The quick brown fox jumps over the lazy dog", 0x0, 0x2e4ff723}, + {"The quick brown fox jumps over the lazy dog", 0x9747b28c, 0x2fa826cd}, + } + for vector in vectors { + b := transmute([]u8)vector.s + mm3 := hash.murmur3_x86_32(b, vector.seed) + testing.expectf(t, mm3 == vector.h, "\n\t[CCITT MURMUR3-X86-32(%v)] Expected: 0x%08x, got: 0x%08x", vector.s, vector.h, mm3) + } +} + +@test +test_murmur3_x86_128 :: proc(t: ^testing.T) { + vectors :: [?]struct{s: string, seed: u32, h: u128}{ + {"", 0x0, 0x0 }, + {"", 0x1, 0x88c4adec54d201b954d201b954d201b9}, + {"", 0xffffffff, 0x051e08a9989d49f7989d49f7989d49f7}, + {"test", 0x0, 0x6f02ef30550c7d68550c7d68550c7d68}, + {"test", 0x9747b28c, 0x0bcc5d99d98130f9d98130f9d98130f9}, + {"Hello, world!", 0x0, 0x26acdba7f0638dfc402b42630afdd4c3}, + {"Hello, world!", 0x9747b28c, 0x756d5460bb872216b7d48b7c53c8c636}, + {"The quick brown fox jumps over the lazy dog", 0x0, 0x2f1583c3ecee2c675d7bf66ce5e91d2c}, + {"The quick brown fox jumps over the lazy dog", 0x9747b28c, 0x8ad4d55e4cb861718ea73a9ccdb6793e}, + } + + for vector in vectors { + b := transmute([]u8)vector.s + mm3 := hash.murmur3_x86_128(b, vector.seed) + testing.expectf(t, mm3 == vector.h, "\n\t[CCITT MURMUR3-X86-128(%v)] Expected: 0x%32x, got: 0x%32x", vector.s, vector.h, mm3) + } +} + +@test +test_murmur3_x64_128 :: proc(t: ^testing.T) { + vectors :: [?]struct{s: string, seed: u32, h: u128}{ + {"", 0x0, 0x0 }, + {"", 0x1, 0x4610abe56eff5cb551622daa78f83583}, + {"", 0xffffffff, 0x6af1df4d9d3bc9ec857421121ee6446b}, + {"test", 0x0, 0xac7d28cc74bde19d9a128231f9bd4d82}, + {"test", 0x9747b28c, 0xa066a6b76c55301864a6e65666d07937}, + {"Hello, world!", 0x0, 0xf1512dd1d2d665df2c326650a8f3c564}, + {"Hello, world!", 0x9747b28c, 0xedc485d662a8392ef85e7e7631d576ba}, + {"The quick brown fox jumps over the lazy dog", 0x0, 0xe34bbc7bbc071b6c7a433ca9c49a9347}, + {"The quick brown fox jumps over the lazy dog", 0x9747b28c, 0x738a7f3bd2633121f94573727ec016e5}, + } + + for vector in vectors { + b := transmute([]u8)vector.s + mm3 := hash.murmur3_x64_128(b, vector.seed) + testing.expectf(t, mm3 == vector.h, "\n\t[CCITT MURMUR3-X64-128(%v)] Expected: 0x%32x, got: 0x%32x", vector.s, vector.h, mm3) + } +}