authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2019-07-01 23:23:26+12:00
committergravatar for sahnvour@pm.meSahnvour <sahnvour@pm.me> 2019-08-04 12:34:05+02:00
log83dffc70afe4956c56f570ce5c854b17cbd6f218
tree4da3ccfc0de4043296d610e341536329062e0250
parentc9ce43f59fc777055612aeea58db0849390bc204

Add iterative wyhash api

3 files changed, 113 insertions(+), 63 deletions(-)

std/hash.zig+3-1
...@@ -17,6 +17,7 @@ pub const SipHash128 = siphash.SipHash128;...@@ -17,6 +17,7 @@ pub const SipHash128 = siphash.SipHash128;
17pub const murmur = @import("hash/murmur.zig");17pub const murmur = @import("hash/murmur.zig");
18pub const Murmur2_32 = murmur.Murmur2_32;18pub const Murmur2_32 = murmur.Murmur2_32;
1919
20
20pub const Murmur2_64 = murmur.Murmur2_64;21pub const Murmur2_64 = murmur.Murmur2_64;
21pub const Murmur3_32 = murmur.Murmur3_32;22pub const Murmur3_32 = murmur.Murmur3_32;
2223
...@@ -24,7 +25,8 @@ pub const cityhash = @import("hash/cityhash.zig");...@@ -24,7 +25,8 @@ pub const cityhash = @import("hash/cityhash.zig");
24pub const CityHash32 = cityhash.CityHash32;25pub const CityHash32 = cityhash.CityHash32;
25pub const CityHash64 = cityhash.CityHash64;26pub const CityHash64 = cityhash.CityHash64;
2627
27pub const wyhash = @import("hash/wyhash.zig").hash;28const wyhash = @import("hash/wyhash.zig");
29pub const Wyhash = wyhash.Wyhash;
2830
29test "hash" {31test "hash" {
30 _ = @import("hash/adler.zig");32 _ = @import("hash/adler.zig");
std/hash/wyhash.zig+107-59
...@@ -10,7 +10,7 @@ const primes = [_]u64{...@@ -10,7 +10,7 @@ const primes = [_]u64{
10};10};
1111
12fn read_bytes(comptime bytes: u8, data: []const u8) u64 {12fn read_bytes(comptime bytes: u8, data: []const u8) u64 {
13 return mem.readVarInt(u64, data[0..bytes], @import("builtin").endian);13 return mem.readVarInt(u64, data[0..bytes], .Little);
14}14}
1515
16fn read_8bytes_swapped(data: []const u8) u64 {16fn read_8bytes_swapped(data: []const u8) u64 {
...@@ -18,7 +18,7 @@ fn read_8bytes_swapped(data: []const u8) u64 {...@@ -18,7 +18,7 @@ fn read_8bytes_swapped(data: []const u8) u64 {
18}18}
1919
20fn mum(a: u64, b: u64) u64 {20fn mum(a: u64, b: u64) u64 {
21 var r: u128 = @intCast(u128, a) * @intCast(u128, b);21 var r = std.math.mulWide(u64, a, b);
22 r = (r >> 64) ^ r;22 r = (r >> 64) ^ r;
23 return @truncate(u64, r);23 return @truncate(u64, r);
24}24}
...@@ -31,69 +31,117 @@ fn mix1(a: u64, b: u64, seed: u64) u64 {...@@ -31,69 +31,117 @@ fn mix1(a: u64, b: u64, seed: u64) u64 {
31 return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);31 return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);
32}32}
3333
34pub fn hash(key: []const u8, initial_seed: u64) u64 {34pub const Wyhash = struct {
35 var seed = initial_seed;35 seed: u64,
3636
37 var i: usize = 0;37 buf: [32]u8,
38 while (i + 32 <= key.len) : (i += 32) {38 buf_len: usize,
39 seed = mix0(39 msg_len: usize,
40 read_bytes(8, key[i..]),40
41 read_bytes(8, key[i + 8 ..]),41 pub fn init(seed: u64) Wyhash {
42 seed,42 return Wyhash{
43 .seed = seed,
44 .buf = undefined,
45 .buf_len = 0,
46 .msg_len = 0,
47 };
48 }
49
50 fn round(self: *Wyhash, b: []const u8) void {
51 std.debug.assert(b.len == 32);
52
53 self.seed = mix0(
54 read_bytes(8, b[0..]),
55 read_bytes(8, b[8..]),
56 self.seed,
43 ) ^ mix1(57 ) ^ mix1(
44 read_bytes(8, key[i + 16 ..]),58 read_bytes(8, b[16..]),
45 read_bytes(8, key[i + 24 ..]),59 read_bytes(8, b[24..]),
46 seed,60 self.seed,
47 );61 );
48 }62 }
4963
50 const rem_len = @truncate(u5, key.len);64 pub fn update(self: *Wyhash, b: []const u8) void {
51 const rem_key = key[i..];65 var off: usize = 0;
52 seed = switch (rem_len) {66
53 0 => seed,67 // Partial from previous.
54 1 => mix0(read_bytes(1, rem_key), primes[4], seed),68 if (self.buf_len != 0 and self.buf_len + b.len > 32) {
55 2 => mix0(read_bytes(2, rem_key), primes[4], seed),69 off += 32 - self.buf_len;
56 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed),70 mem.copy(u8, self.buf[self.buf_len..], b[0..off]);
57 4 => mix0(read_bytes(4, rem_key), primes[4], seed),71 self.round(self.buf[0..]);
58 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed),72 self.buf_len = 0;
59 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed),73 }
60 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed),74
61 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed),75 // Full middle blocks.
62 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed),76 while (off + 32 <= b.len) : (off += 32) {
63 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed),77 @inlineCall(self.round, b[off .. off + 32]);
64 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed),78 }
65 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed),79
66 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed),80 // Remainder for next pass.
67 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed),81 mem.copy(u8, self.buf[self.buf_len..], b[off..]);
68 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed),82 self.buf_len += @intCast(u8, b[off..].len);
69 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed),83 self.msg_len += b.len;
70 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed),84 }
71 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed),85
72 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed),86 pub fn final(self: *Wyhash) u64 {
73 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed),87 const seed = self.seed;
74 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed),88 const rem_len = @intCast(u5, self.buf_len);
75 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed),89 const rem_key = self.buf[0..self.buf_len];
76 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed),90
77 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed),91 self.seed = switch (rem_len) {
78 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed),92 0 => seed,
79 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed),93 1 => mix0(read_bytes(1, rem_key), primes[4], seed),
80 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed),94 2 => mix0(read_bytes(2, rem_key), primes[4], seed),
81 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed),95 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed),
82 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed),96 4 => mix0(read_bytes(4, rem_key), primes[4], seed),
83 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed),97 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed),
84 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed),98 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed),
85 };99 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed),
86100 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed),
87 return mum(seed ^ key.len, primes[4]);101 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed),
88}102 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed),
103 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed),
104 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed),
105 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed),
106 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed),
107 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed),
108 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed),
109 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed),
110 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed),
111 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed),
112 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed),
113 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed),
114 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed),
115 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed),
116 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed),
117 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed),
118 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed),
119 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed),
120 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed),
121 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed),
122 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed),
123 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed),
124 };
125
126 return mum(self.seed ^ self.msg_len, primes[4]);
127 }
128
129 pub fn hash(seed: u64, input: []const u8) u64 {
130 var c = Wyhash.init(seed);
131 c.update(input);
132 return c.final();
133 }
134};
89135
90test "test vectors" {136test "test vectors" {
91 const expectEqual = std.testing.expectEqual;137 const expectEqual = std.testing.expectEqual;
92 expectEqual(hash("", 0), 0x0);138 const hash = Wyhash.hash;
93 expectEqual(hash("a", 1), 0xbed235177f41d328);139
94 expectEqual(hash("abc", 2), 0xbe348debe59b27c3);140 expectEqual(hash(0, ""), 0x0);
95 expectEqual(hash("message digest", 3), 0x37320f657213a290);141 expectEqual(hash(1, "a"), 0xbed235177f41d328);
96 expectEqual(hash("abcdefghijklmnopqrstuvwxyz", 4), 0xd0b270e1d8a7019c);142 expectEqual(hash(2, "abc"), 0xbe348debe59b27c3);
97 expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 5), 0x602a1894d3bbfe7f);143 expectEqual(hash(3, "message digest"), 0x37320f657213a290);
98 expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890", 6), 0x829e9c148b75970e);144 expectEqual(hash(4, "abcdefghijklmnopqrstuvwxyz"), 0xd0b270e1d8a7019c);
145 expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
146 expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
99}147}
std/hash_map.zig+3-3
...@@ -5,7 +5,7 @@ const testing = std.testing;...@@ -5,7 +5,7 @@ const testing = std.testing;
5const math = std.math;5const math = std.math;
6const mem = std.mem;6const mem = std.mem;
7const meta = std.meta;7const meta = std.meta;
8const wyhash = std.hash.wyhash;8const Wyhash = std.hash.Wyhash;
9const Allocator = mem.Allocator;9const Allocator = mem.Allocator;
10const builtin = @import("builtin");10const builtin = @import("builtin");
1111
...@@ -557,7 +557,7 @@ pub fn autoHash(key: var, seed: u64) u64 {...@@ -557,7 +557,7 @@ pub fn autoHash(key: var, seed: u64) u64 {
557 builtin.TypeId.EnumLiteral,557 builtin.TypeId.EnumLiteral,
558 => @compileError("cannot hash this type"),558 => @compileError("cannot hash this type"),
559559
560 builtin.TypeId.Int => return wyhash(std.mem.asBytes(&key), seed),560 builtin.TypeId.Int => return Wyhash.hash(seed, std.mem.asBytes(&key)),
561561
562 builtin.TypeId.Float => |info| return autoHash(@bitCast(@IntType(false, info.bits), key), seed),562 builtin.TypeId.Float => |info| return autoHash(@bitCast(@IntType(false, info.bits), key), seed),
563563
...@@ -594,7 +594,7 @@ pub fn autoHash(key: var, seed: u64) u64 {...@@ -594,7 +594,7 @@ pub fn autoHash(key: var, seed: u64) u64 {
594 // If there's no unused bits in the child type, we can just hash594 // If there's no unused bits in the child type, we can just hash
595 // this as an array of bytes.595 // this as an array of bytes.
596 if (info.child.bit_count % 8 == 0) {596 if (info.child.bit_count % 8 == 0) {
597 return wyhash(mem.asBytes(&key), seed);597 return Wyhash.hash(seed, mem.asBytes(&key));
598 }598 }
599599
600 // Otherwise, hash every element.600 // Otherwise, hash every element.