authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-01-21 02:49:14+11:00
committergravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-02-20 09:09:05+11:00
log19984d8751894608dc3cd2271c8e2e40c476d5df
tree7bf3bba9d26f94c43e15b3baebc507fd8e73e6a9
parent476bdc8b0b02cbd09f6a856aa7dc548dea565109

std.hash: add XxHash64 and XxHash32

2 files changed, 273 insertions(+), 0 deletions(-)

lib/std/hash.zig+5
...@@ -32,6 +32,10 @@ pub const CityHash64 = cityhash.CityHash64;...@@ -32,6 +32,10 @@ pub const CityHash64 = cityhash.CityHash64;
32const wyhash = @import("hash/wyhash.zig");32const wyhash = @import("hash/wyhash.zig");
33pub const Wyhash = wyhash.Wyhash;33pub const Wyhash = wyhash.Wyhash;
3434
35const xxhash = @import("hash/xxhash.zig");
36pub const XxHash64 = xxhash.XxHash64;
37pub const XxHash32 = xxhash.XxHash32;
38
35test "hash" {39test "hash" {
36 _ = adler;40 _ = adler;
37 _ = auto_hash;41 _ = auto_hash;
...@@ -40,4 +44,5 @@ test "hash" {...@@ -40,4 +44,5 @@ test "hash" {
40 _ = murmur;44 _ = murmur;
41 _ = cityhash;45 _ = cityhash;
42 _ = wyhash;46 _ = wyhash;
47 _ = xxhash;
43}48}
lib/std/hash/xxhash.zig created+268
...@@ -0,0 +1,268 @@
1const std = @import("std");
2const mem = std.mem;
3const expectEqual = std.testing.expectEqual;
4
5inline fn rotl(comptime count: comptime_int, value: anytype) @TypeOf(value) {
6 return (value << count) | (value >> (@bitSizeOf(@TypeOf(value)) - count));
7}
8
9pub const XxHash64 = struct {
10 acc1: u64,
11 acc2: u64,
12 acc3: u64,
13 acc4: u64,
14
15 seed: u64,
16 buf: [32]u8,
17 buf_len: usize,
18 byte_count: usize,
19
20 const prime_1 = 0x9E3779B185EBCA87; // 0b1001111000110111011110011011000110000101111010111100101010000111
21 const prime_2 = 0xC2B2AE3D27D4EB4F; // 0b1100001010110010101011100011110100100111110101001110101101001111
22 const prime_3 = 0x165667B19E3779F9; // 0b0001011001010110011001111011000110011110001101110111100111111001
23 const prime_4 = 0x85EBCA77C2B2AE63; // 0b1000010111101011110010100111011111000010101100101010111001100011
24 const prime_5 = 0x27D4EB2F165667C5; // 0b0010011111010100111010110010111100010110010101100110011111000101
25
26 pub fn init(seed: u64) XxHash64 {
27 return XxHash64{
28 .seed = seed,
29 .acc1 = seed +% prime_1 +% prime_2,
30 .acc2 = seed +% prime_2,
31 .acc3 = seed,
32 .acc4 = seed -% prime_1,
33 .buf = undefined,
34 .buf_len = 0,
35 .byte_count = 0,
36 };
37 }
38
39 pub fn update(self: *XxHash64, input: []const u8) void {
40 if (input.len < 32 - self.buf_len) {
41 mem.copy(u8, self.buf[self.buf_len..], input);
42 self.buf_len += input.len;
43 return;
44 }
45
46 var i: usize = 0;
47
48 if (self.buf_len > 0) {
49 i = 32 - self.buf_len;
50 mem.copy(u8, self.buf[self.buf_len..], input[0..i]);
51 self.processStripe(&self.buf);
52 self.buf_len = 0;
53 }
54
55 while (i + 32 <= input.len) : (i += 32) {
56 self.processStripe(input[i..][0..32]);
57 }
58
59 const remaining_bytes = input[i..];
60 mem.copy(u8, &self.buf, remaining_bytes);
61 self.buf_len = remaining_bytes.len;
62 }
63
64 inline fn processStripe(self: *XxHash64, buf: *const [32]u8) void {
65 self.acc1 = round(self.acc1, mem.readIntLittle(u64, buf[0..8]));
66 self.acc2 = round(self.acc2, mem.readIntLittle(u64, buf[8..16]));
67 self.acc3 = round(self.acc3, mem.readIntLittle(u64, buf[16..24]));
68 self.acc4 = round(self.acc4, mem.readIntLittle(u64, buf[24..32]));
69 self.byte_count += 32;
70 }
71
72 inline fn round(acc: u64, lane: u64) u64 {
73 const a = acc +% (lane *% prime_2);
74 const b = rotl(31, a);
75 return b *% prime_1;
76 }
77
78 pub fn final(self: *XxHash64) u64 {
79 var acc: u64 = undefined;
80
81 if (self.byte_count < 32) {
82 acc = self.seed +% prime_5;
83 } else {
84 acc = rotl(1, self.acc1) +% rotl(7, self.acc2) +% rotl(12, self.acc3) +% rotl(18, self.acc4);
85 acc = mergeAccumulator(acc, self.acc1);
86 acc = mergeAccumulator(acc, self.acc2);
87 acc = mergeAccumulator(acc, self.acc3);
88 acc = mergeAccumulator(acc, self.acc4);
89 }
90
91 acc = acc +% @as(u64, self.byte_count) +% @as(u64, self.buf_len);
92
93 var pos: usize = 0;
94 while (pos + 8 <= self.buf_len) : (pos += 8) {
95 const lane = mem.readIntLittle(u64, self.buf[pos..][0..8]);
96 acc ^= round(0, lane);
97 acc = rotl(27, acc) *% prime_1;
98 acc +%= prime_4;
99 }
100
101 if (pos + 4 <= self.buf_len) {
102 const lane = @as(u64, mem.readIntLittle(u32, self.buf[pos..][0..4]));
103 acc ^= lane *% prime_1;
104 acc = rotl(23, acc) *% prime_2;
105 acc +%= prime_3;
106 pos += 4;
107 }
108
109 while (pos < self.buf_len) : (pos += 1) {
110 const lane = @as(u64, self.buf[pos]);
111 acc ^= lane *% prime_5;
112 acc = rotl(11, acc) *% prime_1;
113 }
114
115 acc ^= acc >> 33;
116 acc *%= prime_2;
117 acc ^= acc >> 29;
118 acc *%= prime_3;
119 acc ^= acc >> 32;
120
121 return acc;
122 }
123
124 inline fn mergeAccumulator(acc: u64, other: u64) u64 {
125 const a = acc ^ round(0, other);
126 const b = a *% prime_1;
127 return b +% prime_4;
128 }
129
130 pub fn hash(input: []const u8) u64 {
131 var hasher = XxHash64.init(0);
132 hasher.update(input);
133 return hasher.final();
134 }
135};
136
137pub const XxHash32 = struct {
138 acc1: u32,
139 acc2: u32,
140 acc3: u32,
141 acc4: u32,
142
143 seed: u32,
144 buf: [16]u8,
145 buf_len: usize,
146 byte_count: usize,
147
148 const prime_1 = 0x9E3779B1; // 0b10011110001101110111100110110001
149 const prime_2 = 0x85EBCA77; // 0b10000101111010111100101001110111
150 const prime_3 = 0xC2B2AE3D; // 0b11000010101100101010111000111101
151 const prime_4 = 0x27D4EB2F; // 0b00100111110101001110101100101111
152 const prime_5 = 0x165667B1; // 0b00010110010101100110011110110001
153
154 pub fn init(seed: u32) XxHash32 {
155 return XxHash32{
156 .seed = seed,
157 .acc1 = seed +% prime_1 +% prime_2,
158 .acc2 = seed +% prime_2,
159 .acc3 = seed,
160 .acc4 = seed -% prime_1,
161 .buf = undefined,
162 .buf_len = 0,
163 .byte_count = 0,
164 };
165 }
166
167 pub fn update(self: *XxHash32, input: []const u8) void {
168 if (input.len < 16 - self.buf_len) {
169 mem.copy(u8, self.buf[self.buf_len..], input);
170 self.buf_len += input.len;
171 return;
172 }
173
174 var i: usize = 0;
175
176 if (self.buf_len > 0) {
177 i = 16 - self.buf_len;
178 mem.copy(u8, self.buf[self.buf_len..], input[0..i]);
179 self.processStripe(&self.buf);
180 self.buf_len = 0;
181 }
182
183 while (i + 16 <= input.len) : (i += 16) {
184 self.processStripe(input[i..][0..16]);
185 }
186
187 const remaining_bytes = input[i..];
188 mem.copy(u8, &self.buf, remaining_bytes);
189 self.buf_len = remaining_bytes.len;
190 }
191
192 inline fn processStripe(self: *XxHash32, buf: *const [16]u8) void {
193 self.acc1 = round(self.acc1, mem.readIntLittle(u32, buf[0..4]));
194 self.acc2 = round(self.acc2, mem.readIntLittle(u32, buf[4..8]));
195 self.acc3 = round(self.acc3, mem.readIntLittle(u32, buf[8..12]));
196 self.acc4 = round(self.acc4, mem.readIntLittle(u32, buf[12..16]));
197 self.byte_count += 16;
198 }
199
200 inline fn round(acc: u32, lane: u32) u32 {
201 const a = acc +% (lane *% prime_2);
202 const b = rotl(13, a);
203 return b *% prime_1;
204 }
205
206 pub fn final(self: *XxHash32) u32 {
207 var acc: u32 = undefined;
208
209 if (self.byte_count < 16) {
210 acc = self.seed +% prime_5;
211 } else {
212 acc = rotl(1, self.acc1) +% rotl(7, self.acc2) +% rotl(12, self.acc3) +% rotl(18, self.acc4);
213 }
214
215 acc = acc +% @intCast(u32, self.byte_count) +% @intCast(u32, self.buf_len);
216
217 var pos: usize = 0;
218 while (pos + 4 <= self.buf_len) : (pos += 4) {
219 const lane = mem.readIntLittle(u32, self.buf[pos..][0..4]);
220 acc +%= lane *% prime_3;
221 acc = rotl(17, acc) *% prime_4;
222 }
223
224 while (pos < self.buf_len) : (pos += 1) {
225 const lane = @as(u32, self.buf[pos]);
226 acc +%= lane *% prime_5;
227 acc = rotl(11, acc) *% prime_1;
228 }
229
230 acc ^= acc >> 15;
231 acc *%= prime_2;
232 acc ^= acc >> 13;
233 acc *%= prime_3;
234 acc ^= acc >> 16;
235
236 return acc;
237 }
238
239 pub fn hash(input: []const u8) u32 {
240 var hasher = XxHash32.init(0);
241 hasher.update(input);
242 return hasher.final();
243 }
244};
245
246test "xxhash64" {
247 const hash = XxHash64.hash;
248
249 try expectEqual(hash(""), 0xef46db3751d8e999);
250 try expectEqual(hash("a"), 0xd24ec4f1a98c6e5b);
251 try expectEqual(hash("abc"), 0x44bc2cf5ad770999);
252 try expectEqual(hash("message digest"), 0x066ed728fceeb3be);
253 try expectEqual(hash("abcdefghijklmnopqrstuvwxyz"), 0xcfe1f278fa89835c);
254 try expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0xaaa46907d3047814);
255 try expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0xe04a477f19ee145d);
256}
257
258test "xxhash32" {
259 const hash = XxHash32.hash;
260
261 try expectEqual(hash(""), 0x02cc5d05);
262 try expectEqual(hash("a"), 0x550d7456);
263 try expectEqual(hash("abc"), 0x32d153ff);
264 try expectEqual(hash("message digest"), 0x7c948494);
265 try expectEqual(hash("abcdefghijklmnopqrstuvwxyz"), 0x63a14d5f);
266 try expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x9c285e64);
267 try expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x9c05f475);
268}