authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2019-08-28 21:25:50+12:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-08-28 21:25:50+12:00
logac477f3c9a94b6d32a56d89a01ae24c68143ee5d
tree3da2335391f4e14ec992f779524d62d9a44f5b3e
parent47fcbfdc51bcb9c9d518a76b8eee122833893660
parentfbcdf78cbd5ff955d1aec0e55026168eed8d43f6
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3060 from Sahnvour/hashing

auto_hash with deep/shallow hashing

9 files changed, 374 insertions(+), 108 deletions(-)

std/event/fs.zig+15-5
......@@ -719,6 +719,16 @@ pub const WatchEventId = enum {
719719 Delete,
720720};
721721
722fn eqlString(a: []const u16, b: []const u16) bool {
723 if (a.len != b.len) return false;
724 if (a.ptr == b.ptr) return true;
725 return mem.compare(u16, a, b) == .Equal;
726}
727
728fn hashString(s: []const u16) u32 {
729 return @truncate(u32, std.hash.Wyhash.hash(0, @sliceToBytes(s)));
730}
731
722732//pub const WatchEventError = error{
723733// UserResourceLimitReached,
724734// SystemResources,
......@@ -736,7 +746,7 @@ pub const WatchEventId = enum {
736746// file_table: FileTable,
737747// table_lock: event.Lock,
738748//
739// const FileTable = std.AutoHashMap([]const u8, *Put);
749// const FileTable = std.StringHashmap(*Put);
740750// const Put = struct {
741751// putter: anyframe,
742752// value_ptr: *V,
......@@ -755,8 +765,8 @@ pub const WatchEventId = enum {
755765// all_putters: std.atomic.Queue(anyframe),
756766// ref_count: std.atomic.Int(usize),
757767//
758// const DirTable = std.AutoHashMap([]const u8, *Dir);
759// const FileTable = std.AutoHashMap([]const u16, V);
768// const DirTable = std.StringHashMap(*Dir);
769// const FileTable = std.HashMap([]const u16, V, hashString, eqlString);
760770//
761771// const Dir = struct {
762772// putter: anyframe,
......@@ -772,7 +782,7 @@ pub const WatchEventId = enum {
772782// table_lock: event.Lock,
773783//
774784// const WdTable = std.AutoHashMap(i32, Dir);
775// const FileTable = std.AutoHashMap([]const u8, V);
785// const FileTable = std.StringHashMap(V);
776786//
777787// const Dir = struct {
778788// dirname: []const u8,
......@@ -780,7 +790,7 @@ pub const WatchEventId = enum {
780790// };
781791// };
782792//
783// const FileToHandle = std.AutoHashMap([]const u8, anyframe);
793// const FileToHandle = std.StringHashMap(anyframe);
784794//
785795// const Self = @This();
786796//
std/hash/auto_hash.zig+219-62
......@@ -3,9 +3,76 @@ const builtin = @import("builtin");
33const mem = std.mem;
44const meta = std.meta;
55
6/// Describes how pointer types should be hashed.
7pub const HashStrategy = enum {
8 /// Do not follow pointers, only hash their value.
9 Shallow,
10
11 /// Follow pointers, hash the pointee content.
12 /// Only dereferences one level, ie. it is changed into .Shallow when a
13 /// pointer type is encountered.
14 Deep,
15
16 /// Follow pointers, hash the pointee content.
17 /// Dereferences all pointers encountered.
18 /// Assumes no cycle.
19 DeepRecursive,
20};
21
22/// Helper function to hash a pointer and mutate the strategy if needed.
23pub fn hashPointer(hasher: var, key: var, comptime strat: HashStrategy) void {
24 const info = @typeInfo(@typeOf(key));
25
26 switch (info.Pointer.size) {
27 builtin.TypeInfo.Pointer.Size.One => switch (strat) {
28 .Shallow => hash(hasher, @ptrToInt(key), .Shallow),
29 .Deep => hash(hasher, key.*, .Shallow),
30 .DeepRecursive => hash(hasher, key.*, .DeepRecursive),
31 },
32
33 builtin.TypeInfo.Pointer.Size.Slice => switch (strat) {
34 .Shallow => {
35 hashPointer(hasher, key.ptr, .Shallow);
36 hash(hasher, key.len, .Shallow);
37 },
38 .Deep => hashArray(hasher, key, .Shallow),
39 .DeepRecursive => hashArray(hasher, key, .DeepRecursive),
40 },
41
42 builtin.TypeInfo.Pointer.Size.Many,
43 builtin.TypeInfo.Pointer.Size.C,
44 => switch (strat) {
45 .Shallow => hash(hasher, @ptrToInt(key), .Shallow),
46 else => @compileError(
47 \\ unknown-length pointers and C pointers cannot be hashed deeply.
48 \\ Consider providing your own hash function.
49 ),
50 },
51 }
52}
53
54/// Helper function to hash a set of contiguous objects, from an array or slice.
55pub fn hashArray(hasher: var, key: var, comptime strat: HashStrategy) void {
56 switch (strat) {
57 .Shallow => {
58 // TODO detect via a trait when Key has no padding bits to
59 // hash it as an array of bytes.
60 // Otherwise, hash every element.
61 for (key) |element| {
62 hash(hasher, element, .Shallow);
63 }
64 },
65 else => {
66 for (key) |element| {
67 hash(hasher, element, strat);
68 }
69 },
70 }
71}
72
673/// Provides generic hashing for any eligible type.
7/// Only hashes `key` itself, pointers are not followed.
8pub fn autoHash(hasher: var, key: var) void {
74/// Strategy is provided to determine if pointers should be followed or not.
75pub fn hash(hasher: var, key: var, comptime strat: HashStrategy) void {
976 const Key = @typeOf(key);
1077 switch (@typeInfo(Key)) {
1178 .NoReturn,
......@@ -26,35 +93,18 @@ pub fn autoHash(hasher: var, key: var) void {
2693 // TODO Check if the situation is better after #561 is resolved.
2794 .Int => @inlineCall(hasher.update, std.mem.asBytes(&key)),
2895
29 .Float => |info| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)),
96 .Float => |info| hash(hasher, @bitCast(@IntType(false, info.bits), key), strat),
3097
31 .Bool => autoHash(hasher, @boolToInt(key)),
32 .Enum => autoHash(hasher, @enumToInt(key)),
33 .ErrorSet => autoHash(hasher, @errorToInt(key)),
34 .AnyFrame, .Fn => autoHash(hasher, @ptrToInt(key)),
98 .Bool => hash(hasher, @boolToInt(key), strat),
99 .Enum => hash(hasher, @enumToInt(key), strat),
100 .ErrorSet => hash(hasher, @errorToInt(key), strat),
101 .AnyFrame, .Fn => hash(hasher, @ptrToInt(key), strat),
35102
36 .Pointer => |info| switch (info.size) {
37 builtin.TypeInfo.Pointer.Size.One,
38 builtin.TypeInfo.Pointer.Size.Many,
39 builtin.TypeInfo.Pointer.Size.C,
40 => autoHash(hasher, @ptrToInt(key)),
103 .Pointer => @inlineCall(hashPointer, hasher, key, strat),
41104
42 builtin.TypeInfo.Pointer.Size.Slice => {
43 autoHash(hasher, key.ptr);
44 autoHash(hasher, key.len);
45 },
46 },
105 .Optional => if (key) |k| hash(hasher, k, strat),
47106
48 .Optional => if (key) |k| autoHash(hasher, k),
49
50 .Array => {
51 // TODO detect via a trait when Key has no padding bits to
52 // hash it as an array of bytes.
53 // Otherwise, hash every element.
54 for (key) |element| {
55 autoHash(hasher, element);
56 }
57 },
107 .Array => hashArray(hasher, key, strat),
58108
59109 .Vector => |info| {
60110 if (info.child.bit_count % 8 == 0) {
......@@ -67,7 +117,7 @@ pub fn autoHash(hasher: var, key: var) void {
67117 const array: [info.len]info.child = key;
68118 comptime var i: u32 = 0;
69119 inline while (i < info.len) : (i += 1) {
70 autoHash(hasher, array[i]);
120 hash(hasher, array[i], strat);
71121 }
72122 }
73123 },
......@@ -79,19 +129,19 @@ pub fn autoHash(hasher: var, key: var) void {
79129 inline for (info.fields) |field| {
80130 // We reuse the hash of the previous field as the seed for the
81131 // next one so that they're dependant.
82 autoHash(hasher, @field(key, field.name));
132 hash(hasher, @field(key, field.name), strat);
83133 }
84134 },
85135
86136 .Union => |info| blk: {
87137 if (info.tag_type) |tag_type| {
88138 const tag = meta.activeTag(key);
89 const s = autoHash(hasher, tag);
139 const s = hash(hasher, tag, strat);
90140 inline for (info.fields) |field| {
91141 const enum_field = field.enum_field.?;
92142 if (enum_field.value == @enumToInt(tag)) {
93 autoHash(hasher, @field(key, enum_field.name));
94 // TODO use a labelled break when it does not crash the compiler.
143 hash(hasher, @field(key, enum_field.name), strat);
144 // TODO use a labelled break when it does not crash the compiler. cf #2908
95145 // break :blk;
96146 return;
97147 }
......@@ -102,25 +152,77 @@ pub fn autoHash(hasher: var, key: var) void {
102152
103153 .ErrorUnion => blk: {
104154 const payload = key catch |err| {
105 autoHash(hasher, err);
155 hash(hasher, err, strat);
106156 break :blk;
107157 };
108 autoHash(hasher, payload);
158 hash(hasher, payload, strat);
109159 },
110160 }
111161}
112162
163/// Provides generic hashing for any eligible type.
164/// Only hashes `key` itself, pointers are not followed.
165/// Slices are rejected to avoid ambiguity on the user's intention.
166pub fn autoHash(hasher: var, key: var) void {
167 const Key = @typeOf(key);
168 if (comptime meta.trait.isSlice(Key))
169 @compileError("std.auto_hash.autoHash does not allow slices (here " ++ @typeName(Key) ++ " because the intent is unclear. Consider using std.auto_hash.hash or providing your own hash function instead.");
170
171 hash(hasher, key, .Shallow);
172}
173
113174const testing = std.testing;
114175const Wyhash = std.hash.Wyhash;
115176
116fn testAutoHash(key: var) u64 {
177fn testHash(key: var) u64 {
117178 // Any hash could be used here, for testing autoHash.
118179 var hasher = Wyhash.init(0);
119 autoHash(&hasher, key);
180 hash(&hasher, key, .Shallow);
120181 return hasher.final();
121182}
122183
123test "autoHash slice" {
184fn testHashShallow(key: var) u64 {
185 // Any hash could be used here, for testing autoHash.
186 var hasher = Wyhash.init(0);
187 hash(&hasher, key, .Shallow);
188 return hasher.final();
189}
190
191fn testHashDeep(key: var) u64 {
192 // Any hash could be used here, for testing autoHash.
193 var hasher = Wyhash.init(0);
194 hash(&hasher, key, .Deep);
195 return hasher.final();
196}
197
198fn testHashDeepRecursive(key: var) u64 {
199 // Any hash could be used here, for testing autoHash.
200 var hasher = Wyhash.init(0);
201 hash(&hasher, key, .DeepRecursive);
202 return hasher.final();
203}
204
205test "hash pointer" {
206 const array = [_]u32{ 123, 123, 123 };
207 const a = &array[0];
208 const b = &array[1];
209 const c = &array[2];
210 const d = a;
211
212 testing.expect(testHashShallow(a) == testHashShallow(d));
213 testing.expect(testHashShallow(a) != testHashShallow(c));
214 testing.expect(testHashShallow(a) != testHashShallow(b));
215
216 testing.expect(testHashDeep(a) == testHashDeep(a));
217 testing.expect(testHashDeep(a) == testHashDeep(c));
218 testing.expect(testHashDeep(a) == testHashDeep(b));
219
220 testing.expect(testHashDeepRecursive(a) == testHashDeepRecursive(a));
221 testing.expect(testHashDeepRecursive(a) == testHashDeepRecursive(c));
222 testing.expect(testHashDeepRecursive(a) == testHashDeepRecursive(b));
223}
224
225test "hash slice shallow" {
124226 // Allocate one array dynamically so that we're assured it is not merged
125227 // with the other by the optimization passes.
126228 const array1 = try std.heap.direct_allocator.create([6]u32);
......@@ -130,23 +232,78 @@ test "autoHash slice" {
130232 const a = array1[0..];
131233 const b = array2[0..];
132234 const c = array1[0..3];
133 testing.expect(testAutoHash(a) == testAutoHash(a));
134 testing.expect(testAutoHash(a) != testAutoHash(array1));
135 testing.expect(testAutoHash(a) != testAutoHash(b));
136 testing.expect(testAutoHash(a) != testAutoHash(c));
235 testing.expect(testHashShallow(a) == testHashShallow(a));
236 testing.expect(testHashShallow(a) != testHashShallow(array1));
237 testing.expect(testHashShallow(a) != testHashShallow(b));
238 testing.expect(testHashShallow(a) != testHashShallow(c));
239}
240
241test "hash slice deep" {
242 // Allocate one array dynamically so that we're assured it is not merged
243 // with the other by the optimization passes.
244 const array1 = try std.heap.direct_allocator.create([6]u32);
245 defer std.heap.direct_allocator.destroy(array1);
246 array1.* = [_]u32{ 1, 2, 3, 4, 5, 6 };
247 const array2 = [_]u32{ 1, 2, 3, 4, 5, 6 };
248 const a = array1[0..];
249 const b = array2[0..];
250 const c = array1[0..3];
251 testing.expect(testHashDeep(a) == testHashDeep(a));
252 testing.expect(testHashDeep(a) == testHashDeep(array1));
253 testing.expect(testHashDeep(a) == testHashDeep(b));
254 testing.expect(testHashDeep(a) != testHashDeep(c));
255}
256
257test "hash struct deep" {
258 const Foo = struct {
259 a: u32,
260 b: f64,
261 c: *bool,
262
263 const Self = @This();
264
265 pub fn init(allocator: *mem.Allocator, a_: u32, b_: f64, c_: bool) !Self {
266 const ptr = try allocator.create(bool);
267 ptr.* = c_;
268 return Self{ .a = a_, .b = b_, .c = ptr };
269 }
270 };
271
272 const allocator = std.heap.direct_allocator;
273 const foo = try Foo.init(allocator, 123, 1.0, true);
274 const bar = try Foo.init(allocator, 123, 1.0, true);
275 const baz = try Foo.init(allocator, 123, 1.0, false);
276 defer allocator.destroy(foo.c);
277 defer allocator.destroy(bar.c);
278 defer allocator.destroy(baz.c);
279
280 testing.expect(testHashDeep(foo) == testHashDeep(bar));
281 testing.expect(testHashDeep(foo) != testHashDeep(baz));
282 testing.expect(testHashDeep(bar) != testHashDeep(baz));
283
284 var hasher = Wyhash.init(0);
285 const h = testHashDeep(foo);
286 autoHash(&hasher, foo.a);
287 autoHash(&hasher, foo.b);
288 autoHash(&hasher, foo.c.*);
289 testing.expectEqual(h, hasher.final());
290
291 const h2 = testHashDeepRecursive(&foo);
292 testing.expect(h2 != testHashDeep(&foo));
293 testing.expect(h2 == testHashDeep(foo));
137294}
138295
139test "testAutoHash optional" {
296test "testHash optional" {
140297 const a: ?u32 = 123;
141298 const b: ?u32 = null;
142 testing.expectEqual(testAutoHash(a), testAutoHash(u32(123)));
143 testing.expect(testAutoHash(a) != testAutoHash(b));
144 testing.expectEqual(testAutoHash(b), 0);
299 testing.expectEqual(testHash(a), testHash(u32(123)));
300 testing.expect(testHash(a) != testHash(b));
301 testing.expectEqual(testHash(b), 0);
145302}
146303
147test "testAutoHash array" {
304test "testHash array" {
148305 const a = [_]u32{ 1, 2, 3 };
149 const h = testAutoHash(a);
306 const h = testHash(a);
150307 var hasher = Wyhash.init(0);
151308 autoHash(&hasher, u32(1));
152309 autoHash(&hasher, u32(2));
......@@ -154,14 +311,14 @@ test "testAutoHash array" {
154311 testing.expectEqual(h, hasher.final());
155312}
156313
157test "testAutoHash struct" {
314test "testHash struct" {
158315 const Foo = struct {
159316 a: u32 = 1,
160317 b: u32 = 2,
161318 c: u32 = 3,
162319 };
163320 const f = Foo{};
164 const h = testAutoHash(f);
321 const h = testHash(f);
165322 var hasher = Wyhash.init(0);
166323 autoHash(&hasher, u32(1));
167324 autoHash(&hasher, u32(2));
......@@ -169,7 +326,7 @@ test "testAutoHash struct" {
169326 testing.expectEqual(h, hasher.final());
170327}
171328
172test "testAutoHash union" {
329test "testHash union" {
173330 const Foo = union(enum) {
174331 A: u32,
175332 B: f32,
......@@ -179,24 +336,24 @@ test "testAutoHash union" {
179336 const a = Foo{ .A = 18 };
180337 var b = Foo{ .B = 12.34 };
181338 const c = Foo{ .C = 18 };
182 testing.expect(testAutoHash(a) == testAutoHash(a));
183 testing.expect(testAutoHash(a) != testAutoHash(b));
184 testing.expect(testAutoHash(a) != testAutoHash(c));
339 testing.expect(testHash(a) == testHash(a));
340 testing.expect(testHash(a) != testHash(b));
341 testing.expect(testHash(a) != testHash(c));
185342
186343 b = Foo{ .A = 18 };
187 testing.expect(testAutoHash(a) == testAutoHash(b));
344 testing.expect(testHash(a) == testHash(b));
188345}
189346
190test "testAutoHash vector" {
347test "testHash vector" {
191348 const a: @Vector(4, u32) = [_]u32{ 1, 2, 3, 4 };
192349 const b: @Vector(4, u32) = [_]u32{ 1, 2, 3, 5 };
193350 const c: @Vector(4, u31) = [_]u31{ 1, 2, 3, 4 };
194 testing.expect(testAutoHash(a) == testAutoHash(a));
195 testing.expect(testAutoHash(a) != testAutoHash(b));
196 testing.expect(testAutoHash(a) != testAutoHash(c));
351 testing.expect(testHash(a) == testHash(a));
352 testing.expect(testHash(a) != testHash(b));
353 testing.expect(testHash(a) != testHash(c));
197354}
198355
199test "testAutoHash error union" {
356test "testHash error union" {
200357 const Errors = error{Test};
201358 const Foo = struct {
202359 a: u32 = 1,
......@@ -205,7 +362,7 @@ test "testAutoHash error union" {
205362 };
206363 const f = Foo{};
207364 const g: Errors!Foo = Errors.Test;
208 testing.expect(testAutoHash(f) != testAutoHash(g));
209 testing.expect(testAutoHash(f) == testAutoHash(Foo{}));
210 testing.expect(testAutoHash(g) == testAutoHash(Errors.Test));
365 testing.expect(testHash(f) != testHash(g));
366 testing.expect(testHash(f) == testHash(Foo{}));
367 testing.expect(testHash(g) == testHash(Errors.Test));
211368}
std/hash/benchmark.zig+1-1
......@@ -86,7 +86,7 @@ const Result = struct {
8686 throughput: u64,
8787};
8888
89const block_size: usize = 8192;
89const block_size: usize = 8 * 8192;
9090
9191pub fn benchmarkHash(comptime H: var, bytes: usize) !Result {
9292 var h = blk: {
std/hash/wyhash.zig+117-21
......@@ -10,7 +10,8 @@ const primes = [_]u64{
1010};
1111
1212fn read_bytes(comptime bytes: u8, data: []const u8) u64 {
13 return mem.readVarInt(u64, data[0..bytes], .Little);
13 const T = @IntType(false, 8 * bytes);
14 return mem.readIntSliceLittle(T, data[0..bytes]);
1415}
1516
1617fn read_8bytes_swapped(data: []const u8) u64 {
......@@ -31,18 +32,21 @@ fn mix1(a: u64, b: u64, seed: u64) u64 {
3132 return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);
3233}
3334
34pub const Wyhash = struct {
35// Wyhash version which does not store internal state for handling partial buffers.
36// This is needed so that we can maximize the speed for the short key case, which will
37// use the non-iterative api which the public Wyhash exposes.
38const WyhashStateless = struct {
3539 seed: u64,
3640 msg_len: usize,
3741
38 pub fn init(seed: u64) Wyhash {
39 return Wyhash{
42 pub fn init(seed: u64) WyhashStateless {
43 return WyhashStateless{
4044 .seed = seed,
4145 .msg_len = 0,
4246 };
4347 }
4448
45 fn round(self: *Wyhash, b: []const u8) void {
49 fn round(self: *WyhashStateless, b: []const u8) void {
4650 std.debug.assert(b.len == 32);
4751
4852 self.seed = mix0(
......@@ -56,12 +60,25 @@ pub const Wyhash = struct {
5660 );
5761 }
5862
59 fn partial(self: *Wyhash, b: []const u8) void {
60 const rem_key = b;
61 const rem_len = b.len;
63 pub fn update(self: *WyhashStateless, b: []const u8) void {
64 std.debug.assert(b.len % 32 == 0);
65
66 var off: usize = 0;
67 while (off < b.len) : (off += 32) {
68 @inlineCall(self.round, b[off .. off + 32]);
69 }
6270
63 var seed = self.seed;
64 seed = switch (@intCast(u5, rem_len)) {
71 self.msg_len += b.len;
72 }
73
74 pub fn final(self: *WyhashStateless, b: []const u8) u64 {
75 std.debug.assert(b.len < 32);
76
77 const seed = self.seed;
78 const rem_len = @intCast(u5, b.len);
79 const rem_key = b[0..rem_len];
80
81 self.seed = switch (rem_len) {
6582 0 => seed,
6683 1 => mix0(read_bytes(1, rem_key), primes[4], seed),
6784 2 => mix0(read_bytes(2, rem_key), primes[4], seed),
......@@ -95,34 +112,70 @@ pub const Wyhash = struct {
95112 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed),
96113 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed),
97114 };
98 self.seed = seed;
115
116 self.msg_len += b.len;
117 return mum(self.seed ^ self.msg_len, primes[4]);
118 }
119
120 pub fn hash(seed: u64, input: []const u8) u64 {
121 const aligned_len = input.len - (input.len % 32);
122
123 var c = WyhashStateless.init(seed);
124 @inlineCall(c.update, input[0..aligned_len]);
125 return @inlineCall(c.final, input[aligned_len..]);
126 }
127};
128
129/// Fast non-cryptographic 64bit hash function.
130/// See https://github.com/wangyi-fudan/wyhash
131pub const Wyhash = struct {
132 state: WyhashStateless,
133
134 buf: [32]u8,
135 buf_len: usize,
136
137 pub fn init(seed: u64) Wyhash {
138 return Wyhash{
139 .state = WyhashStateless.init(seed),
140 .buf = undefined,
141 .buf_len = 0,
142 };
99143 }
100144
101145 pub fn update(self: *Wyhash, b: []const u8) void {
102146 var off: usize = 0;
103147
104 // Full middle blocks.
105 while (off + 32 <= b.len) : (off += 32) {
106 @inlineCall(self.round, b[off .. off + 32]);
148 if (self.buf_len != 0 and self.buf_len + b.len >= 32) {
149 off += 32 - self.buf_len;
150 mem.copy(u8, self.buf[self.buf_len..], b[0..off]);
151 self.state.update(self.buf[0..]);
152 self.buf_len = 0;
107153 }
108154
109 self.partial(b[off..]);
110 self.msg_len += b.len;
155 const remain_len = b.len - off;
156 const aligned_len = remain_len - (remain_len % 32);
157 self.state.update(b[off .. off + aligned_len]);
158
159 mem.copy(u8, self.buf[self.buf_len..], b[off + aligned_len ..]);
160 self.buf_len += @intCast(u8, b[off + aligned_len ..].len);
111161 }
112162
113163 pub fn final(self: *Wyhash) u64 {
114 return mum(self.seed ^ self.msg_len, primes[4]);
164 const seed = self.state.seed;
165 const rem_len = @intCast(u5, self.buf_len);
166 const rem_key = self.buf[0..self.buf_len];
167
168 return self.state.final(rem_key);
115169 }
116170
117171 pub fn hash(seed: u64, input: []const u8) u64 {
118 var c = Wyhash.init(seed);
119 @inlineCall(c.update, input);
120 return @inlineCall(c.final);
172 return WyhashStateless.hash(seed, input);
121173 }
122174};
123175
176const expectEqual = std.testing.expectEqual;
177
124178test "test vectors" {
125 const expectEqual = std.testing.expectEqual;
126179 const hash = Wyhash.hash;
127180
128181 expectEqual(hash(0, ""), 0x0);
......@@ -133,3 +186,46 @@ test "test vectors" {
133186 expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
134187 expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
135188}
189
190test "test vectors streaming" {
191 var wh = Wyhash.init(5);
192 for ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") |e| {
193 wh.update(mem.asBytes(&e));
194 }
195 expectEqual(wh.final(), 0x602a1894d3bbfe7f);
196
197 const pattern = "1234567890";
198 const count = 8;
199 const result = 0x829e9c148b75970e;
200 expectEqual(Wyhash.hash(6, pattern ** 8), result);
201
202 wh = Wyhash.init(6);
203 var i: u32 = 0;
204 while (i < count) : (i += 1) {
205 wh.update(pattern);
206 }
207 expectEqual(wh.final(), result);
208}
209
210test "iterative non-divisible update" {
211 var buf: [8192]u8 = undefined;
212 for (buf) |*e, i| {
213 e.* = @truncate(u8, i);
214 }
215
216 const seed = 0x128dad08f;
217
218 var end: usize = 32;
219 while (end < buf.len) : (end += 32) {
220 const non_iterative_hash = Wyhash.hash(seed, buf[0..end]);
221
222 var wy = Wyhash.init(seed);
223 var i: usize = 0;
224 while (i < end) : (i += 33) {
225 wy.update(buf[i..std.math.min(i + 33, end)]);
226 }
227 const iterative_hash = wy.final();
228
229 std.testing.expectEqual(iterative_hash, non_iterative_hash);
230 }
231}
std/hash_map.zig+15
......@@ -17,6 +17,21 @@ pub fn AutoHashMap(comptime K: type, comptime V: type) type {
1717 return HashMap(K, V, getAutoHashFn(K), getAutoEqlFn(K));
1818}
1919
20/// Builtin hashmap for strings as keys.
21pub fn StringHashMap(comptime V: type) type {
22 return HashMap([]const u8, V, hashString, eqlString);
23}
24
25pub fn eqlString(a: []const u8, b: []const u8) bool {
26 if (a.len != b.len) return false;
27 if (a.ptr == b.ptr) return true;
28 return mem.compare(u8, a, b) == .Equal;
29}
30
31pub fn hashString(s: []const u8) u32 {
32 return @truncate(u32, std.hash.Wyhash.hash(0, s));
33}
34
2035pub fn HashMap(comptime K: type, comptime V: type, comptime hash: fn (key: K) u32, comptime eql: fn (a: K, b: K) bool) type {
2136 return struct {
2237 entries: []Entry,
std/http/headers.zig+1-11
......@@ -102,19 +102,9 @@ test "HeaderEntry" {
102102 testing.expectEqualSlices(u8, "x", e.value);
103103}
104104
105fn stringEql(a: []const u8, b: []const u8) bool {
106 if (a.len != b.len) return false;
107 if (a.ptr == b.ptr) return true;
108 return mem.compare(u8, a, b) == .Equal;
109}
110
111fn stringHash(s: []const u8) u32 {
112 return @truncate(u32, std.hash.Wyhash.hash(0, s));
113}
114
115105const HeaderList = std.ArrayList(HeaderEntry);
116106const HeaderIndexList = std.ArrayList(usize);
117const HeaderIndex = std.HashMap([]const u8, HeaderIndexList, stringHash, stringEql);
107const HeaderIndex = std.StringHashMap(HeaderIndexList);
118108
119109pub const Headers = struct {
120110 // the owned header field name is stored in the index as part of the key
std/std.zig+1
......@@ -17,6 +17,7 @@ pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;
1717pub const StaticallyInitializedMutex = @import("statically_initialized_mutex.zig").StaticallyInitializedMutex;
1818pub const SegmentedList = @import("segmented_list.zig").SegmentedList;
1919pub const SpinLock = @import("spinlock.zig").SpinLock;
20pub const StringHashMap = @import("hash_map.zig").StringHashMap;
2021pub const ChildProcess = @import("child_process.zig").ChildProcess;
2122pub const TailQueue = @import("linked_list.zig").TailQueue;
2223pub const Thread = @import("thread.zig").Thread;
tools/process_headers.zig+2-5
......@@ -504,12 +504,9 @@ const Contents = struct {
504504 }
505505};
506506
507comptime {
508 @compileError("the behavior of std.AutoHashMap changed and []const u8 will be treated as a pointer. will need to update the hash maps to actually do some kind of hashing on the slices.");
509}
510const HashToContents = std.AutoHashMap([]const u8, Contents);
507const HashToContents = std.StringHashMap(Contents);
511508const TargetToHash = std.HashMap(DestTarget, []const u8, DestTarget.hash, DestTarget.eql);
512const PathTable = std.AutoHashMap([]const u8, *TargetToHash);
509const PathTable = std.StringHashMap(*TargetToHash);
513510
514511const LibCVendor = enum {
515512 musl,
tools/update_glibc.zig+3-3
......@@ -118,7 +118,7 @@ const FunctionSet = struct {
118118 list: std.ArrayList(VersionedFn),
119119 fn_vers_list: FnVersionList,
120120};
121const FnVersionList = std.AutoHashMap([]const u8, std.ArrayList(usize));
121const FnVersionList = std.StringHashMap(std.ArrayList(usize));
122122
123123const VersionedFn = struct {
124124 ver: []const u8, // example: "GLIBC_2.15"
......@@ -140,8 +140,8 @@ pub fn main() !void {
140140 const prefix = try fs.path.join(allocator, [_][]const u8{ in_glibc_dir, "sysdeps", "unix", "sysv", "linux" });
141141 const glibc_out_dir = try fs.path.join(allocator, [_][]const u8{ zig_src_dir, "libc", "glibc" });
142142
143 var global_fn_set = std.AutoHashMap([]const u8, Function).init(allocator);
144 var global_ver_set = std.AutoHashMap([]const u8, usize).init(allocator);
143 var global_fn_set = std.StringHashMap(Function).init(allocator);
144 var global_ver_set = std.StringHashMap(usize).init(allocator);
145145 var target_functions = std.AutoHashMap(usize, FunctionSet).init(allocator);
146146
147147 for (abi_lists) |*abi_list| {