authorgravatar for Sahnvour@users.noreply.github.comSahnvour <Sahnvour@users.noreply.github.com> 2019-08-04 21:02:00+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-08-04 21:02:00+02:00
log8c99a5199324a1aabba698a5127e9ce1df294aa7
tree47a49324921c442e74b8e44f8471dabe0e5878cc
parentf01cb8cc16bd048adefdec5a426d5ef33dff9168
parent54255ee32e1e6c83b04c3e5f2f1dd7e8aa5e0dd7
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #2797 from Sahnvour/hashing

hash algorithm improvements

6 files changed, 525 insertions(+), 113 deletions(-)

std/hash.zig+10
...@@ -1,6 +1,9 @@...@@ -1,6 +1,9 @@
1const adler = @import("hash/adler.zig");1const adler = @import("hash/adler.zig");
2pub const Adler32 = adler.Adler32;2pub const Adler32 = adler.Adler32;
33
4const auto_hash = @import("hash/auto_hash.zig");
5pub const autoHash = auto_hash.autoHash;
6
4// pub for polynomials + generic crc32 construction7// pub for polynomials + generic crc32 construction
5pub const crc = @import("hash/crc.zig");8pub const crc = @import("hash/crc.zig");
6pub const Crc32 = crc.Crc32;9pub const Crc32 = crc.Crc32;
...@@ -16,6 +19,8 @@ pub const SipHash128 = siphash.SipHash128;...@@ -16,6 +19,8 @@ pub const SipHash128 = siphash.SipHash128;
1619
17pub const murmur = @import("hash/murmur.zig");20pub const murmur = @import("hash/murmur.zig");
18pub const Murmur2_32 = murmur.Murmur2_32;21pub const Murmur2_32 = murmur.Murmur2_32;
22
23
19pub const Murmur2_64 = murmur.Murmur2_64;24pub const Murmur2_64 = murmur.Murmur2_64;
20pub const Murmur3_32 = murmur.Murmur3_32;25pub const Murmur3_32 = murmur.Murmur3_32;
2126
...@@ -23,11 +28,16 @@ pub const cityhash = @import("hash/cityhash.zig");...@@ -23,11 +28,16 @@ pub const cityhash = @import("hash/cityhash.zig");
23pub const CityHash32 = cityhash.CityHash32;28pub const CityHash32 = cityhash.CityHash32;
24pub const CityHash64 = cityhash.CityHash64;29pub const CityHash64 = cityhash.CityHash64;
2530
31const wyhash = @import("hash/wyhash.zig");
32pub const Wyhash = wyhash.Wyhash;
33
26test "hash" {34test "hash" {
27 _ = @import("hash/adler.zig");35 _ = @import("hash/adler.zig");
36 _ = @import("hash/auto_hash.zig");
28 _ = @import("hash/crc.zig");37 _ = @import("hash/crc.zig");
29 _ = @import("hash/fnv.zig");38 _ = @import("hash/fnv.zig");
30 _ = @import("hash/siphash.zig");39 _ = @import("hash/siphash.zig");
31 _ = @import("hash/murmur.zig");40 _ = @import("hash/murmur.zig");
32 _ = @import("hash/cityhash.zig");41 _ = @import("hash/cityhash.zig");
42 _ = @import("hash/wyhash.zig");
33}43}
std/hash/auto_hash.zig created+210
...@@ -0,0 +1,210 @@
1const std = @import("std");
2const builtin = @import("builtin");
3const mem = std.mem;
4const meta = std.meta;
5
6/// Provides generic hashing for any eligible type.
7/// Only hashes `key` itself, pointers are not followed.
8pub fn autoHash(hasher: var, key: var) void {
9 const Key = @typeOf(key);
10 switch (@typeInfo(Key)) {
11 builtin.TypeId.NoReturn,
12 builtin.TypeId.Opaque,
13 builtin.TypeId.Undefined,
14 builtin.TypeId.ArgTuple,
15 builtin.TypeId.Void,
16 builtin.TypeId.Null,
17 builtin.TypeId.BoundFn,
18 builtin.TypeId.ComptimeFloat,
19 builtin.TypeId.ComptimeInt,
20 builtin.TypeId.Type,
21 builtin.TypeId.EnumLiteral,
22 => @compileError("cannot hash this type"),
23
24 // Help the optimizer see that hashing an int is easy by inlining!
25 // TODO Check if the situation is better after #561 is resolved.
26 builtin.TypeId.Int => @inlineCall(hasher.update, std.mem.asBytes(&key)),
27
28 builtin.TypeId.Float => |info| autoHash(hasher, @bitCast(@IntType(false, info.bits), key)),
29
30 builtin.TypeId.Bool => autoHash(hasher, @boolToInt(key)),
31 builtin.TypeId.Enum => autoHash(hasher, @enumToInt(key)),
32 builtin.TypeId.ErrorSet => autoHash(hasher, @errorToInt(key)),
33 builtin.TypeId.Promise, builtin.TypeId.Fn => autoHash(hasher, @ptrToInt(key)),
34
35 builtin.TypeId.Pointer => |info| switch (info.size) {
36 builtin.TypeInfo.Pointer.Size.One,
37 builtin.TypeInfo.Pointer.Size.Many,
38 builtin.TypeInfo.Pointer.Size.C,
39 => autoHash(hasher, @ptrToInt(key)),
40
41 builtin.TypeInfo.Pointer.Size.Slice => {
42 autoHash(hasher, key.ptr);
43 autoHash(hasher, key.len);
44 },
45 },
46
47 builtin.TypeId.Optional => if (key) |k| autoHash(hasher, k),
48
49 builtin.TypeId.Array => {
50 // TODO detect via a trait when Key has no padding bits to
51 // hash it as an array of bytes.
52 // Otherwise, hash every element.
53 for (key) |element| {
54 autoHash(hasher, element);
55 }
56 },
57
58 builtin.TypeId.Vector => |info| {
59 if (info.child.bit_count % 8 == 0) {
60 // If there's no unused bits in the child type, we can just hash
61 // this as an array of bytes.
62 hasher.update(mem.asBytes(&key));
63 } else {
64 // Otherwise, hash every element.
65 // TODO remove the copy to an array once field access is done.
66 const array: [info.len]info.child = key;
67 comptime var i: u32 = 0;
68 inline while (i < info.len) : (i += 1) {
69 autoHash(hasher, array[i]);
70 }
71 }
72 },
73
74 builtin.TypeId.Struct => |info| {
75 // TODO detect via a trait when Key has no padding bits to
76 // hash it as an array of bytes.
77 // Otherwise, hash every field.
78 inline for (info.fields) |field| {
79 // We reuse the hash of the previous field as the seed for the
80 // next one so that they're dependant.
81 autoHash(hasher, @field(key, field.name));
82 }
83 },
84
85 builtin.TypeId.Union => |info| blk: {
86 if (info.tag_type) |tag_type| {
87 const tag = meta.activeTag(key);
88 const s = autoHash(hasher, tag);
89 inline for (info.fields) |field| {
90 const enum_field = field.enum_field.?;
91 if (enum_field.value == @enumToInt(tag)) {
92 autoHash(hasher, @field(key, enum_field.name));
93 // TODO use a labelled break when it does not crash the compiler.
94 // break :blk;
95 return;
96 }
97 }
98 unreachable;
99 } else @compileError("cannot hash untagged union type: " ++ @typeName(Key) ++ ", provide your own hash function");
100 },
101
102 builtin.TypeId.ErrorUnion => blk: {
103 const payload = key catch |err| {
104 autoHash(hasher, err);
105 break :blk;
106 };
107 autoHash(hasher, payload);
108 },
109 }
110}
111
112const testing = std.testing;
113const Wyhash = std.hash.Wyhash;
114
115fn testAutoHash(key: var) u64 {
116 // Any hash could be used here, for testing autoHash.
117 var hasher = Wyhash.init(0);
118 autoHash(&hasher, key);
119 return hasher.final();
120}
121
122test "autoHash slice" {
123 // Allocate one array dynamically so that we're assured it is not merged
124 // with the other by the optimization passes.
125 const array1 = try std.heap.direct_allocator.create([6]u32);
126 defer std.heap.direct_allocator.destroy(array1);
127 array1.* = [_]u32{ 1, 2, 3, 4, 5, 6 };
128 const array2 = [_]u32{ 1, 2, 3, 4, 5, 6 };
129 const a = array1[0..];
130 const b = array2[0..];
131 const c = array1[0..3];
132 testing.expect(testAutoHash(a) == testAutoHash(a));
133 testing.expect(testAutoHash(a) != testAutoHash(array1));
134 testing.expect(testAutoHash(a) != testAutoHash(b));
135 testing.expect(testAutoHash(a) != testAutoHash(c));
136}
137
138test "testAutoHash optional" {
139 const a: ?u32 = 123;
140 const b: ?u32 = null;
141 testing.expectEqual(testAutoHash(a), testAutoHash(u32(123)));
142 testing.expect(testAutoHash(a) != testAutoHash(b));
143 testing.expectEqual(testAutoHash(b), 0);
144}
145
146test "testAutoHash array" {
147 const a = [_]u32{ 1, 2, 3 };
148 const h = testAutoHash(a);
149 var hasher = Wyhash.init(0);
150 autoHash(&hasher, u32(1));
151 autoHash(&hasher, u32(2));
152 autoHash(&hasher, u32(3));
153 testing.expectEqual(h, hasher.final());
154}
155
156test "testAutoHash struct" {
157 const Foo = struct {
158 a: u32 = 1,
159 b: u32 = 2,
160 c: u32 = 3,
161 };
162 const f = Foo{};
163 const h = testAutoHash(f);
164 var hasher = Wyhash.init(0);
165 autoHash(&hasher, u32(1));
166 autoHash(&hasher, u32(2));
167 autoHash(&hasher, u32(3));
168 testing.expectEqual(h, hasher.final());
169}
170
171test "testAutoHash union" {
172 const Foo = union(enum) {
173 A: u32,
174 B: f32,
175 C: u32,
176 };
177
178 const a = Foo{ .A = 18 };
179 var b = Foo{ .B = 12.34 };
180 const c = Foo{ .C = 18 };
181 testing.expect(testAutoHash(a) == testAutoHash(a));
182 testing.expect(testAutoHash(a) != testAutoHash(b));
183 testing.expect(testAutoHash(a) != testAutoHash(c));
184
185 b = Foo{ .A = 18 };
186 testing.expect(testAutoHash(a) == testAutoHash(b));
187}
188
189test "testAutoHash vector" {
190 const a: @Vector(4, u32) = [_]u32{ 1, 2, 3, 4 };
191 const b: @Vector(4, u32) = [_]u32{ 1, 2, 3, 5 };
192 const c: @Vector(4, u31) = [_]u31{ 1, 2, 3, 4 };
193 testing.expect(testAutoHash(a) == testAutoHash(a));
194 testing.expect(testAutoHash(a) != testAutoHash(b));
195 testing.expect(testAutoHash(a) != testAutoHash(c));
196}
197
198test "testAutoHash error union" {
199 const Errors = error{Test};
200 const Foo = struct {
201 a: u32 = 1,
202 b: u32 = 2,
203 c: u32 = 3,
204 };
205 const f = Foo{};
206 const g: Errors!Foo = Errors.Test;
207 testing.expect(testAutoHash(f) != testAutoHash(g));
208 testing.expect(testAutoHash(f) == testAutoHash(Foo{}));
209 testing.expect(testAutoHash(g) == testAutoHash(Errors.Test));
210}
std/hash/throughput_test.zig created+148
...@@ -0,0 +1,148 @@
1const builtin = @import("builtin");
2const std = @import("std");
3const time = std.time;
4const Timer = time.Timer;
5const hash = std.hash;
6
7const KiB = 1024;
8const MiB = 1024 * KiB;
9const GiB = 1024 * MiB;
10
11var prng = std.rand.DefaultPrng.init(0);
12
13const Hash = struct {
14 ty: type,
15 name: []const u8,
16 init_u8s: ?[]const u8 = null,
17 init_u64: ?u64 = null,
18};
19
20const siphash_key = "0123456789abcdef";
21
22const hashes = [_]Hash{
23 Hash{ .ty = hash.Wyhash, .name = "wyhash", .init_u64 = 0 },
24 Hash{ .ty = hash.SipHash64(1, 3), .name = "siphash(1,3)", .init_u8s = siphash_key },
25 Hash{ .ty = hash.SipHash64(2, 4), .name = "siphash(2,4)", .init_u8s = siphash_key },
26 Hash{ .ty = hash.Fnv1a_64, .name = "fnv1a" },
27 Hash{ .ty = hash.Crc32, .name = "crc32" },
28};
29
30const Result = struct {
31 hash: u64,
32 throughput: u64,
33};
34
35pub fn benchmarkHash(comptime H: var, bytes: usize) !Result {
36 var h = blk: {
37 if (H.init_u8s) |init| {
38 break :blk H.ty.init(init);
39 }
40 if (H.init_u64) |init| {
41 break :blk H.ty.init(init);
42 }
43 break :blk H.ty.init();
44 };
45
46 var block: [8192]u8 = undefined;
47 prng.random.bytes(block[0..]);
48
49 var offset: usize = 0;
50 var timer = try Timer.start();
51 const start = timer.lap();
52 while (offset < bytes) : (offset += block.len) {
53 h.update(block[0..]);
54 }
55 const end = timer.read();
56
57 const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s;
58 const throughput = @floatToInt(u64, @intToFloat(f64, bytes) / elapsed_s);
59
60 return Result{
61 .hash = h.final(),
62 .throughput = throughput,
63 };
64}
65
66fn usage() void {
67 std.debug.warn(
68 \\throughput_test [options]
69 \\
70 \\Options:
71 \\ --filter [test-name]
72 \\ --seed [int]
73 \\ --count [int]
74 \\ --help
75 \\
76 );
77}
78
79fn mode(comptime x: comptime_int) comptime_int {
80 return if (builtin.mode == builtin.Mode.Debug) x / 64 else x;
81}
82
83// TODO(#1358): Replace with builtin formatted padding when available.
84fn printPad(stdout: var, s: []const u8) !void {
85 var i: usize = 0;
86 while (i < 12 - s.len) : (i += 1) {
87 try stdout.print(" ");
88 }
89 try stdout.print("{}", s);
90}
91
92pub fn main() !void {
93 var stdout_file = try std.io.getStdOut();
94 var stdout_out_stream = stdout_file.outStream();
95 const stdout = &stdout_out_stream.stream;
96
97 var buffer: [1024]u8 = undefined;
98 var fixed = std.heap.FixedBufferAllocator.init(buffer[0..]);
99 const args = try std.process.argsAlloc(&fixed.allocator);
100
101 var filter: ?[]u8 = "";
102 var count: usize = mode(128 * MiB);
103
104 var i: usize = 1;
105 while (i < args.len) : (i += 1) {
106 if (std.mem.eql(u8, args[i], "--seed")) {
107 i += 1;
108 if (i == args.len) {
109 usage();
110 std.os.exit(1);
111 }
112
113 const seed = try std.fmt.parseUnsigned(u32, args[i], 10);
114 prng.seed(seed);
115 } else if (std.mem.eql(u8, args[i], "--filter")) {
116 i += 1;
117 if (i == args.len) {
118 usage();
119 std.os.exit(1);
120 }
121
122 filter = args[i];
123 } else if (std.mem.eql(u8, args[i], "--count")) {
124 i += 1;
125 if (i == args.len) {
126 usage();
127 std.os.exit(1);
128 }
129
130 const c = try std.fmt.parseUnsigned(u32, args[i], 10);
131 count = c * MiB;
132 } else if (std.mem.eql(u8, args[i], "--help")) {
133 usage();
134 return;
135 } else {
136 usage();
137 std.os.exit(1);
138 }
139 }
140
141 inline for (hashes) |H| {
142 if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) {
143 const result = try benchmarkHash(H, count);
144 try printPad(stdout, H.name);
145 try stdout.print(": {:4} MiB/s [{:16}]\n", result.throughput / (1 * MiB), result.hash);
146 }
147 }
148}
std/hash/wyhash.zig created+135
...@@ -0,0 +1,135 @@
1const std = @import("std");
2const mem = std.mem;
3
4const primes = [_]u64{
5 0xa0761d6478bd642f,
6 0xe7037ed1a0b428db,
7 0x8ebc6af09c88c6e3,
8 0x589965cc75374cc3,
9 0x1d8e4e27c47d124f,
10};
11
12fn read_bytes(comptime bytes: u8, data: []const u8) u64 {
13 return mem.readVarInt(u64, data[0..bytes], .Little);
14}
15
16fn read_8bytes_swapped(data: []const u8) u64 {
17 return (read_bytes(4, data) << 32 | read_bytes(4, data[4..]));
18}
19
20fn mum(a: u64, b: u64) u64 {
21 var r = std.math.mulWide(u64, a, b);
22 r = (r >> 64) ^ r;
23 return @truncate(u64, r);
24}
25
26fn mix0(a: u64, b: u64, seed: u64) u64 {
27 return mum(a ^ seed ^ primes[0], b ^ seed ^ primes[1]);
28}
29
30fn mix1(a: u64, b: u64, seed: u64) u64 {
31 return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);
32}
33
34pub const Wyhash = struct {
35 seed: u64,
36 msg_len: usize,
37
38 pub fn init(seed: u64) Wyhash {
39 return Wyhash{
40 .seed = seed,
41 .msg_len = 0,
42 };
43 }
44
45 fn round(self: *Wyhash, b: []const u8) void {
46 std.debug.assert(b.len == 32);
47
48 self.seed = mix0(
49 read_bytes(8, b[0..]),
50 read_bytes(8, b[8..]),
51 self.seed,
52 ) ^ mix1(
53 read_bytes(8, b[16..]),
54 read_bytes(8, b[24..]),
55 self.seed,
56 );
57 }
58
59 fn partial(self: *Wyhash, b: []const u8) void {
60 const rem_key = b;
61 const rem_len = b.len;
62
63 var seed = self.seed;
64 seed = switch (@intCast(u5, rem_len)) {
65 0 => seed,
66 1 => mix0(read_bytes(1, rem_key), primes[4], seed),
67 2 => mix0(read_bytes(2, rem_key), primes[4], seed),
68 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed),
69 4 => mix0(read_bytes(4, rem_key), primes[4], seed),
70 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed),
71 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed),
72 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed),
73 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed),
74 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed),
75 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed),
76 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed),
77 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed),
78 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed),
79 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed),
80 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed),
81 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed),
82 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed),
83 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed),
84 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed),
85 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed),
86 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed),
87 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed),
88 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed),
89 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed),
90 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed),
91 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed),
92 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed),
93 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed),
94 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed),
95 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed),
96 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed),
97 };
98 self.seed = seed;
99 }
100
101 pub fn update(self: *Wyhash, b: []const u8) void {
102 var off: usize = 0;
103
104 // Full middle blocks.
105 while (off + 32 <= b.len) : (off += 32) {
106 @inlineCall(self.round, b[off .. off + 32]);
107 }
108
109 self.partial(b[off..]);
110 self.msg_len += b.len;
111 }
112
113 pub fn final(self: *Wyhash) u64 {
114 return mum(self.seed ^ self.msg_len, primes[4]);
115 }
116
117 pub fn hash(seed: u64, input: []const u8) u64 {
118 var c = Wyhash.init(seed);
119 c.update(input);
120 return c.final();
121 }
122};
123
124test "test vectors" {
125 const expectEqual = std.testing.expectEqual;
126 const hash = Wyhash.hash;
127
128 expectEqual(hash(0, ""), 0x0);
129 expectEqual(hash(1, "a"), 0xbed235177f41d328);
130 expectEqual(hash(2, "abc"), 0xbe348debe59b27c3);
131 expectEqual(hash(3, "message digest"), 0x37320f657213a290);
132 expectEqual(hash(4, "abcdefghijklmnopqrstuvwxyz"), 0xd0b270e1d8a7019c);
133 expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
134 expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
135}
std/hash_map.zig+11-112
...@@ -4,6 +4,9 @@ const assert = debug.assert;...@@ -4,6 +4,9 @@ const assert = debug.assert;
4const testing = std.testing;4const testing = std.testing;
5const math = std.math;5const math = std.math;
6const mem = std.mem;6const mem = std.mem;
7const meta = std.meta;
8const autoHash = std.hash.autoHash;
9const Wyhash = std.hash.Wyhash;
7const Allocator = mem.Allocator;10const Allocator = mem.Allocator;
8const builtin = @import("builtin");11const builtin = @import("builtin");
912
...@@ -448,15 +451,17 @@ test "iterator hash map" {...@@ -448,15 +451,17 @@ test "iterator hash map" {
448 try reset_map.putNoClobber(2, 22);451 try reset_map.putNoClobber(2, 22);
449 try reset_map.putNoClobber(3, 33);452 try reset_map.putNoClobber(3, 33);
450453
454 // TODO this test depends on the hashing algorithm, because it assumes the
455 // order of the elements in the hashmap. This should not be the case.
451 var keys = [_]i32{456 var keys = [_]i32{
457 1,
452 3,458 3,
453 2,459 2,
454 1,
455 };460 };
456 var values = [_]i32{461 var values = [_]i32{
462 11,
457 33,463 33,
458 22,464 22,
459 11,
460 };465 };
461466
462 var it = reset_map.iterator();467 var it = reset_map.iterator();
...@@ -518,8 +523,9 @@ pub fn getTrivialEqlFn(comptime K: type) (fn (K, K) bool) {...@@ -518,8 +523,9 @@ pub fn getTrivialEqlFn(comptime K: type) (fn (K, K) bool) {
518pub fn getAutoHashFn(comptime K: type) (fn (K) u32) {523pub fn getAutoHashFn(comptime K: type) (fn (K) u32) {
519 return struct {524 return struct {
520 fn hash(key: K) u32 {525 fn hash(key: K) u32 {
521 comptime var rng = comptime std.rand.DefaultPrng.init(0);526 var hasher = Wyhash.init(0);
522 return autoHash(key, &rng.random, u32);527 autoHash(&hasher, key);
528 return @truncate(u32, hasher.final());
523 }529 }
524 }.hash;530 }.hash;
525}531}
...@@ -527,114 +533,7 @@ pub fn getAutoHashFn(comptime K: type) (fn (K) u32) {...@@ -527,114 +533,7 @@ pub fn getAutoHashFn(comptime K: type) (fn (K) u32) {
527pub fn getAutoEqlFn(comptime K: type) (fn (K, K) bool) {533pub fn getAutoEqlFn(comptime K: type) (fn (K, K) bool) {
528 return struct {534 return struct {
529 fn eql(a: K, b: K) bool {535 fn eql(a: K, b: K) bool {
530 return autoEql(a, b);536 return meta.eql(a, b);
531 }537 }
532 }.eql;538 }.eql;
533}539}
534
535// TODO improve these hash functions
536pub fn autoHash(key: var, comptime rng: *std.rand.Random, comptime HashInt: type) HashInt {
537 switch (@typeInfo(@typeOf(key))) {
538 builtin.TypeId.NoReturn,
539 builtin.TypeId.Opaque,
540 builtin.TypeId.Undefined,
541 builtin.TypeId.ArgTuple,
542 => @compileError("cannot hash this type"),
543
544 builtin.TypeId.Void,
545 builtin.TypeId.Null,
546 => return 0,
547
548 builtin.TypeId.Int => |info| {
549 const unsigned_x = @bitCast(@IntType(false, info.bits), key);
550 if (info.bits <= HashInt.bit_count) {
551 return HashInt(unsigned_x) ^ comptime rng.scalar(HashInt);
552 } else {
553 return @truncate(HashInt, unsigned_x ^ comptime rng.scalar(@typeOf(unsigned_x)));
554 }
555 },
556
557 builtin.TypeId.Float => |info| {
558 return autoHash(@bitCast(@IntType(false, info.bits), key), rng, HashInt);
559 },
560 builtin.TypeId.Bool => return autoHash(@boolToInt(key), rng, HashInt),
561 builtin.TypeId.Enum => return autoHash(@enumToInt(key), rng, HashInt),
562 builtin.TypeId.ErrorSet => return autoHash(@errorToInt(key), rng, HashInt),
563 builtin.TypeId.Promise, builtin.TypeId.Fn => return autoHash(@ptrToInt(key), rng, HashInt),
564
565 builtin.TypeId.BoundFn,
566 builtin.TypeId.ComptimeFloat,
567 builtin.TypeId.ComptimeInt,
568 builtin.TypeId.Type,
569 builtin.TypeId.EnumLiteral,
570 => return 0,
571
572 builtin.TypeId.Pointer => |info| switch (info.size) {
573 builtin.TypeInfo.Pointer.Size.One => @compileError("TODO auto hash for single item pointers"),
574 builtin.TypeInfo.Pointer.Size.Many => @compileError("TODO auto hash for many item pointers"),
575 builtin.TypeInfo.Pointer.Size.C => @compileError("TODO auto hash C pointers"),
576 builtin.TypeInfo.Pointer.Size.Slice => {
577 const interval = std.math.max(1, key.len / 256);
578 var i: usize = 0;
579 var h = comptime rng.scalar(HashInt);
580 while (i < key.len) : (i += interval) {
581 h ^= autoHash(key[i], rng, HashInt);
582 }
583 return h;
584 },
585 },
586
587 builtin.TypeId.Optional => @compileError("TODO auto hash for optionals"),
588 builtin.TypeId.Array => @compileError("TODO auto hash for arrays"),
589 builtin.TypeId.Vector => @compileError("TODO auto hash for vectors"),
590 builtin.TypeId.Struct => @compileError("TODO auto hash for structs"),
591 builtin.TypeId.Union => @compileError("TODO auto hash for unions"),
592 builtin.TypeId.ErrorUnion => @compileError("TODO auto hash for unions"),
593 }
594}
595
596pub fn autoEql(a: var, b: @typeOf(a)) bool {
597 switch (@typeInfo(@typeOf(a))) {
598 builtin.TypeId.NoReturn,
599 builtin.TypeId.Opaque,
600 builtin.TypeId.Undefined,
601 builtin.TypeId.ArgTuple,
602 => @compileError("cannot test equality of this type"),
603 builtin.TypeId.Void,
604 builtin.TypeId.Null,
605 => return true,
606 builtin.TypeId.Bool,
607 builtin.TypeId.Int,
608 builtin.TypeId.Float,
609 builtin.TypeId.ComptimeFloat,
610 builtin.TypeId.ComptimeInt,
611 builtin.TypeId.EnumLiteral,
612 builtin.TypeId.Promise,
613 builtin.TypeId.Enum,
614 builtin.TypeId.BoundFn,
615 builtin.TypeId.Fn,
616 builtin.TypeId.ErrorSet,
617 builtin.TypeId.Type,
618 => return a == b,
619
620 builtin.TypeId.Pointer => |info| switch (info.size) {
621 builtin.TypeInfo.Pointer.Size.One => @compileError("TODO auto eql for single item pointers"),
622 builtin.TypeInfo.Pointer.Size.Many => @compileError("TODO auto eql for many item pointers"),
623 builtin.TypeInfo.Pointer.Size.C => @compileError("TODO auto eql for C pointers"),
624 builtin.TypeInfo.Pointer.Size.Slice => {
625 if (a.len != b.len) return false;
626 for (a) |a_item, i| {
627 if (!autoEql(a_item, b[i])) return false;
628 }
629 return true;
630 },
631 },
632
633 builtin.TypeId.Optional => @compileError("TODO auto eql for optionals"),
634 builtin.TypeId.Array => @compileError("TODO auto eql for arrays"),
635 builtin.TypeId.Struct => @compileError("TODO auto eql for structs"),
636 builtin.TypeId.Union => @compileError("TODO auto eql for unions"),
637 builtin.TypeId.ErrorUnion => @compileError("TODO auto eql for unions"),
638 builtin.TypeId.Vector => @compileError("TODO auto eql for vectors"),
639 }
640}
std/http/headers.zig+11-1
...@@ -102,9 +102,19 @@ test "HeaderEntry" {...@@ -102,9 +102,19 @@ test "HeaderEntry" {
102 testing.expectEqualSlices(u8, "x", e.value);102 testing.expectEqualSlices(u8, "x", e.value);
103}103}
104104
105fn stringEql(a: []const u8, b: []const u8) bool {
106 if (a.len != b.len) return false;
107 if (a.ptr == b.ptr) return true;
108 return mem.compare(u8, a, b) == .Equal;
109}
110
111fn stringHash(s: []const u8) u32 {
112 return @truncate(u32, std.hash.Wyhash.hash(0, s));
113}
114
105const HeaderList = std.ArrayList(HeaderEntry);115const HeaderList = std.ArrayList(HeaderEntry);
106const HeaderIndexList = std.ArrayList(usize);116const HeaderIndexList = std.ArrayList(usize);
107const HeaderIndex = std.AutoHashMap([]const u8, HeaderIndexList);117const HeaderIndex = std.HashMap([]const u8, HeaderIndexList, stringHash, stringEql);
108118
109pub const Headers = struct {119pub const Headers = struct {
110 // the owned header field name is stored in the index as part of the key120 // the owned header field name is stored in the index as part of the key