authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-09-01 19:09:05+12:00
committergravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-09-02 15:37:49+12:00
log26d61812a8dbad204517d00d3a1f0e52275eceeb
treeeffc228521adc62aa18e87565be05414e7c11aa2
parentbb2eb4443034d0a8c051f5eef184062078502ae8

std/hash: add smhasher verification tests

Not all hashes are added just yet as these need to be generated manually from reference implementations as they are not included by default in smhasher.

6 files changed, 59 insertions(+), 72 deletions(-)

lib/std/hash/cityhash.zig+8-51
......@@ -342,64 +342,21 @@ pub const CityHash64 = struct {
342342 }
343343};
344344
345fn SMHasherTest(comptime hash_fn: anytype) u32 {
346 const HashResult = @typeInfo(@TypeOf(hash_fn)).Fn.return_type.?;
347
348 var key: [256]u8 = undefined;
349 var hashes_bytes: [256 * @sizeOf(HashResult)]u8 = undefined;
350
351 @memset(&key, 0);
352 @memset(&hashes_bytes, 0);
353
354 var i: u32 = 0;
355 while (i < 256) : (i += 1) {
356 key[i] = @as(u8, @intCast(i));
357
358 var h: HashResult = hash_fn(key[0..i], 256 - i);
359
360 // comptime can't really do reinterpret casting yet,
361 // so we need to write the bytes manually.
362 for (hashes_bytes[i * @sizeOf(HashResult) ..][0..@sizeOf(HashResult)]) |*byte| {
363 byte.* = @as(u8, @truncate(h));
364 h = h >> 8;
365 }
366 }
367
368 return @as(u32, @truncate(hash_fn(&hashes_bytes, 0)));
369}
370
371345fn CityHash32hashIgnoreSeed(str: []const u8, seed: u32) u32 {
372346 _ = seed;
373347 return CityHash32.hash(str);
374348}
375349
350const verify = @import("verify.zig");
351
376352test "cityhash32" {
377 const Test = struct {
378 fn doTest() !void {
379 // Note: SMHasher doesn't provide a 32bit version of the algorithm.
380 // Note: The implementation was verified against the Google Abseil version.
381 try std.testing.expectEqual(SMHasherTest(CityHash32hashIgnoreSeed), 0x68254F81);
382 try std.testing.expectEqual(SMHasherTest(CityHash32hashIgnoreSeed), 0x68254F81);
383 }
384 };
385 try Test.doTest();
386 // TODO This is uncommented to prevent OOM on the CI server. Re-enable this test
387 // case once we ship stage2.
388 //@setEvalBranchQuota(50000);
389 //comptime Test.doTest();
353 // Note: SMHasher doesn't provide a 32bit version of the algorithm.
354 // Note: The implementation was verified against the Google Abseil version.
355 try std.testing.expectEqual(verify.smhasher(CityHash32hashIgnoreSeed), 0x68254F81);
390356}
391357
392358test "cityhash64" {
393 const Test = struct {
394 fn doTest() !void {
395 // Note: This is not compliant with the SMHasher implementation of CityHash64!
396 // Note: The implementation was verified against the Google Abseil version.
397 try std.testing.expectEqual(SMHasherTest(CityHash64.hashWithSeed), 0x5FABC5C5);
398 }
399 };
400 try Test.doTest();
401 // TODO This is uncommented to prevent OOM on the CI server. Re-enable this test
402 // case once we ship stage2.
403 //@setEvalBranchQuota(50000);
404 //comptime Test.doTest();
359 // Note: This is not compliant with the SMHasher implementation of CityHash64!
360 // Note: The implementation was verified against the Google Abseil version.
361 try std.testing.expectEqual(verify.smhasher(CityHash64.hashWithSeed), 0x5FABC5C5);
405362}
lib/std/hash/fnv.zig+1-1
......@@ -4,7 +4,7 @@
44//
55// https://tools.ietf.org/html/draft-eastlake-fnv-14
66
7const std = @import("../std.zig");
7const std = @import("std");
88const testing = std.testing;
99
1010pub const Fnv1a_32 = Fnv1a(u32, 0x01000193, 0x811c9dc5);
lib/std/hash/murmur.zig+4-20
......@@ -279,26 +279,10 @@ pub const Murmur3_32 = struct {
279279 }
280280};
281281
282fn SMHasherTest(comptime hash_fn: anytype, comptime hashbits: u32) u32 {
283 const hashbytes = hashbits / 8;
284 var key: [256]u8 = [1]u8{0} ** 256;
285 var hashes: [hashbytes * 256]u8 = [1]u8{0} ** (hashbytes * 256);
286
287 var i: u32 = 0;
288 while (i < 256) : (i += 1) {
289 key[i] = @as(u8, @truncate(i));
290
291 var h = hash_fn(key[0..i], 256 - i);
292 if (native_endian == .Big)
293 h = @byteSwap(h);
294 @memcpy(hashes[i * hashbytes ..][0..hashbytes], @as([*]u8, @ptrCast(&h)));
295 }
296
297 return @as(u32, @truncate(hash_fn(&hashes, 0)));
298}
282const verify = @import("verify.zig");
299283
300284test "murmur2_32" {
301 try testing.expectEqual(SMHasherTest(Murmur2_32.hashWithSeed, 32), 0x27864C1E);
285 try testing.expectEqual(verify.smhasher(Murmur2_32.hashWithSeed), 0x27864C1E);
302286 var v0: u32 = 0x12345678;
303287 var v1: u64 = 0x1234567812345678;
304288 var v0le: u32 = v0;
......@@ -312,7 +296,7 @@ test "murmur2_32" {
312296}
313297
314298test "murmur2_64" {
315 try std.testing.expectEqual(SMHasherTest(Murmur2_64.hashWithSeed, 64), 0x1F0D3804);
299 try std.testing.expectEqual(verify.smhasher(Murmur2_64.hashWithSeed), 0x1F0D3804);
316300 var v0: u32 = 0x12345678;
317301 var v1: u64 = 0x1234567812345678;
318302 var v0le: u32 = v0;
......@@ -326,7 +310,7 @@ test "murmur2_64" {
326310}
327311
328312test "murmur3_32" {
329 try std.testing.expectEqual(SMHasherTest(Murmur3_32.hashWithSeed, 32), 0xB0F57EE3);
313 try std.testing.expectEqual(verify.smhasher(Murmur3_32.hashWithSeed), 0xB0F57EE3);
330314 var v0: u32 = 0x12345678;
331315 var v1: u64 = 0x1234567812345678;
332316 var v0le: u32 = v0;
lib/std/hash/verify.zig created+35
......@@ -0,0 +1,35 @@
1const std = @import("std");
2
3fn hashMaybeSeed(comptime hash_fn: anytype, seed: anytype, buf: []const u8) @typeInfo(@TypeOf(hash_fn)).Fn.return_type.? {
4 const HashFn = @typeInfo(@TypeOf(hash_fn)).Fn;
5 if (HashFn.params.len > 1) {
6 if (@typeInfo(HashFn.params[0].type.?) == .Int) {
7 return hash_fn(@intCast(seed), buf);
8 } else {
9 return hash_fn(buf, @intCast(seed));
10 }
11 } else {
12 return hash_fn(buf);
13 }
14}
15
16// Returns a verification code, the same as user by SMHasher.
17//
18// Hash keys of the form {0}, {0,1}, {0,1,2}... up to N=255, using 256-N as seed.
19// First four-bytes of the hash, interpreted as little-endian is the verification code.
20pub fn smhasher(comptime hash_fn: anytype) u32 {
21 const HashFnTy = @typeInfo(@TypeOf(hash_fn)).Fn;
22 const HashResult = HashFnTy.return_type.?;
23 const hash_size = @sizeOf(HashResult);
24
25 var buf: [256]u8 = undefined;
26 var buf_all: [256 * hash_size]u8 = undefined;
27
28 for (0..256) |i| {
29 buf[i] = @intCast(i);
30 const h = hashMaybeSeed(hash_fn, 256 - i, buf[0..i]);
31 std.mem.writeIntLittle(HashResult, buf_all[i * hash_size ..][0..hash_size], h);
32 }
33
34 return @truncate(hashMaybeSeed(hash_fn, 0, buf_all[0..]));
35}
lib/std/hash/wyhash.zig+5
......@@ -196,6 +196,7 @@ pub const Wyhash = struct {
196196 }
197197};
198198
199const verify = @import("verify.zig");
199200const expectEqual = std.testing.expectEqual;
200201
201202const TestVector = struct {
......@@ -229,6 +230,10 @@ test "test vectors at comptime" {
229230 }
230231}
231232
233test "smhasher" {
234 try expectEqual(verify.smhasher(Wyhash.hash), 0xBD5E840C);
235}
236
232237test "test vectors streaming" {
233238 const step = 5;
234239
lib/std/hash/xxhash.zig+6
......@@ -438,6 +438,8 @@ fn validateType(comptime T: type) void {
438438 }
439439}
440440
441const verify = @import("verify.zig");
442
441443fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64) !void {
442444 try expectEqual(expected, H.hash(0, input));
443445
......@@ -455,6 +457,8 @@ test "xxhash64" {
455457 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0xcfe1f278fa89835c);
456458 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0xaaa46907d3047814);
457459 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0xe04a477f19ee145d);
460
461 try expectEqual(verify.smhasher(H.hash), 0x024B7CF4);
458462}
459463
460464test "xxhash32" {
......@@ -467,4 +471,6 @@ test "xxhash32" {
467471 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0x63a14d5f);
468472 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x9c285e64);
469473 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0x9c05f475);
474
475 try expectEqual(verify.smhasher(H.hash), 0xBA88B743);
470476}