authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-09-01 19:09:05+12:00
committergravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2023-09-02 15:37:49+12:00
log26d61812a8dbad204517d00d3a1f0e52275eceeb
treeeffc228521adc62aa18e87565be05414e7c11aa2
parentbb2eb4443034d0a8c051f5eef184062078502ae8

std/hash: add smhasher verification tests

Not all hashes are added just yet as these need to be generated manually from reference implementations as they are not included by default in smhasher.

6 files changed, 59 insertions(+), 72 deletions(-)

lib/std/hash/cityhash.zig+8-51
...@@ -342,64 +342,21 @@ pub const CityHash64 = struct {...@@ -342,64 +342,21 @@ pub const CityHash64 = struct {
342 }342 }
343};343};
344344
345fn SMHasherTest(comptime hash_fn: anytype) u32 {
346 const HashResult = @typeInfo(@TypeOf(hash_fn)).Fn.return_type.?;
347
348 var key: [256]u8 = undefined;
349 var hashes_bytes: [256 * @sizeOf(HashResult)]u8 = undefined;
350
351 @memset(&key, 0);
352 @memset(&hashes_bytes, 0);
353
354 var i: u32 = 0;
355 while (i < 256) : (i += 1) {
356 key[i] = @as(u8, @intCast(i));
357
358 var h: HashResult = hash_fn(key[0..i], 256 - i);
359
360 // comptime can't really do reinterpret casting yet,
361 // so we need to write the bytes manually.
362 for (hashes_bytes[i * @sizeOf(HashResult) ..][0..@sizeOf(HashResult)]) |*byte| {
363 byte.* = @as(u8, @truncate(h));
364 h = h >> 8;
365 }
366 }
367
368 return @as(u32, @truncate(hash_fn(&hashes_bytes, 0)));
369}
370
371fn CityHash32hashIgnoreSeed(str: []const u8, seed: u32) u32 {345fn CityHash32hashIgnoreSeed(str: []const u8, seed: u32) u32 {
372 _ = seed;346 _ = seed;
373 return CityHash32.hash(str);347 return CityHash32.hash(str);
374}348}
375349
350const verify = @import("verify.zig");
351
376test "cityhash32" {352test "cityhash32" {
377 const Test = struct {353 // Note: SMHasher doesn't provide a 32bit version of the algorithm.
378 fn doTest() !void {354 // Note: The implementation was verified against the Google Abseil version.
379 // Note: SMHasher doesn't provide a 32bit version of the algorithm.355 try std.testing.expectEqual(verify.smhasher(CityHash32hashIgnoreSeed), 0x68254F81);
380 // Note: The implementation was verified against the Google Abseil version.
381 try std.testing.expectEqual(SMHasherTest(CityHash32hashIgnoreSeed), 0x68254F81);
382 try std.testing.expectEqual(SMHasherTest(CityHash32hashIgnoreSeed), 0x68254F81);
383 }
384 };
385 try Test.doTest();
386 // TODO This is uncommented to prevent OOM on the CI server. Re-enable this test
387 // case once we ship stage2.
388 //@setEvalBranchQuota(50000);
389 //comptime Test.doTest();
390}356}
391357
392test "cityhash64" {358test "cityhash64" {
393 const Test = struct {359 // Note: This is not compliant with the SMHasher implementation of CityHash64!
394 fn doTest() !void {360 // Note: The implementation was verified against the Google Abseil version.
395 // Note: This is not compliant with the SMHasher implementation of CityHash64!361 try std.testing.expectEqual(verify.smhasher(CityHash64.hashWithSeed), 0x5FABC5C5);
396 // Note: The implementation was verified against the Google Abseil version.
397 try std.testing.expectEqual(SMHasherTest(CityHash64.hashWithSeed), 0x5FABC5C5);
398 }
399 };
400 try Test.doTest();
401 // TODO This is uncommented to prevent OOM on the CI server. Re-enable this test
402 // case once we ship stage2.
403 //@setEvalBranchQuota(50000);
404 //comptime Test.doTest();
405}362}
lib/std/hash/fnv.zig+1-1
...@@ -4,7 +4,7 @@...@@ -4,7 +4,7 @@
4//4//
5// https://tools.ietf.org/html/draft-eastlake-fnv-145// https://tools.ietf.org/html/draft-eastlake-fnv-14
66
7const std = @import("../std.zig");7const std = @import("std");
8const testing = std.testing;8const testing = std.testing;
99
10pub const Fnv1a_32 = Fnv1a(u32, 0x01000193, 0x811c9dc5);10pub const Fnv1a_32 = Fnv1a(u32, 0x01000193, 0x811c9dc5);
lib/std/hash/murmur.zig+4-20
...@@ -279,26 +279,10 @@ pub const Murmur3_32 = struct {...@@ -279,26 +279,10 @@ pub const Murmur3_32 = struct {
279 }279 }
280};280};
281281
282fn SMHasherTest(comptime hash_fn: anytype, comptime hashbits: u32) u32 {282const verify = @import("verify.zig");
283 const hashbytes = hashbits / 8;
284 var key: [256]u8 = [1]u8{0} ** 256;
285 var hashes: [hashbytes * 256]u8 = [1]u8{0} ** (hashbytes * 256);
286
287 var i: u32 = 0;
288 while (i < 256) : (i += 1) {
289 key[i] = @as(u8, @truncate(i));
290
291 var h = hash_fn(key[0..i], 256 - i);
292 if (native_endian == .Big)
293 h = @byteSwap(h);
294 @memcpy(hashes[i * hashbytes ..][0..hashbytes], @as([*]u8, @ptrCast(&h)));
295 }
296
297 return @as(u32, @truncate(hash_fn(&hashes, 0)));
298}
299283
300test "murmur2_32" {284test "murmur2_32" {
301 try testing.expectEqual(SMHasherTest(Murmur2_32.hashWithSeed, 32), 0x27864C1E);285 try testing.expectEqual(verify.smhasher(Murmur2_32.hashWithSeed), 0x27864C1E);
302 var v0: u32 = 0x12345678;286 var v0: u32 = 0x12345678;
303 var v1: u64 = 0x1234567812345678;287 var v1: u64 = 0x1234567812345678;
304 var v0le: u32 = v0;288 var v0le: u32 = v0;
...@@ -312,7 +296,7 @@ test "murmur2_32" {...@@ -312,7 +296,7 @@ test "murmur2_32" {
312}296}
313297
314test "murmur2_64" {298test "murmur2_64" {
315 try std.testing.expectEqual(SMHasherTest(Murmur2_64.hashWithSeed, 64), 0x1F0D3804);299 try std.testing.expectEqual(verify.smhasher(Murmur2_64.hashWithSeed), 0x1F0D3804);
316 var v0: u32 = 0x12345678;300 var v0: u32 = 0x12345678;
317 var v1: u64 = 0x1234567812345678;301 var v1: u64 = 0x1234567812345678;
318 var v0le: u32 = v0;302 var v0le: u32 = v0;
...@@ -326,7 +310,7 @@ test "murmur2_64" {...@@ -326,7 +310,7 @@ test "murmur2_64" {
326}310}
327311
328test "murmur3_32" {312test "murmur3_32" {
329 try std.testing.expectEqual(SMHasherTest(Murmur3_32.hashWithSeed, 32), 0xB0F57EE3);313 try std.testing.expectEqual(verify.smhasher(Murmur3_32.hashWithSeed), 0xB0F57EE3);
330 var v0: u32 = 0x12345678;314 var v0: u32 = 0x12345678;
331 var v1: u64 = 0x1234567812345678;315 var v1: u64 = 0x1234567812345678;
332 var v0le: u32 = v0;316 var v0le: u32 = v0;
lib/std/hash/verify.zig created+35
...@@ -0,0 +1,35 @@
1const std = @import("std");
2
3fn hashMaybeSeed(comptime hash_fn: anytype, seed: anytype, buf: []const u8) @typeInfo(@TypeOf(hash_fn)).Fn.return_type.? {
4 const HashFn = @typeInfo(@TypeOf(hash_fn)).Fn;
5 if (HashFn.params.len > 1) {
6 if (@typeInfo(HashFn.params[0].type.?) == .Int) {
7 return hash_fn(@intCast(seed), buf);
8 } else {
9 return hash_fn(buf, @intCast(seed));
10 }
11 } else {
12 return hash_fn(buf);
13 }
14}
15
16// Returns a verification code, the same as user by SMHasher.
17//
18// Hash keys of the form {0}, {0,1}, {0,1,2}... up to N=255, using 256-N as seed.
19// First four-bytes of the hash, interpreted as little-endian is the verification code.
20pub fn smhasher(comptime hash_fn: anytype) u32 {
21 const HashFnTy = @typeInfo(@TypeOf(hash_fn)).Fn;
22 const HashResult = HashFnTy.return_type.?;
23 const hash_size = @sizeOf(HashResult);
24
25 var buf: [256]u8 = undefined;
26 var buf_all: [256 * hash_size]u8 = undefined;
27
28 for (0..256) |i| {
29 buf[i] = @intCast(i);
30 const h = hashMaybeSeed(hash_fn, 256 - i, buf[0..i]);
31 std.mem.writeIntLittle(HashResult, buf_all[i * hash_size ..][0..hash_size], h);
32 }
33
34 return @truncate(hashMaybeSeed(hash_fn, 0, buf_all[0..]));
35}
lib/std/hash/wyhash.zig+5
...@@ -196,6 +196,7 @@ pub const Wyhash = struct {...@@ -196,6 +196,7 @@ pub const Wyhash = struct {
196 }196 }
197};197};
198198
199const verify = @import("verify.zig");
199const expectEqual = std.testing.expectEqual;200const expectEqual = std.testing.expectEqual;
200201
201const TestVector = struct {202const TestVector = struct {
...@@ -229,6 +230,10 @@ test "test vectors at comptime" {...@@ -229,6 +230,10 @@ test "test vectors at comptime" {
229 }230 }
230}231}
231232
233test "smhasher" {
234 try expectEqual(verify.smhasher(Wyhash.hash), 0xBD5E840C);
235}
236
232test "test vectors streaming" {237test "test vectors streaming" {
233 const step = 5;238 const step = 5;
234239
lib/std/hash/xxhash.zig+6
...@@ -438,6 +438,8 @@ fn validateType(comptime T: type) void {...@@ -438,6 +438,8 @@ fn validateType(comptime T: type) void {
438 }438 }
439}439}
440440
441const verify = @import("verify.zig");
442
441fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64) !void {443fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64) !void {
442 try expectEqual(expected, H.hash(0, input));444 try expectEqual(expected, H.hash(0, input));
443445
...@@ -455,6 +457,8 @@ test "xxhash64" {...@@ -455,6 +457,8 @@ test "xxhash64" {
455 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0xcfe1f278fa89835c);457 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0xcfe1f278fa89835c);
456 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0xaaa46907d3047814);458 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0xaaa46907d3047814);
457 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0xe04a477f19ee145d);459 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0xe04a477f19ee145d);
460
461 try expectEqual(verify.smhasher(H.hash), 0x024B7CF4);
458}462}
459463
460test "xxhash32" {464test "xxhash32" {
...@@ -467,4 +471,6 @@ test "xxhash32" {...@@ -467,4 +471,6 @@ test "xxhash32" {
467 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0x63a14d5f);471 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0x63a14d5f);
468 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x9c285e64);472 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x9c285e64);
469 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0x9c05f475);473 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0x9c05f475);
474
475 try expectEqual(verify.smhasher(H.hash), 0xBA88B743);
470}476}