authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-09-13 18:22:56-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-09-13 18:22:56-04:00
log0e2f002a7b45acb5ed62365b9290b09912e5c709
tree203f4aa2b6bbe5811d8bcbf374e0521ba971ea59
parent10a99f8f64bf7cf336c990b90c72d81f5f767b4d
parentf056e01c2398d47673e0ac65fe6f294a35a35f7c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17046 from tiehuis/improve-hash-tests

improve std/hash test coverage

10 files changed, 202 insertions(+), 112 deletions(-)

lib/std/hash/adler.zig+7-1
......@@ -3,7 +3,7 @@
33// https://tools.ietf.org/html/rfc1950#section-9
44// https://github.com/madler/zlib/blob/master/adler32.c
55
6const std = @import("../std.zig");
6const std = @import("std");
77const testing = std.testing;
88
99pub const Adler32 = struct {
......@@ -126,3 +126,9 @@ test "adler32 very long with variation" {
126126
127127 try testing.expectEqual(@as(u32, 0x5af38d6e), std.hash.Adler32.hash(long[0..]));
128128}
129
130const verify = @import("verify.zig");
131
132test "adler32 iterative" {
133 try verify.iterativeApi(Adler32);
134}
lib/std/hash/cityhash.zig+16-45
......@@ -342,64 +342,35 @@ pub const CityHash64 = struct {
342342 }
343343};
344344
345fn SMHasherTest(comptime hash_fn: anytype) u32 {
346 const HashResult = @typeInfo(@TypeOf(hash_fn)).Fn.return_type.?;
347
348 var key: [256]u8 = undefined;
349 var hashes_bytes: [256 * @sizeOf(HashResult)]u8 = undefined;
350
351 @memset(&key, 0);
352 @memset(&hashes_bytes, 0);
353
354 var i: u32 = 0;
355 while (i < 256) : (i += 1) {
356 key[i] = @as(u8, @intCast(i));
357
358 var h: HashResult = hash_fn(key[0..i], 256 - i);
359
360 // comptime can't really do reinterpret casting yet,
361 // so we need to write the bytes manually.
362 for (hashes_bytes[i * @sizeOf(HashResult) ..][0..@sizeOf(HashResult)]) |*byte| {
363 byte.* = @as(u8, @truncate(h));
364 h = h >> 8;
365 }
366 }
367
368 return @as(u32, @truncate(hash_fn(&hashes_bytes, 0)));
369}
370
371345fn CityHash32hashIgnoreSeed(str: []const u8, seed: u32) u32 {
372346 _ = seed;
373347 return CityHash32.hash(str);
374348}
375349
350const verify = @import("verify.zig");
351
376352test "cityhash32" {
377353 const Test = struct {
378 fn doTest() !void {
379 // Note: SMHasher doesn't provide a 32bit version of the algorithm.
380 // Note: The implementation was verified against the Google Abseil version.
381 try std.testing.expectEqual(SMHasherTest(CityHash32hashIgnoreSeed), 0x68254F81);
382 try std.testing.expectEqual(SMHasherTest(CityHash32hashIgnoreSeed), 0x68254F81);
354 fn do() !void {
355 // SMHasher doesn't provide a 32bit version of the algorithm.
356 // The implementation was verified against the Google Abseil version.
357 try std.testing.expectEqual(verify.smhasher(CityHash32hashIgnoreSeed), 0x68254F81);
383358 }
384359 };
385 try Test.doTest();
386 // TODO This is uncommented to prevent OOM on the CI server. Re-enable this test
387 // case once we ship stage2.
388 //@setEvalBranchQuota(50000);
389 //comptime Test.doTest();
360 try Test.do();
361 @setEvalBranchQuota(75000);
362 try comptime Test.do();
390363}
391364
392365test "cityhash64" {
393366 const Test = struct {
394 fn doTest() !void {
395 // Note: This is not compliant with the SMHasher implementation of CityHash64!
396 // Note: The implementation was verified against the Google Abseil version.
397 try std.testing.expectEqual(SMHasherTest(CityHash64.hashWithSeed), 0x5FABC5C5);
367 fn do() !void {
368 // This is not compliant with the SMHasher implementation of CityHash64!
369 // The implementation was verified against the Google Abseil version.
370 try std.testing.expectEqual(verify.smhasher(CityHash64.hashWithSeed), 0x5FABC5C5);
398371 }
399372 };
400 try Test.doTest();
401 // TODO This is uncommented to prevent OOM on the CI server. Re-enable this test
402 // case once we ship stage2.
403 //@setEvalBranchQuota(50000);
404 //comptime Test.doTest();
373 try Test.do();
374 @setEvalBranchQuota(75000);
375 try comptime Test.do();
405376}
lib/std/hash/crc.zig+11-1
......@@ -5,7 +5,7 @@
55// - Crc32SmallWithPoly uses only 64 bytes of memory but is slower. Be aware that this is
66// still moderately fast just slow relative to the slicing approach.
77
8const std = @import("../std.zig");
8const std = @import("std");
99const builtin = @import("builtin");
1010const debug = std.debug;
1111const testing = std.testing;
......@@ -194,6 +194,8 @@ pub fn Crc32WithPoly(comptime poly: Polynomial) type {
194194 };
195195}
196196
197const verify = @import("verify.zig");
198
197199test "crc32 ieee" {
198200 const Crc32Ieee = Crc32WithPoly(.IEEE);
199201
......@@ -210,6 +212,10 @@ test "crc32 castagnoli" {
210212 try testing.expect(Crc32Castagnoli.hash("abc") == 0x364b3fb7);
211213}
212214
215test "crc32 iterative" {
216 try verify.iterativeApi(Crc32WithPoly(.IEEE));
217}
218
213219// half-byte lookup table implementation.
214220pub fn Crc32SmallWithPoly(comptime poly: Polynomial) type {
215221 return struct {
......@@ -258,6 +264,10 @@ pub fn Crc32SmallWithPoly(comptime poly: Polynomial) type {
258264 };
259265}
260266
267test "small crc32 iterative" {
268 try verify.iterativeApi(Crc32SmallWithPoly(.IEEE));
269}
270
261271test "small crc32 ieee" {
262272 const Crc32Ieee = Crc32SmallWithPoly(.IEEE);
263273
lib/std/hash/crc/catalog_test.zig+1-1
......@@ -1,6 +1,6 @@
11//! This file is auto-generated by tools/update_crc_catalog.zig.
22
3const std = @import("../../std.zig");
3const std = @import("std");
44const testing = std.testing;
55const catalog = @import("catalog.zig");
66
lib/std/hash/fnv.zig+6-1
......@@ -4,7 +4,7 @@
44//
55// https://tools.ietf.org/html/draft-eastlake-fnv-14
66
7const std = @import("../std.zig");
7const std = @import("std");
88const testing = std.testing;
99
1010pub const Fnv1a_32 = Fnv1a(u32, 0x01000193, 0x811c9dc5);
......@@ -40,19 +40,24 @@ fn Fnv1a(comptime T: type, comptime prime: T, comptime offset: T) type {
4040 };
4141}
4242
43const verify = @import("verify.zig");
44
4345test "fnv1a-32" {
4446 try testing.expect(Fnv1a_32.hash("") == 0x811c9dc5);
4547 try testing.expect(Fnv1a_32.hash("a") == 0xe40c292c);
4648 try testing.expect(Fnv1a_32.hash("foobar") == 0xbf9cf968);
49 try verify.iterativeApi(Fnv1a_32);
4750}
4851
4952test "fnv1a-64" {
5053 try testing.expect(Fnv1a_64.hash("") == 0xcbf29ce484222325);
5154 try testing.expect(Fnv1a_64.hash("a") == 0xaf63dc4c8601ec8c);
5255 try testing.expect(Fnv1a_64.hash("foobar") == 0x85944171f73967e8);
56 try verify.iterativeApi(Fnv1a_64);
5357}
5458
5559test "fnv1a-128" {
5660 try testing.expect(Fnv1a_128.hash("") == 0x6c62272e07bb014262b821756295c58d);
5761 try testing.expect(Fnv1a_128.hash("a") == 0xd228cb696f1a8caf78912b704e4a8964);
62 try verify.iterativeApi(Fnv1a_128);
5863}
lib/std/hash/murmur.zig+34-20
......@@ -279,26 +279,9 @@ pub const Murmur3_32 = struct {
279279 }
280280};
281281
282fn SMHasherTest(comptime hash_fn: anytype, comptime hashbits: u32) u32 {
283 const hashbytes = hashbits / 8;
284 var key: [256]u8 = [1]u8{0} ** 256;
285 var hashes: [hashbytes * 256]u8 = [1]u8{0} ** (hashbytes * 256);
286
287 var i: u32 = 0;
288 while (i < 256) : (i += 1) {
289 key[i] = @as(u8, @truncate(i));
290
291 var h = hash_fn(key[0..i], 256 - i);
292 if (native_endian == .Big)
293 h = @byteSwap(h);
294 @memcpy(hashes[i * hashbytes ..][0..hashbytes], @as([*]u8, @ptrCast(&h)));
295 }
296
297 return @as(u32, @truncate(hash_fn(&hashes, 0)));
298}
282const verify = @import("verify.zig");
299283
300284test "murmur2_32" {
301 try testing.expectEqual(SMHasherTest(Murmur2_32.hashWithSeed, 32), 0x27864C1E);
302285 var v0: u32 = 0x12345678;
303286 var v1: u64 = 0x1234567812345678;
304287 var v0le: u32 = v0;
......@@ -311,8 +294,18 @@ test "murmur2_32" {
311294 try testing.expectEqual(Murmur2_32.hash(@as([*]u8, @ptrCast(&v1le))[0..8]), Murmur2_32.hashUint64(v1));
312295}
313296
297test "murmur2_32 smhasher" {
298 const Test = struct {
299 fn do() !void {
300 try testing.expectEqual(verify.smhasher(Murmur2_32.hashWithSeed), 0x27864C1E);
301 }
302 };
303 try Test.do();
304 @setEvalBranchQuota(30000);
305 try comptime Test.do();
306}
307
314308test "murmur2_64" {
315 try std.testing.expectEqual(SMHasherTest(Murmur2_64.hashWithSeed, 64), 0x1F0D3804);
316309 var v0: u32 = 0x12345678;
317310 var v1: u64 = 0x1234567812345678;
318311 var v0le: u32 = v0;
......@@ -325,8 +318,18 @@ test "murmur2_64" {
325318 try testing.expectEqual(Murmur2_64.hash(@as([*]u8, @ptrCast(&v1le))[0..8]), Murmur2_64.hashUint64(v1));
326319}
327320
321test "mumur2_64 smhasher" {
322 const Test = struct {
323 fn do() !void {
324 try std.testing.expectEqual(verify.smhasher(Murmur2_64.hashWithSeed), 0x1F0D3804);
325 }
326 };
327 try Test.do();
328 @setEvalBranchQuota(30000);
329 try comptime Test.do();
330}
331
328332test "murmur3_32" {
329 try std.testing.expectEqual(SMHasherTest(Murmur3_32.hashWithSeed, 32), 0xB0F57EE3);
330333 var v0: u32 = 0x12345678;
331334 var v1: u64 = 0x1234567812345678;
332335 var v0le: u32 = v0;
......@@ -338,3 +341,14 @@ test "murmur3_32" {
338341 try testing.expectEqual(Murmur3_32.hash(@as([*]u8, @ptrCast(&v0le))[0..4]), Murmur3_32.hashUint32(v0));
339342 try testing.expectEqual(Murmur3_32.hash(@as([*]u8, @ptrCast(&v1le))[0..8]), Murmur3_32.hashUint64(v1));
340343}
344
345test "mumur3_32 smhasher" {
346 const Test = struct {
347 fn do() !void {
348 try std.testing.expectEqual(verify.smhasher(Murmur3_32.hashWithSeed), 0xB0F57EE3);
349 }
350 };
351 try Test.do();
352 @setEvalBranchQuota(30000);
353 try comptime Test.do();
354}
lib/std/hash/verify.zig created+62
......@@ -0,0 +1,62 @@
1const std = @import("std");
2
3fn hashMaybeSeed(comptime hash_fn: anytype, seed: anytype, buf: []const u8) @typeInfo(@TypeOf(hash_fn)).Fn.return_type.? {
4 const HashFn = @typeInfo(@TypeOf(hash_fn)).Fn;
5 if (HashFn.params.len > 1) {
6 if (@typeInfo(HashFn.params[0].type.?) == .Int) {
7 return hash_fn(@intCast(seed), buf);
8 } else {
9 return hash_fn(buf, @intCast(seed));
10 }
11 } else {
12 return hash_fn(buf);
13 }
14}
15
16fn initMaybeSeed(comptime Hash: anytype, seed: anytype) Hash {
17 const HashFn = @typeInfo(@TypeOf(Hash.init)).Fn;
18 if (HashFn.params.len == 1) {
19 return Hash.init(@intCast(seed));
20 } else {
21 return Hash.init();
22 }
23}
24
25// Returns a verification code, the same as used by SMHasher.
26//
27// Hash keys of the form {0}, {0,1}, {0,1,2}... up to N=255, using 256-N as seed.
28// First four-bytes of the hash, interpreted as little-endian is the verification code.
29pub fn smhasher(comptime hash_fn: anytype) u32 {
30 const HashFnTy = @typeInfo(@TypeOf(hash_fn)).Fn;
31 const HashResult = HashFnTy.return_type.?;
32 const hash_size = @sizeOf(HashResult);
33
34 var buf: [256]u8 = undefined;
35 var buf_all: [256 * hash_size]u8 = undefined;
36
37 for (0..256) |i| {
38 buf[i] = @intCast(i);
39 const h = hashMaybeSeed(hash_fn, 256 - i, buf[0..i]);
40 std.mem.writeIntLittle(HashResult, buf_all[i * hash_size ..][0..hash_size], h);
41 }
42
43 return @truncate(hashMaybeSeed(hash_fn, 0, buf_all[0..]));
44}
45
46pub fn iterativeApi(comptime Hash: anytype) !void {
47 // Sum(1..32) = 528
48 var buf: [528]u8 = [_]u8{0} ** 528;
49 var len: usize = 0;
50 const seed = 0;
51
52 var hasher = initMaybeSeed(Hash, seed);
53 for (1..32) |i| {
54 const r = hashMaybeSeed(Hash.hash, seed, buf[0 .. len + i]);
55 hasher.update(buf[len..][0..i]);
56 const f1 = hasher.final();
57 const f2 = hasher.final();
58 if (f1 != f2) return error.IterativeHashWasNotIdempotent;
59 if (f1 != r) return error.IterativeHashDidNotMatchDirect;
60 len += i;
61 }
62}
lib/std/hash/wyhash.zig+18-42
......@@ -66,7 +66,7 @@ pub const Wyhash = struct {
6666 }
6767
6868 pub fn final(self: *Wyhash) u64 {
69 var input = self.buf[0..self.buf_len];
69 var input: []const u8 = self.buf[0..self.buf_len];
7070 var newSelf = self.shallowCopy(); // ensure idempotency
7171
7272 if (self.total_len <= 16) {
......@@ -196,6 +196,7 @@ pub const Wyhash = struct {
196196 }
197197};
198198
199const verify = @import("verify.zig");
199200const expectEqual = std.testing.expectEqual;
200201
201202const TestVector = struct {
......@@ -229,51 +230,26 @@ test "test vectors at comptime" {
229230 }
230231}
231232
232test "test vectors streaming" {
233 const step = 5;
234
235 for (vectors) |e| {
236 var wh = Wyhash.init(e.seed);
237 var i: usize = 0;
238 while (i < e.input.len) : (i += step) {
239 const len = if (i + step > e.input.len) e.input.len - i else step;
240 wh.update(e.input[i..][0..len]);
233test "smhasher" {
234 const Test = struct {
235 fn do() !void {
236 try expectEqual(verify.smhasher(Wyhash.hash), 0xBD5E840C);
241237 }
242 try expectEqual(e.expected, wh.final());
243 }
244}
245
246test "test ensure idempotent final call" {
247 const e: TestVector = .{ .seed = 6, .expected = 0xc39cab13b115aad3, .input = "12345678901234567890123456789012345678901234567890123456789012345678901234567890" };
248 var wh = Wyhash.init(e.seed);
249 wh.update(e.input);
250
251 for (0..10) |_| {
252 try expectEqual(e.expected, wh.final());
253 }
238 };
239 try Test.do();
240 @setEvalBranchQuota(50000);
241 try comptime Test.do();
254242}
255243
256test "iterative non-divisible update" {
257 var buf: [8192]u8 = undefined;
258 for (&buf, 0..) |*e, i| {
259 e.* = @as(u8, @truncate(i));
260 }
261
262 const seed = 0x128dad08f;
263
264 var end: usize = 32;
265 while (end < buf.len) : (end += 32) {
266 const non_iterative_hash = Wyhash.hash(seed, buf[0..end]);
267
268 var wy = Wyhash.init(seed);
269 var i: usize = 0;
270 while (i < end) : (i += 33) {
271 wy.update(buf[i..@min(i + 33, end)]);
244test "iterative api" {
245 const Test = struct {
246 fn do() !void {
247 try verify.iterativeApi(Wyhash);
272248 }
273 const iterative_hash = wy.final();
274
275 try std.testing.expectEqual(iterative_hash, non_iterative_hash);
276 }
249 };
250 try Test.do();
251 @setEvalBranchQuota(50000);
252 try comptime Test.do();
277253}
278254
279255test "iterative maintains last sixteen" {
lib/std/hash/xxhash.zig+46
......@@ -438,6 +438,8 @@ fn validateType(comptime T: type) void {
438438 }
439439}
440440
441const verify = @import("verify.zig");
442
441443fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64) !void {
442444 try expectEqual(expected, H.hash(0, input));
443445
......@@ -457,6 +459,28 @@ test "xxhash64" {
457459 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0xe04a477f19ee145d);
458460}
459461
462test "xxhash64 smhasher" {
463 const Test = struct {
464 fn do() !void {
465 try expectEqual(verify.smhasher(XxHash64.hash), 0x024B7CF4);
466 }
467 };
468 try Test.do();
469 @setEvalBranchQuota(75000);
470 comptime try Test.do();
471}
472
473test "xxhash64 iterative api" {
474 const Test = struct {
475 fn do() !void {
476 try verify.iterativeApi(XxHash64);
477 }
478 };
479 try Test.do();
480 @setEvalBranchQuota(30000);
481 comptime try Test.do();
482}
483
460484test "xxhash32" {
461485 const H = XxHash32;
462486
......@@ -468,3 +492,25 @@ test "xxhash32" {
468492 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x9c285e64);
469493 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0x9c05f475);
470494}
495
496test "xxhash32 smhasher" {
497 const Test = struct {
498 fn do() !void {
499 try expectEqual(verify.smhasher(XxHash32.hash), 0xBA88B743);
500 }
501 };
502 try Test.do();
503 @setEvalBranchQuota(75000);
504 comptime try Test.do();
505}
506
507test "xxhash32 iterative api" {
508 const Test = struct {
509 fn do() !void {
510 try verify.iterativeApi(XxHash32);
511 }
512 };
513 try Test.do();
514 @setEvalBranchQuota(30000);
515 comptime try Test.do();
516}
tools/update_crc_catalog.zig+1-1
......@@ -55,7 +55,7 @@ pub fn main() anyerror!void {
5555 try test_writer.writeAll(
5656 \\//! This file is auto-generated by tools/update_crc_catalog.zig.
5757 \\
58 \\const std = @import("../../std.zig");
58 \\const std = @import("std");
5959 \\const testing = std.testing;
6060 \\const catalog = @import("catalog.zig");
6161 \\