authorgravatar for 50984334+francescoalemanno@users.noreply.github.comFrancesco Alemanno <50984334+francescoalemanno@users.noreply.github.com> 2024-11-02 00:11:44+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-11-24 15:29:20-08:00
logae6c24b49053853686699913297764361a6d6303
tree64ad8ec23158fd7cffe9e4a23dc87287b85ec6c6
parentd09fd249c090c68f6bba9731572e52f51afa7536

std.hash.int: better handle odd bit sizes

Uses the non rational solution of a quadratic, I made it work up to 256 bits, added Mathematica code in case anyone wants to verify the magic constant. integers between sizes 3...15 were affected by fatal bias, it is best to make them pass through the generic solution. Thanks to RetroDev256 & Andrew feedback.

1 files changed, 65 insertions(+), 60 deletions(-)

lib/std/hash.zig+65-60
......@@ -37,78 +37,83 @@ pub const XxHash3 = xxhash.XxHash3;
3737pub const XxHash64 = xxhash.XxHash64;
3838pub const XxHash32 = xxhash.XxHash32;
3939
40/// Deprecated in favor of `int`.
41pub fn uint32(input: u32) u32 {
42 return int(input);
43}
44
45/// Applies a bit-mangling transformation to an unsigned integer type `T`.
46/// Optimized per type: for `u16` and `u32`, Skeeto's xorshift-multiply; for `u64`, Maiga's mx3.
47/// Falls back on an avalanche pattern for other integer types, ensuring high entropy.
40/// Easy & fast hash function for integer types
4841pub fn int(input: anytype) @TypeOf(input) {
42 // This function is only intended for integer types
4943 const info = @typeInfo(@TypeOf(input)).int;
50 if (info.signedness == .signed) {
51 const Unsigned = @Type(.{ .int = .{ .signedness = .unsigned, .bits = info.bits } });
52 const casted: Unsigned = @bitCast(input);
53 return @bitCast(int(casted));
54 } else if (info.bits < 4) {
55 return @truncate(int(@as(u4, input)));
56 }
57 var x = input;
58 switch (info.bits) {
59 16 => {
60 // https://github.com/skeeto/hash-prospector
61 // 3-round xorshift-multiply (-Xn3)
62 // bias = 0.0045976709018820602
63 x = (x ^ (x >> 7)) *% 0x2993;
64 x = (x ^ (x >> 5)) *% 0xe877;
65 x = (x ^ (x >> 9)) *% 0x0235;
66 x = x ^ (x >> 10);
67 },
68 32 => {
69 // https://github.com/skeeto/hash-prospector
70 x = (x ^ (x >> 17)) *% 0xed5ad4bb;
71 x = (x ^ (x >> 11)) *% 0xac4c1b51;
72 x = (x ^ (x >> 15)) *% 0x31848bab;
73 x = x ^ (x >> 14);
74 },
75 64 => {
76 // https://github.com/jonmaiga/mx3
77 // https://github.com/jonmaiga/mx3/blob/48924ee743d724aea2cafd2b4249ef8df57fa8b9/mx3.h#L17
78 const c = 0xbea225f9eb34556d;
79 x = (x ^ (x >> 32)) *% c;
80 x = (x ^ (x >> 29)) *% c;
81 x = (x ^ (x >> 32)) *% c;
82 x = x ^ (x >> 29);
83 },
84 else => {
85 // This construction provides robust avalanche properties, but it is not optimal for any given size.
86 const hsize = info.bits >> 1;
87 const c = comptime blk: {
88 const max = (1 << info.bits) - 1;
89 var mul = 1;
90 while (mul * 3 < max) mul *= 3;
91 break :blk ((mul ^ (mul >> hsize)) | 1);
92 };
93 inline for (0..2) |_| {
94 x = (x ^ (x >> hsize + 1)) *% c;
95 x = (x ^ (x >> hsize - 1)) *% c;
44 const bits = info.bits;
45 // Convert input to unsigned integer (easier to deal with)
46 const Uint = @Type(.{ .int = .{ .bits = bits, .signedness = .unsigned } });
47 const u_input: Uint = @bitCast(input);
48 if (bits > 256) @compileError("bit widths > 256 are unsupported, use std.hash.autoHash functionality.");
49 // For bit widths that don't have a dedicated function, use a heuristic
50 // construction with a multiplier suited to diffusion -
51 // a mod 2^bits where a^2 - 46 * a + 1 = 0 mod 2^(bits + 4),
52 // on Mathematica: bits = 256; BaseForm[Solve[1 - 46 a + a^2 == 0, a, Modulus -> 2^(bits + 4)][[-1]][[1]][[2]], 16]
53 const mult: Uint = @truncate(0xfac2e27ed2036860a062b5f264d80a512b00aa459b448bf1eca24d41c96f59e5b);
54 // The bit width of the input integer determines how to hash it
55 const output = switch (bits) {
56 0...2 => u_input *% mult,
57 16 => uint16(u_input),
58 32 => uint32(u_input),
59 64 => uint64(u_input),
60 else => blk: {
61 var x: Uint = u_input;
62 inline for (0..4) |_| {
63 x ^= x >> (bits / 2);
64 x *%= mult;
9665 }
97 x ^= (x >> hsize);
66 break :blk x;
9867 },
99 }
68 };
69 return @bitCast(output);
70}
71
72/// Source: https://github.com/skeeto/hash-prospector
73fn uint16(input: u16) u16 {
74 var x: u16 = input;
75 x = (x ^ (x >> 7)) *% 0x2993;
76 x = (x ^ (x >> 5)) *% 0xe877;
77 x = (x ^ (x >> 9)) *% 0x0235;
78 x = x ^ (x >> 10);
79 return x;
80}
81
82/// DEPRECATED: use std.hash.int()
83/// Source: https://github.com/skeeto/hash-prospector
84pub fn uint32(input: u32) u32 {
85 var x: u32 = input;
86 x = (x ^ (x >> 17)) *% 0xed5ad4bb;
87 x = (x ^ (x >> 11)) *% 0xac4c1b51;
88 x = (x ^ (x >> 15)) *% 0x31848bab;
89 x = x ^ (x >> 14);
90 return x;
91}
92
93/// Source: https://github.com/jonmaiga/mx3
94fn uint64(input: u64) u64 {
95 var x: u64 = input;
96 const c = 0xbea225f9eb34556d;
97 x = (x ^ (x >> 32)) *% c;
98 x = (x ^ (x >> 29)) *% c;
99 x = (x ^ (x >> 32)) *% c;
100 x = x ^ (x >> 29);
100101 return x;
101102}
102103
103104test int {
104105 const expectEqual = @import("std").testing.expectEqual;
105 try expectEqual(0xC, int(@as(u4, 1)));
106 try expectEqual(0x4F, int(@as(u8, 1)));
107 try expectEqual(0x4F, int(@as(i8, 1)));
106 try expectEqual(0x1, int(@as(u1, 1)));
107 try expectEqual(0x3, int(@as(u2, 1)));
108 try expectEqual(0x4, int(@as(u3, 1)));
109 try expectEqual(0xD6, int(@as(u8, 1)));
108110 try expectEqual(0x2880, int(@as(u16, 1)));
111 try expectEqual(0x2880, int(@as(i16, 1)));
112 try expectEqual(0x838380, int(@as(u24, 1)));
109113 try expectEqual(0x42741D6, int(@as(u32, 1)));
114 try expectEqual(0x42741D6, int(@as(i32, 1)));
110115 try expectEqual(0x71894DE00D9981F, int(@as(u64, 1)));
111 try expectEqual(0x50BC2BB18910C3DE0BAA2CE0D0C5B83E, int(@as(u128, 1)));
116 try expectEqual(0x71894DE00D9981F, int(@as(i64, 1)));
112117}
113118
114119test {