authorgravatar for 50984334+francescoalemanno@users.noreply.github.comFrancesco Alemanno <50984334+francescoalemanno@users.noreply.github.com> 2024-11-02 00:11:44+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-11-24 15:29:20-08:00
logae6c24b49053853686699913297764361a6d6303
tree64ad8ec23158fd7cffe9e4a23dc87287b85ec6c6
parentd09fd249c090c68f6bba9731572e52f51afa7536

std.hash.int: better handle odd bit sizes

Uses the non rational solution of a quadratic, I made it work up to 256 bits, added Mathematica code in case anyone wants to verify the magic constant. integers between sizes 3...15 were affected by fatal bias, it is best to make them pass through the generic solution. Thanks to RetroDev256 & Andrew feedback.

1 files changed, 65 insertions(+), 60 deletions(-)

lib/std/hash.zig+65-60
...@@ -37,78 +37,83 @@ pub const XxHash3 = xxhash.XxHash3;...@@ -37,78 +37,83 @@ pub const XxHash3 = xxhash.XxHash3;
37pub const XxHash64 = xxhash.XxHash64;37pub const XxHash64 = xxhash.XxHash64;
38pub const XxHash32 = xxhash.XxHash32;38pub const XxHash32 = xxhash.XxHash32;
3939
40/// Deprecated in favor of `int`.40/// Easy & fast hash function for integer types
41pub fn uint32(input: u32) u32 {
42 return int(input);
43}
44
45/// Applies a bit-mangling transformation to an unsigned integer type `T`.
46/// Optimized per type: for `u16` and `u32`, Skeeto's xorshift-multiply; for `u64`, Maiga's mx3.
47/// Falls back on an avalanche pattern for other integer types, ensuring high entropy.
48pub fn int(input: anytype) @TypeOf(input) {41pub fn int(input: anytype) @TypeOf(input) {
42 // This function is only intended for integer types
49 const info = @typeInfo(@TypeOf(input)).int;43 const info = @typeInfo(@TypeOf(input)).int;
50 if (info.signedness == .signed) {44 const bits = info.bits;
51 const Unsigned = @Type(.{ .int = .{ .signedness = .unsigned, .bits = info.bits } });45 // Convert input to unsigned integer (easier to deal with)
52 const casted: Unsigned = @bitCast(input);46 const Uint = @Type(.{ .int = .{ .bits = bits, .signedness = .unsigned } });
53 return @bitCast(int(casted));47 const u_input: Uint = @bitCast(input);
54 } else if (info.bits < 4) {48 if (bits > 256) @compileError("bit widths > 256 are unsupported, use std.hash.autoHash functionality.");
55 return @truncate(int(@as(u4, input)));49 // For bit widths that don't have a dedicated function, use a heuristic
56 }50 // construction with a multiplier suited to diffusion -
57 var x = input;51 // a mod 2^bits where a^2 - 46 * a + 1 = 0 mod 2^(bits + 4),
58 switch (info.bits) {52 // on Mathematica: bits = 256; BaseForm[Solve[1 - 46 a + a^2 == 0, a, Modulus -> 2^(bits + 4)][[-1]][[1]][[2]], 16]
59 16 => {53 const mult: Uint = @truncate(0xfac2e27ed2036860a062b5f264d80a512b00aa459b448bf1eca24d41c96f59e5b);
60 // https://github.com/skeeto/hash-prospector54 // The bit width of the input integer determines how to hash it
61 // 3-round xorshift-multiply (-Xn3)55 const output = switch (bits) {
62 // bias = 0.004597670901882060256 0...2 => u_input *% mult,
63 x = (x ^ (x >> 7)) *% 0x2993;57 16 => uint16(u_input),
64 x = (x ^ (x >> 5)) *% 0xe877;58 32 => uint32(u_input),
65 x = (x ^ (x >> 9)) *% 0x0235;59 64 => uint64(u_input),
66 x = x ^ (x >> 10);60 else => blk: {
67 },61 var x: Uint = u_input;
68 32 => {62 inline for (0..4) |_| {
69 // https://github.com/skeeto/hash-prospector63 x ^= x >> (bits / 2);
70 x = (x ^ (x >> 17)) *% 0xed5ad4bb;64 x *%= mult;
71 x = (x ^ (x >> 11)) *% 0xac4c1b51;
72 x = (x ^ (x >> 15)) *% 0x31848bab;
73 x = x ^ (x >> 14);
74 },
75 64 => {
76 // https://github.com/jonmaiga/mx3
77 // https://github.com/jonmaiga/mx3/blob/48924ee743d724aea2cafd2b4249ef8df57fa8b9/mx3.h#L17
78 const c = 0xbea225f9eb34556d;
79 x = (x ^ (x >> 32)) *% c;
80 x = (x ^ (x >> 29)) *% c;
81 x = (x ^ (x >> 32)) *% c;
82 x = x ^ (x >> 29);
83 },
84 else => {
85 // This construction provides robust avalanche properties, but it is not optimal for any given size.
86 const hsize = info.bits >> 1;
87 const c = comptime blk: {
88 const max = (1 << info.bits) - 1;
89 var mul = 1;
90 while (mul * 3 < max) mul *= 3;
91 break :blk ((mul ^ (mul >> hsize)) | 1);
92 };
93 inline for (0..2) |_| {
94 x = (x ^ (x >> hsize + 1)) *% c;
95 x = (x ^ (x >> hsize - 1)) *% c;
96 }65 }
97 x ^= (x >> hsize);66 break :blk x;
98 },67 },
99 }68 };
69 return @bitCast(output);
70}
71
72/// Source: https://github.com/skeeto/hash-prospector
73fn uint16(input: u16) u16 {
74 var x: u16 = input;
75 x = (x ^ (x >> 7)) *% 0x2993;
76 x = (x ^ (x >> 5)) *% 0xe877;
77 x = (x ^ (x >> 9)) *% 0x0235;
78 x = x ^ (x >> 10);
79 return x;
80}
81
82/// DEPRECATED: use std.hash.int()
83/// Source: https://github.com/skeeto/hash-prospector
84pub fn uint32(input: u32) u32 {
85 var x: u32 = input;
86 x = (x ^ (x >> 17)) *% 0xed5ad4bb;
87 x = (x ^ (x >> 11)) *% 0xac4c1b51;
88 x = (x ^ (x >> 15)) *% 0x31848bab;
89 x = x ^ (x >> 14);
90 return x;
91}
92
93/// Source: https://github.com/jonmaiga/mx3
94fn uint64(input: u64) u64 {
95 var x: u64 = input;
96 const c = 0xbea225f9eb34556d;
97 x = (x ^ (x >> 32)) *% c;
98 x = (x ^ (x >> 29)) *% c;
99 x = (x ^ (x >> 32)) *% c;
100 x = x ^ (x >> 29);
100 return x;101 return x;
101}102}
102103
103test int {104test int {
104 const expectEqual = @import("std").testing.expectEqual;105 const expectEqual = @import("std").testing.expectEqual;
105 try expectEqual(0xC, int(@as(u4, 1)));106 try expectEqual(0x1, int(@as(u1, 1)));
106 try expectEqual(0x4F, int(@as(u8, 1)));107 try expectEqual(0x3, int(@as(u2, 1)));
107 try expectEqual(0x4F, int(@as(i8, 1)));108 try expectEqual(0x4, int(@as(u3, 1)));
109 try expectEqual(0xD6, int(@as(u8, 1)));
108 try expectEqual(0x2880, int(@as(u16, 1)));110 try expectEqual(0x2880, int(@as(u16, 1)));
111 try expectEqual(0x2880, int(@as(i16, 1)));
112 try expectEqual(0x838380, int(@as(u24, 1)));
109 try expectEqual(0x42741D6, int(@as(u32, 1)));113 try expectEqual(0x42741D6, int(@as(u32, 1)));
114 try expectEqual(0x42741D6, int(@as(i32, 1)));
110 try expectEqual(0x71894DE00D9981F, int(@as(u64, 1)));115 try expectEqual(0x71894DE00D9981F, int(@as(u64, 1)));
111 try expectEqual(0x50BC2BB18910C3DE0BAA2CE0D0C5B83E, int(@as(u128, 1)));116 try expectEqual(0x71894DE00D9981F, int(@as(i64, 1)));
112}117}
113118
114test {119test {