authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2019-08-27 20:13:57+12:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-08-27 20:13:57+12:00
log326b7b794b01a1194df3785e497460bada756c29
tree9fa2594364a239b0981f6b16ce0ed095fb0f9cf7
parent1df75da918c5bf4bfa3f6836e79040628c533657
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Improve siphash performance for small keys by up to 30% (#3124)

This removes the partial buffer handling from the full slice api. `./benchmark --filter siphash --count 1024` old siphash(1,3) iterative: 3388 MiB/s [67532e53a0d210bf] small keys: 1258 MiB/s [948c91176a000000] siphash(2,4) iterative: 2061 MiB/s [f792d39bff42f819] small keys: 902 MiB/s [e1ecba6614000000] new siphash(1,3) iterative: 3410 MiB/s [67532e53a0d210bf] small keys: 1639 MiB/s [948c91176a000000] siphash(2,4) iterative: 2053 MiB/s [f792d39bff42f819] small keys: 1074 MiB/s [e1ecba6614000000]

1 files changed, 107 insertions(+), 45 deletions(-)

std/hash/siphash.zig+107-45
...@@ -21,7 +21,7 @@ pub fn SipHash128(comptime c_rounds: usize, comptime d_rounds: usize) type {...@@ -21,7 +21,7 @@ pub fn SipHash128(comptime c_rounds: usize, comptime d_rounds: usize) type {
21 return SipHash(u128, c_rounds, d_rounds);21 return SipHash(u128, c_rounds, d_rounds);
22}22}
2323
24fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) type {24fn SipHashStateless(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) type {
25 assert(T == u64 or T == u128);25 assert(T == u64 or T == u128);
26 assert(c_rounds > 0 and d_rounds > 0);26 assert(c_rounds > 0 and d_rounds > 0);
2727
...@@ -34,10 +34,6 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)...@@ -34,10 +34,6 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
34 v1: u64,34 v1: u64,
35 v2: u64,35 v2: u64,
36 v3: u64,36 v3: u64,
37
38 // streaming cache
39 buf: [8]u8,
40 buf_len: usize,
41 msg_len: u8,37 msg_len: u8,
4238
43 pub fn init(key: []const u8) Self {39 pub fn init(key: []const u8) Self {
...@@ -51,9 +47,6 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)...@@ -51,9 +47,6 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
51 .v1 = k1 ^ 0x646f72616e646f6d,47 .v1 = k1 ^ 0x646f72616e646f6d,
52 .v2 = k0 ^ 0x6c7967656e657261,48 .v2 = k0 ^ 0x6c7967656e657261,
53 .v3 = k1 ^ 0x7465646279746573,49 .v3 = k1 ^ 0x7465646279746573,
54
55 .buf = undefined,
56 .buf_len = 0,
57 .msg_len = 0,50 .msg_len = 0,
58 };51 };
5952
...@@ -64,73 +57,66 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)...@@ -64,73 +57,66 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
64 return d;57 return d;
65 }58 }
6659
67 pub fn update(d: *Self, b: []const u8) void {60 pub fn update(self: *Self, b: []const u8) void {
68 var off: usize = 0;61 std.debug.assert(b.len % 8 == 0);
69
70 // Partial from previous.
71 if (d.buf_len != 0 and d.buf_len + b.len > 8) {
72 off += 8 - d.buf_len;
73 mem.copy(u8, d.buf[d.buf_len..], b[0..off]);
74 d.round(d.buf[0..]);
75 d.buf_len = 0;
76 }
7762
78 // Full middle blocks.63 var off: usize = 0;
79 while (off + 8 <= b.len) : (off += 8) {64 while (off < b.len) : (off += 8) {
80 d.round(b[off .. off + 8]);65 @inlineCall(self.round, b[off .. off + 8]);
81 }66 }
8267
83 // Remainder for next pass.68 self.msg_len +%= @truncate(u8, b.len);
84 mem.copy(u8, d.buf[d.buf_len..], b[off..]);
85 d.buf_len += @intCast(u8, b[off..].len);
86 d.msg_len +%= @truncate(u8, b.len);
87 }69 }
8870
89 pub fn final(d: *Self) T {71 pub fn final(self: *Self, b: []const u8) T {
90 // Padding72 std.debug.assert(b.len < 8);
91 mem.set(u8, d.buf[d.buf_len..], 0);73
92 d.buf[7] = d.msg_len;74 self.msg_len +%= @truncate(u8, b.len);
93 d.round(d.buf[0..]);75
76 var buf = [_]u8{0} ** 8;
77 mem.copy(u8, buf[0..], b[0..]);
78 buf[7] = self.msg_len;
79 self.round(buf[0..]);
9480
95 if (T == u128) {81 if (T == u128) {
96 d.v2 ^= 0xee;82 self.v2 ^= 0xee;
97 } else {83 } else {
98 d.v2 ^= 0xff;84 self.v2 ^= 0xff;
99 }85 }
10086
101 comptime var i: usize = 0;87 comptime var i: usize = 0;
102 inline while (i < d_rounds) : (i += 1) {88 inline while (i < d_rounds) : (i += 1) {
103 @inlineCall(sipRound, d);89 @inlineCall(sipRound, self);
104 }90 }
10591
106 const b1 = d.v0 ^ d.v1 ^ d.v2 ^ d.v3;92 const b1 = self.v0 ^ self.v1 ^ self.v2 ^ self.v3;
107 if (T == u64) {93 if (T == u64) {
108 return b1;94 return b1;
109 }95 }
11096
111 d.v1 ^= 0xdd;97 self.v1 ^= 0xdd;
11298
113 comptime var j: usize = 0;99 comptime var j: usize = 0;
114 inline while (j < d_rounds) : (j += 1) {100 inline while (j < d_rounds) : (j += 1) {
115 @inlineCall(sipRound, d);101 @inlineCall(sipRound, self);
116 }102 }
117103
118 const b2 = d.v0 ^ d.v1 ^ d.v2 ^ d.v3;104 const b2 = self.v0 ^ self.v1 ^ self.v2 ^ self.v3;
119 return (u128(b2) << 64) | b1;105 return (u128(b2) << 64) | b1;
120 }106 }
121107
122 fn round(d: *Self, b: []const u8) void {108 fn round(self: *Self, b: []const u8) void {
123 assert(b.len == 8);109 assert(b.len == 8);
124110
125 const m = mem.readIntSliceLittle(u64, b[0..]);111 const m = mem.readIntSliceLittle(u64, b[0..]);
126 d.v3 ^= m;112 self.v3 ^= m;
127113
128 comptime var i: usize = 0;114 comptime var i: usize = 0;
129 inline while (i < c_rounds) : (i += 1) {115 inline while (i < c_rounds) : (i += 1) {
130 @inlineCall(sipRound, d);116 @inlineCall(sipRound, self);
131 }117 }
132118
133 d.v0 ^= m;119 self.v0 ^= m;
134 }120 }
135121
136 fn sipRound(d: *Self) void {122 fn sipRound(d: *Self) void {
...@@ -151,9 +137,61 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)...@@ -151,9 +137,61 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
151 }137 }
152138
153 pub fn hash(key: []const u8, input: []const u8) T {139 pub fn hash(key: []const u8, input: []const u8) T {
140 const aligned_len = input.len - (input.len % 8);
141
154 var c = Self.init(key);142 var c = Self.init(key);
155 @inlineCall(c.update, input);143 @inlineCall(c.update, input[0..aligned_len]);
156 return @inlineCall(c.final);144 return @inlineCall(c.final, input[aligned_len..]);
145 }
146 };
147}
148
149pub fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) type {
150 assert(T == u64 or T == u128);
151 assert(c_rounds > 0 and d_rounds > 0);
152
153 return struct {
154 const State = SipHashStateless(T, c_rounds, d_rounds);
155 const Self = @This();
156 const digest_size = 64;
157 const block_size = 64;
158
159 state: State,
160 buf: [8]u8,
161 buf_len: usize,
162
163 pub fn init(key: []const u8) Self {
164 return Self{
165 .state = State.init(key),
166 .buf = undefined,
167 .buf_len = 0,
168 };
169 }
170
171 pub fn update(self: *Self, b: []const u8) void {
172 var off: usize = 0;
173
174 if (self.buf_len != 0 and self.buf_len + b.len >= 8) {
175 off += 8 - self.buf_len;
176 mem.copy(u8, self.buf[self.buf_len..], b[0..off]);
177 self.state.update(self.buf[0..]);
178 self.buf_len = 0;
179 }
180
181 const remain_len = b.len - off;
182 const aligned_len = remain_len - (remain_len % 8);
183 self.state.update(b[off .. off + aligned_len]);
184
185 mem.copy(u8, self.buf[self.buf_len..], b[off + aligned_len ..]);
186 self.buf_len += @intCast(u8, b[off + aligned_len ..].len);
187 }
188
189 pub fn final(self: *Self) T {
190 return self.state.final(self.buf[0..self.buf_len]);
191 }
192
193 pub fn hash(key: []const u8, input: []const u8) T {
194 return State.hash(key, input);
157 }195 }
158 };196 };
159}197}
...@@ -237,7 +275,7 @@ test "siphash64-2-4 sanity" {...@@ -237,7 +275,7 @@ test "siphash64-2-4 sanity" {
237 buffer[i] = @intCast(u8, i);275 buffer[i] = @intCast(u8, i);
238276
239 const expected = mem.readIntLittle(u64, &vector);277 const expected = mem.readIntLittle(u64, &vector);
240 testing.expect(siphash.hash(test_key, buffer[0..i]) == expected);278 testing.expectEqual(siphash.hash(test_key, buffer[0..i]), expected);
241 }279 }
242}280}
243281
...@@ -316,6 +354,30 @@ test "siphash128-2-4 sanity" {...@@ -316,6 +354,30 @@ test "siphash128-2-4 sanity" {
316 buffer[i] = @intCast(u8, i);354 buffer[i] = @intCast(u8, i);
317355
318 const expected = mem.readIntLittle(u128, &vector);356 const expected = mem.readIntLittle(u128, &vector);
319 testing.expect(siphash.hash(test_key, buffer[0..i]) == expected);357 testing.expectEqual(siphash.hash(test_key, buffer[0..i]), expected);
358 }
359}
360
361test "iterative non-divisible update" {
362 var buf: [1024]u8 = undefined;
363 for (buf) |*e, i| {
364 e.* = @truncate(u8, i);
365 }
366
367 const key = "0x128dad08f12307";
368 const Siphash = SipHash64(2, 4);
369
370 var end: usize = 9;
371 while (end < buf.len) : (end += 9) {
372 const non_iterative_hash = Siphash.hash(key, buf[0..end]);
373
374 var wy = Siphash.init(key);
375 var i: usize = 0;
376 while (i < end) : (i += 7) {
377 wy.update(buf[i..std.math.min(i + 7, end)]);
378 }
379 const iterative_hash = wy.final();
380
381 std.testing.expectEqual(iterative_hash, non_iterative_hash);
320 }382 }
321}383}