authorgravatar for david@vortan.devDavid Rubin <david@vortan.dev> 2025-06-27 15:34:32-07:00
committergravatar for sinon@vortan.devDavid Rubin <sinon@vortan.dev> 2026-06-24 02:12:43-05:00
log874b6e39db08f66bb0f702d2177b91535b2a8a02
treeec3d2f077423d8c161751afea595979d603a68c7
parentf26cdb2771a4bb4d5f1d5acc446ec51c3e177f75

hash: implement fast crc32c

6 files changed, 301 insertions(+), 34 deletions(-)

lib/std/hash/crc.zig+12-8
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1//! This file is auto-generated by tools/update_crc_catalog.zig.1//! This file is auto-generated by tools/update_crc_catalog.zig.
22
3const builtin = @import("builtin");
3const impl = @import("crc/impl.zig");4const impl = @import("crc/impl.zig");
45
5pub const Crc = impl.Crc;6pub const Crc = impl.Crc;
...@@ -13,6 +14,17 @@ test {...@@ -13,6 +14,17 @@ test {
13 _ = @import("crc/test.zig");14 _ = @import("crc/test.zig");
14}15}
1516
17pub const Crc32Iscsi = switch (builtin.cpu.hasAll(.x86, &.{ .@"64bit", .crc32 }) and builtin.zig_backend == .stage2_llvm) {
18 true => @import("crc/Crc32c.zig"),
19 else => Crc(u32, .{
20 .polynomial = 0x1edc6f41,
21 .initial = 0xffffffff,
22 .reflect_input = true,
23 .reflect_output = true,
24 .xor_output = 0xffffffff,
25 }),
26};
27
16pub const Crc3Gsm = Crc(u3, .{28pub const Crc3Gsm = Crc(u3, .{
17 .polynomial = 0x3,29 .polynomial = 0x3,
18 .initial = 0x0,30 .initial = 0x0,
...@@ -797,14 +809,6 @@ pub const Crc32Cksum = Crc(u32, .{...@@ -797,14 +809,6 @@ pub const Crc32Cksum = Crc(u32, .{
797 .xor_output = 0xffffffff,809 .xor_output = 0xffffffff,
798});810});
799811
800pub const Crc32Iscsi = Crc(u32, .{
801 .polynomial = 0x1edc6f41,
802 .initial = 0xffffffff,
803 .reflect_input = true,
804 .reflect_output = true,
805 .xor_output = 0xffffffff,
806});
807
808pub const Crc32IsoHdlc = Crc(u32, .{812pub const Crc32IsoHdlc = Crc(u32, .{
809 .polynomial = 0x04c11db7,813 .polynomial = 0x04c11db7,
810 .initial = 0xffffffff,814 .initial = 0xffffffff,
lib/std/hash/crc/Crc32c.zig created+238
...@@ -0,0 +1,238 @@
1//! Implements CRC-32C (Castagnoli) using the SSE4.2 Intel CRC32 instruction.
2//!
3//! A couple useful links for understanding the approach taken here:
4//! - https://github.com/madler/brotli/blob/1d428d3a9baade233ebc3ac108293256bcb813d1/crc32c.c
5//! - https://github.com/madler/zlib/blob/5a82f71ed1dfc0bec044d9702463dbdf84ea3b71/crc32.c
6//! - http://www.ross.net/crc/download/crc_v3.txt
7
8// Reflected CRC-32C polynomial in binary form.
9const POLY = 0x82f63b78;
10
11const LONG = 8192;
12const SHORT = 256;
13const long_lookup_table = genTable(LONG);
14const short_lookup_table = genTable(SHORT);
15
16const Wrapper = @This();
17
18crc: u32,
19
20pub fn init() Wrapper {
21 return .{ .crc = 0 };
22}
23
24pub fn update(w: *Wrapper, bytes: []const u8) void {
25 w.crc = crc32(w.crc, bytes);
26}
27
28pub fn final(w: Wrapper) u32 {
29 return w.crc;
30}
31
32pub fn hash(bytes: []const u8) u32 {
33 var c = init();
34 c.update(bytes);
35 return c.final();
36}
37
38/// Generates the lookup table for efficiently combining CRCs over a block of a given length `length`.
39/// This works by building an operator that advances the CRC state as if `length` zero-bytes were appended.
40/// We pre-compute 4 tables of 256 entries each (one per byte offset).
41///
42///
43/// The idea behind this table is quite interesting. The CRC state is equivalent to the
44/// remainder of dividing the message polynomial (over GF(2)) by the CRC polynomial.
45///
46/// Advancing the CRC register by `k` zero bits is equivalent to multiplying the current
47/// CRC state by `x^k` modulo the CRC polynomial. This operation can be represented
48/// as a linear transformation in GF(2), i.e, a matrix.
49///
50/// We build up this matrix via repeated squaring:
51/// - odd represents the operator for 1 zero bit (i.e, multiplication by `x^1 mod POLY`)
52/// - even represents the operator for 2 zero bits (`x^2 mod POLY`)
53/// - squaring again gives `x^4 mod POLY`, and so on until we get to the right size.
54///
55/// By squaring the shifting `len`, we build the operator for `x^l mod POLY`.
56fn genTable(length: usize) [4][256]u32 {
57 @setEvalBranchQuota(250000);
58
59 var even: [32]u32 = undefined;
60 zeroes: {
61 var odd: [32]u32 = undefined;
62
63 // Initialize our `odd` array with the operator for a single zero bit:
64 // - odd[0] is the polynomial itself (acts on the MSB).
65 // - odd[1..32] represent shifting a single bit through 31 positions.
66 odd[0] = POLY;
67 var row: u32 = 1;
68 for (1..32) |n| {
69 odd[n] = row;
70 row <<= 1;
71 }
72
73 // even = odd squared: even represents `x^2 mod POLY`.
74 square(&even, &odd);
75 // odd = even squared: odd now represents `x^4 mod POLY`.
76 square(&odd, &even);
77
78 // Continue squaring to double the number of zeroes encoded each time:
79 //
80 // At each point in the process:
81 // - square(even, odd): even gets the operator for twice the current length.
82 // - square(odd, even): odd gets the operator for 4 times the original length.
83 var len = length;
84 while (true) {
85 square(&even, &odd);
86 len >>= 1;
87 if (len == 0) break :zeroes;
88 square(&odd, &even);
89 len >>= 1;
90 if (len == 0) break;
91 }
92
93 @memcpy(&even, &odd);
94 }
95
96 var zeroes: [4][256]u32 = undefined;
97 for (0..256) |n| {
98 zeroes[0][n] = times(&even, n);
99 zeroes[1][n] = times(&even, n << 8);
100 zeroes[2][n] = times(&even, n << 16);
101 zeroes[3][n] = times(&even, n << 24);
102 }
103 return zeroes;
104}
105
106/// Computes `mat * vec` over `GF(2)`, where `mat` is a 32x32 binary matrix and `vec`
107/// is a 32-bit vector. This somewhat "simulates" how bits propagate through the CRC register
108/// during shifting.
109///
110/// - In GF(2) (aka a field where the only values are 0 and 1, aka binary), multiplication is
111/// an `AND`, and addition is `XOR`.
112/// - This dot product determines how each bit in the input vector "contributes" to
113/// the final CRC state, by XORing (adding) rows of the matrix where `vec` has 1s.
114fn times(mat: *const [32]u32, vec: u32) u32 {
115 var sum: u32 = 0;
116 var v = vec;
117 var i: u32 = 0;
118 while (v != 0) {
119 if (v & 1 != 0) sum ^= mat[i];
120 v >>= 1;
121 i += 1;
122 }
123 return sum;
124}
125
126/// Computes the square of a matrix in GF(2), i.e `dst = dst x src`.
127///
128/// This produces the operator for doubling the number of zeroes:
129/// if `src` represents advancing the CRC by `k` zeroes, then `dest` will
130/// represent advancing by 2k zeroes.
131///
132/// Since polynomial multiplication mod POLY is linear, `mat(mat(x)) = mat^2(x)`
133/// gives the effect of two sequential applications of the operator.
134fn square(dst: *[32]u32, src: *const [32]u32) void {
135 for (dst, src) |*d, s| {
136 d.* = times(src, s);
137 }
138}
139
140fn shift(table: *const [4][256]u32, crc: u32) u32 {
141 return table[0][crc & 0xFF] ^ table[1][(crc >> 8) & 0xFF] ^ table[2][(crc >> 16) & 0xFF] ^ table[3][crc >> 24];
142}
143
144fn crc32(crc: u32, input: []const u8) u32 {
145 var crc0: u64 = ~crc;
146
147 // Compute the CRC for up to seven leading bytes to bring the
148 // `next` pointer to an eight-byte boundary.
149 var next = input;
150 while (next.len > 0 and @intFromPtr(next.ptr) & 7 != 0) {
151 asm volatile ("crc32b %[out], %[in]"
152 : [in] "+r" (crc0),
153 : [out] "rm" (next[0]),
154 );
155 next = next[1..];
156 }
157
158 // Compute the CRC on sets of LONG * 3 bytes, executing three independent
159 // CRC instructions, each on LONG bytes. This is an optimization for
160 // targets where the CRC instruction has a throughput of one CRC per
161 // cycle, but a latency of three cycles.
162 while (next.len >= LONG * 3) {
163 var crc1: u64 = 0;
164 var crc2: u64 = 0;
165
166 const start = next.len;
167 while (true) {
168 // Safe @alignCast(), since we've aligned the pointer to 8 bytes before this loop.
169 const long: [*]const u64 = @ptrCast(@alignCast(next));
170 asm volatile (
171 \\crc32q %[out0], %[in0]
172 \\crc32q %[out1], %[in1]
173 \\crc32q %[out2], %[in2]
174 : [in0] "+r" (crc0),
175 [in1] "+r" (crc1),
176 [in2] "+r" (crc2),
177 : [out0] "rm" (long[0 * LONG / 8]),
178 [out1] "rm" (long[1 * LONG / 8]),
179 [out2] "rm" (long[2 * LONG / 8]),
180 );
181 next = next[8..];
182 if (next.len <= start - LONG) break;
183 }
184
185 crc0 = shift(&long_lookup_table, @truncate(crc0)) ^ crc1;
186 crc0 = shift(&long_lookup_table, @truncate(crc0)) ^ crc2;
187 next = next[LONG * 2 ..];
188 }
189
190 // Same thing as above, but for smaller chunks of SHORT bytes.
191 while (next.len >= SHORT * 3) {
192 var crc1: u64 = 0;
193 var crc2: u64 = 0;
194
195 const start = next.len;
196 while (true) {
197 const long: [*]const u64 = @ptrCast(@alignCast(next));
198 asm volatile (
199 \\crc32q %[out0], %[in0]
200 \\crc32q %[out1], %[in1]
201 \\crc32q %[out2], %[in2]
202 : [in0] "+r" (crc0),
203 [in1] "+r" (crc1),
204 [in2] "+r" (crc2),
205 : [out0] "rm" (long[0 * SHORT / 8]),
206 [out1] "rm" (long[1 * SHORT / 8]),
207 [out2] "rm" (long[2 * SHORT / 8]),
208 );
209 next = next[8..];
210 if (next.len <= start - SHORT) break;
211 }
212
213 crc0 = shift(&short_lookup_table, @truncate(crc0)) ^ crc1;
214 crc0 = shift(&short_lookup_table, @truncate(crc0)) ^ crc2;
215 next = next[SHORT * 2 ..];
216 }
217
218 // Compute via 8-byte chunks, until we're left with less than 8 bytes.
219 while (next.len >= 8) {
220 const long: [*]const u64 = @ptrCast(@alignCast(next));
221 asm volatile ("crc32q %[out], %[in]"
222 : [in] "+r" (crc0),
223 : [out] "rm" (long[0]),
224 );
225 next = next[8..];
226 }
227
228 // Finish the last bytes with just single instructions.
229 while (next.len > 0) {
230 asm volatile ("crc32b %[out], %[in]"
231 : [in] "+r" (crc0),
232 : [out] "rm" (next[0]),
233 );
234 next = next[1..];
235 }
236
237 return @truncate(~crc0);
238}
lib/std/hash/crc/impl.zig+14-13
...@@ -23,12 +23,7 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {...@@ -23,12 +23,7 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
23 const I = if (@bitSizeOf(W) < 8) u8 else W;23 const I = if (@bitSizeOf(W) < 8) u8 else W;
24 const lookup_table = blk: {24 const lookup_table = blk: {
25 @setEvalBranchQuota(2500);25 @setEvalBranchQuota(2500);
2626 const poly = reflect(algorithm.polynomial);
27 const poly = if (algorithm.reflect_input)
28 @bitReverse(@as(I, algorithm.polynomial)) >> (@bitSizeOf(I) - @bitSizeOf(W))
29 else
30 @as(I, algorithm.polynomial) << (@bitSizeOf(I) - @bitSizeOf(W));
31
32 var table: [256]I = undefined;27 var table: [256]I = undefined;
33 for (&table, 0..) |*e, i| {28 for (&table, 0..) |*e, i| {
34 var crc: I = i;29 var crc: I = i;
...@@ -52,15 +47,13 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {...@@ -52,15 +47,13 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
52 crc: I,47 crc: I,
5348
54 pub fn init() Self {49 pub fn init() Self {
55 const initial = if (algorithm.reflect_input)50 const initial = reflect(algorithm.initial);
56 @bitReverse(@as(I, algorithm.initial)) >> (@bitSizeOf(I) - @bitSizeOf(W))51 return .{ .crc = initial };
57 else
58 @as(I, algorithm.initial) << (@bitSizeOf(I) - @bitSizeOf(W));
59 return Self{ .crc = initial };
60 }52 }
6153
62 inline fn tableEntry(index: I) I {54 inline fn tableEntry(index: I) I {
63 return lookup_table[@as(u8, @intCast(index & 0xFF))];55 const short: u8 = @truncate(index);
56 return lookup_table[short];
64 }57 }
6558
66 pub fn update(self: *Self, bytes: []const u8) void {59 pub fn update(self: *Self, bytes: []const u8) void {
...@@ -90,7 +83,7 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {...@@ -90,7 +83,7 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
90 if (!algorithm.reflect_output) {83 if (!algorithm.reflect_output) {
91 c >>= @bitSizeOf(I) - @bitSizeOf(W);84 c >>= @bitSizeOf(I) - @bitSizeOf(W);
92 }85 }
93 return @as(W, @intCast(c ^ algorithm.xor_output));86 return @intCast(c ^ algorithm.xor_output);
94 }87 }
9588
96 pub fn hash(bytes: []const u8) W {89 pub fn hash(bytes: []const u8) W {
...@@ -98,5 +91,13 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {...@@ -98,5 +91,13 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
98 c.update(bytes);91 c.update(bytes);
99 return c.final();92 return c.final();
100 }93 }
94
95 fn reflect(x: I) I {
96 const offset = @bitSizeOf(I) - @bitSizeOf(W);
97 if (algorithm.reflect_input)
98 return @bitReverse(x) >> offset
99 else
100 return x << offset;
101 }
101 };102 };
102}103}
lib/std/hash/crc/test.zig+11-11
...@@ -26,6 +26,17 @@ test "crc32 koopman regression" {...@@ -26,6 +26,17 @@ test "crc32 koopman regression" {
26 try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);26 try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);
27}27}
2828
29test "CRC-32/ISCSI" {
30 const Crc32Iscsi = crc.Crc32Iscsi;
31
32 try testing.expectEqual(@as(u32, 0xe3069283), Crc32Iscsi.hash("123456789"));
33
34 var c = Crc32Iscsi.init();
35 c.update("1234");
36 c.update("56789");
37 try testing.expectEqual(@as(u32, 0xe3069283), c.final());
38}
39
29test "CRC-3/GSM" {40test "CRC-3/GSM" {
30 const Crc3Gsm = crc.Crc3Gsm;41 const Crc3Gsm = crc.Crc3Gsm;
3142
...@@ -1104,17 +1115,6 @@ test "CRC-32/CKSUM" {...@@ -1104,17 +1115,6 @@ test "CRC-32/CKSUM" {
1104 try testing.expectEqual(@as(u32, 0x765e7680), c.final());1115 try testing.expectEqual(@as(u32, 0x765e7680), c.final());
1105}1116}
11061117
1107test "CRC-32/ISCSI" {
1108 const Crc32Iscsi = crc.Crc32Iscsi;
1109
1110 try testing.expectEqual(@as(u32, 0xe3069283), Crc32Iscsi.hash("123456789"));
1111
1112 var c = Crc32Iscsi.init();
1113 c.update("1234");
1114 c.update("56789");
1115 try testing.expectEqual(@as(u32, 0xe3069283), c.final());
1116}
1117
1118test "CRC-32/ISO-HDLC" {1118test "CRC-32/ISO-HDLC" {
1119 const Crc32IsoHdlc = crc.Crc32IsoHdlc;1119 const Crc32IsoHdlc = crc.Crc32IsoHdlc;
11201120
tools/crc/catalog.txt+2-1
...@@ -97,7 +97,8 @@ width=32 poly=0xa833982b init=0xffffffff refin=true refout=true xorout=0xff...@@ -97,7 +97,8 @@ width=32 poly=0xa833982b init=0xffffffff refin=true refout=true xorout=0xff
97width=32 poly=0x04c11db7 init=0xffffffff refin=false refout=false xorout=0xffffffff check=0xfc891918 residue=0xc704dd7b name="CRC-32/BZIP2"97width=32 poly=0x04c11db7 init=0xffffffff refin=false refout=false xorout=0xffffffff check=0xfc891918 residue=0xc704dd7b name="CRC-32/BZIP2"
98width=32 poly=0x8001801b init=0x00000000 refin=true refout=true xorout=0x00000000 check=0x6ec2edc4 residue=0x00000000 name="CRC-32/CD-ROM-EDC"98width=32 poly=0x8001801b init=0x00000000 refin=true refout=true xorout=0x00000000 check=0x6ec2edc4 residue=0x00000000 name="CRC-32/CD-ROM-EDC"
99width=32 poly=0x04c11db7 init=0x00000000 refin=false refout=false xorout=0xffffffff check=0x765e7680 residue=0xc704dd7b name="CRC-32/CKSUM"99width=32 poly=0x04c11db7 init=0x00000000 refin=false refout=false xorout=0xffffffff check=0x765e7680 residue=0xc704dd7b name="CRC-32/CKSUM"
100width=32 poly=0x1edc6f41 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xe3069283 residue=0xb798b438 name="CRC-32/ISCSI"100# CRC-32C implementation is defined manually, since it has an accelerated variant.
101# width=32 poly=0x1edc6f41 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xe3069283 residue=0xb798b438 name="CRC-32/ISCSI"
101width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xcbf43926 residue=0xdebb20e3 name="CRC-32/ISO-HDLC"102width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xcbf43926 residue=0xdebb20e3 name="CRC-32/ISO-HDLC"
102width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0x00000000 check=0x340bc6d9 residue=0x00000000 name="CRC-32/JAMCRC"103width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0x00000000 check=0x340bc6d9 residue=0x00000000 name="CRC-32/JAMCRC"
103width=32 poly=0x741b8cd7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0x2d3dd0ae residue=0x00000000 name="CRC-32/KOOPMAN"104width=32 poly=0x741b8cd7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0x2d3dd0ae residue=0x00000000 name="CRC-32/KOOPMAN"
tools/update_crc_catalog.zig+24-1
...@@ -39,6 +39,7 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)...@@ -39,6 +39,7 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)
39 try code_writer.writeAll(39 try code_writer.writeAll(
40 \\//! This file is auto-generated by tools/update_crc_catalog.zig.40 \\//! This file is auto-generated by tools/update_crc_catalog.zig.
41 \\41 \\
42 \\const builtin = @import("builtin");
42 \\const impl = @import("crc/impl.zig");43 \\const impl = @import("crc/impl.zig");
43 \\44 \\
44 \\pub const Crc = impl.Crc;45 \\pub const Crc = impl.Crc;
...@@ -52,6 +53,17 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)...@@ -52,6 +53,17 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)
52 \\ _ = @import("crc/test.zig");53 \\ _ = @import("crc/test.zig");
53 \\}54 \\}
54 \\55 \\
56 \\pub const Crc32Iscsi = switch (builtin.cpu.hasAll(.x86, &.{ .@"64bit", .crc32 }) and builtin.zig_backend == .stage2_llvm) {
57 \\ true => @import("crc/Crc32c.zig"),
58 \\ else => Crc(u32, .{
59 \\ .polynomial = 0x1edc6f41,
60 \\ .initial = 0xffffffff,
61 \\ .reflect_input = true,
62 \\ .reflect_output = true,
63 \\ .xor_output = 0xffffffff,
64 \\ }),
65 \\};
66 \\
55 );67 );
5668
57 var zig_test_file = try crc_target_dir.createFile(io, "test.zig", .{});69 var zig_test_file = try crc_target_dir.createFile(io, "test.zig", .{});
...@@ -83,12 +95,23 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)...@@ -83,12 +95,23 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)
83 \\}95 \\}
84 \\96 \\
85 \\test "crc32 koopman regression" {97 \\test "crc32 koopman regression" {
86 \\ const crc32 = crc.Koopman;98 \\ const crc32 = crc.Crc32Koopman;
87 \\ try testing.expectEqual(crc32.hash(""), 0x00000000);99 \\ try testing.expectEqual(crc32.hash(""), 0x00000000);
88 \\ try testing.expectEqual(crc32.hash("a"), 0x0da2aa8a);100 \\ try testing.expectEqual(crc32.hash("a"), 0x0da2aa8a);
89 \\ try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);101 \\ try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);
90 \\}102 \\}
91 \\103 \\
104 \\test "CRC-32/ISCSI" {
105 \\ const Crc32Iscsi = crc.Crc32Iscsi;
106 \\
107 \\ try testing.expectEqual(@as(u32, 0xe3069283), Crc32Iscsi.hash("123456789"));
108 \\
109 \\ var c = Crc32Iscsi.init();
110 \\ c.update("1234");
111 \\ c.update("56789");
112 \\ try testing.expectEqual(@as(u32, 0xe3069283), c.final());
113 \\}
114 \\
92 );115 );
93116
94 var reader: std.Io.Reader = .fixed(catalog_txt);117 var reader: std.Io.Reader = .fixed(catalog_txt);