Commit 07782d7a465adc9c5b38f08cb143ba869ed54df5 | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-06-26 23:46:46+02:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-06-26 23:46:46+02:00
log	07782d7a465adc9c5b38f08cb143ba869ed54df5
tree	2185d861044ffbe060fad52bc33d669230232a42
parent	05cb944709d2a05a944e60f032638cae5ee01fb0
parent	d20490d25ef007b14b900665fe767ef44bcf3ee2

Merge pull request 'hash: implement fast crc32c' (#30021) from sinon/zig:fast-crc32c into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/30021
Reviewed-by: Andrew Kelley <andrew@ziglang.org>

8 files changed, 307 insertions(+), 35 deletions(-)

lib
- std
  - hash
    crc.zig
    crc
    Crc32c.zig
    impl.zig
    test.zig
src
- codegen
  - c.zig
  - x86_64
    CodeGen.zig
tools
- crc
  - catalog.txt
- update_crc_catalog.zig

lib/std/hash/crc.zig+12-8

...	...	@@ -1,5 +1,6 @@
1	1	//! This file is auto-generated by tools/update_crc_catalog.zig.
2	2
	3	const builtin = @import("builtin");
3	4	const impl = @import("crc/impl.zig");
4	5
5	6	pub const Crc = impl.Crc;
...	...	@@ -13,6 +14,17 @@ test {
13	14	_ = @import("crc/test.zig");
14	15	}
15	16
	17	pub const Crc32Iscsi = switch (builtin.cpu.hasAll(.x86, &.{ .@"64bit", .crc32 })) {
	18	true => @import("crc/Crc32c.zig"),
	19	else => Crc(u32, .{
	20	.polynomial = 0x1edc6f41,
	21	.initial = 0xffffffff,
	22	.reflect_input = true,
	23	.reflect_output = true,
	24	.xor_output = 0xffffffff,
	25	}),
	26	};
	27
16	28	pub const Crc3Gsm = Crc(u3, .{
17	29	.polynomial = 0x3,
18	30	.initial = 0x0,
...	...	@@ -797,14 +809,6 @@ pub const Crc32Cksum = Crc(u32, .{
797	809	.xor_output = 0xffffffff,
798	810	});
799	811
800		pub const Crc32Iscsi = Crc(u32, .{
801		.polynomial = 0x1edc6f41,
802		.initial = 0xffffffff,
803		.reflect_input = true,
804		.reflect_output = true,
805		.xor_output = 0xffffffff,
806		});
807
808	812	pub const Crc32IsoHdlc = Crc(u32, .{
809	813	.polynomial = 0x04c11db7,
810	814	.initial = 0xffffffff,

lib/std/hash/crc/Crc32c.zig created+238

...	...	@@ -0,0 +1,238 @@
	1	//! Implements CRC-32C (Castagnoli) using the SSE4.2 Intel CRC32 instruction.
	2	//!
	3	//! A couple useful links for understanding the approach taken here:
	4	//! - https://github.com/madler/brotli/blob/1d428d3a9baade233ebc3ac108293256bcb813d1/crc32c.c
	5	//! - https://github.com/madler/zlib/blob/5a82f71ed1dfc0bec044d9702463dbdf84ea3b71/crc32.c
	6	//! - http://www.ross.net/crc/download/crc_v3.txt
	7
	8	// Reflected CRC-32C polynomial in binary form.
	9	const POLY = 0x82f63b78;
	10
	11	const LONG = 8192;
	12	const SHORT = 256;
	13	const long_lookup_table = genTable(LONG);
	14	const short_lookup_table = genTable(SHORT);
	15
	16	const Wrapper = @This();
	17
	18	crc: u32,
	19
	20	pub fn init() Wrapper {
	21	return .{ .crc = 0 };
	22	}
	23
	24	pub fn update(w: *Wrapper, bytes: []const u8) void {
	25	w.crc = crc32(w.crc, bytes);
	26	}
	27
	28	pub fn final(w: Wrapper) u32 {
	29	return w.crc;
	30	}
	31
	32	pub fn hash(bytes: []const u8) u32 {
	33	var c = init();
	34	c.update(bytes);
	35	return c.final();
	36	}
	37
	38	/// Generates the lookup table for efficiently combining CRCs over a block of a given length `length`.
	39	/// This works by building an operator that advances the CRC state as if `length` zero-bytes were appended.
	40	/// We pre-compute 4 tables of 256 entries each (one per byte offset).
	41	///
	42	///
	43	/// The idea behind this table is quite interesting. The CRC state is equivalent to the
	44	/// remainder of dividing the message polynomial (over GF(2)) by the CRC polynomial.
	45	///
	46	/// Advancing the CRC register by `k` zero bits is equivalent to multiplying the current
	47	/// CRC state by `x^k` modulo the CRC polynomial. This operation can be represented
	48	/// as a linear transformation in GF(2), i.e, a matrix.
	49	///
	50	/// We build up this matrix via repeated squaring:
	51	/// - odd represents the operator for 1 zero bit (i.e, multiplication by `x^1 mod POLY`)
	52	/// - even represents the operator for 2 zero bits (`x^2 mod POLY`)
	53	/// - squaring again gives `x^4 mod POLY`, and so on until we get to the right size.
	54	///
	55	/// By squaring the shifting `len`, we build the operator for `x^l mod POLY`.
	56	fn genTable(length: usize) [4][256]u32 {
	57	@setEvalBranchQuota(250000);
	58
	59	var even: [32]u32 = undefined;
	60	zeroes: {
	61	var odd: [32]u32 = undefined;
	62
	63	// Initialize our `odd` array with the operator for a single zero bit:
	64	// - odd[0] is the polynomial itself (acts on the MSB).
	65	// - odd[1..32] represent shifting a single bit through 31 positions.
	66	odd[0] = POLY;
	67	var row: u32 = 1;
	68	for (1..32) \|n\| {
	69	odd[n] = row;
	70	row <<= 1;
	71	}
	72
	73	// even = odd squared: even represents `x^2 mod POLY`.
	74	square(&even, &odd);
	75	// odd = even squared: odd now represents `x^4 mod POLY`.
	76	square(&odd, &even);
	77
	78	// Continue squaring to double the number of zeroes encoded each time:
	79	//
	80	// At each point in the process:
	81	// - square(even, odd): even gets the operator for twice the current length.
	82	// - square(odd, even): odd gets the operator for 4 times the original length.
	83	var len = length;
	84	while (true) {
	85	square(&even, &odd);
	86	len >>= 1;
	87	if (len == 0) break :zeroes;
	88	square(&odd, &even);
	89	len >>= 1;
	90	if (len == 0) break;
	91	}
	92
	93	@memcpy(&even, &odd);
	94	}
	95
	96	var zeroes: [4][256]u32 = undefined;
	97	for (0..256) \|n\| {
	98	zeroes[0][n] = times(&even, n);
	99	zeroes[1][n] = times(&even, n << 8);
	100	zeroes[2][n] = times(&even, n << 16);
	101	zeroes[3][n] = times(&even, n << 24);
	102	}
	103	return zeroes;
	104	}
	105
	106	/// Computes `mat * vec` over `GF(2)`, where `mat` is a 32x32 binary matrix and `vec`
	107	/// is a 32-bit vector. This somewhat "simulates" how bits propagate through the CRC register
	108	/// during shifting.
	109	///
	110	/// - In GF(2) (aka a field where the only values are 0 and 1, aka binary), multiplication is
	111	/// an `AND`, and addition is `XOR`.
	112	/// - This dot product determines how each bit in the input vector "contributes" to
	113	/// the final CRC state, by XORing (adding) rows of the matrix where `vec` has 1s.
	114	fn times(mat: *const [32]u32, vec: u32) u32 {
	115	var sum: u32 = 0;
	116	var v = vec;
	117	var i: u32 = 0;
	118	while (v != 0) {
	119	if (v & 1 != 0) sum ^= mat[i];
	120	v >>= 1;
	121	i += 1;
	122	}
	123	return sum;
	124	}
	125
	126	/// Computes the square of a matrix in GF(2), i.e `dst = dst x src`.
	127	///
	128	/// This produces the operator for doubling the number of zeroes:
	129	/// if `src` represents advancing the CRC by `k` zeroes, then `dest` will
	130	/// represent advancing by 2k zeroes.
	131	///
	132	/// Since polynomial multiplication mod POLY is linear, `mat(mat(x)) = mat^2(x)`
	133	/// gives the effect of two sequential applications of the operator.
	134	fn square(dst: [32]u32, src: const [32]u32) void {
	135	for (dst, src) \|*d, s\| {
	136	d.* = times(src, s);
	137	}
	138	}
	139
	140	fn shift(table: *const [4][256]u32, crc: u32) u32 {
	141	return table[0][crc & 0xFF] ^ table[1][(crc >> 8) & 0xFF] ^ table[2][(crc >> 16) & 0xFF] ^ table[3][crc >> 24];
	142	}
	143
	144	fn crc32(crc: u32, input: []const u8) u32 {
	145	var crc0: u64 = ~crc;
	146
	147	// Compute the CRC for up to seven leading bytes to bring the
	148	// `next` pointer to an eight-byte boundary.
	149	var next = input;
	150	while (next.len > 0 and @intFromPtr(next.ptr) & 7 != 0) {
	151	asm volatile ("crc32b %[out], %[in]"
	152	: [in] "+r" (crc0),
	153	: [out] "rm" (next[0]),
	154	);
	155	next = next[1..];
	156	}
	157
	158	// Compute the CRC on sets of LONG * 3 bytes, executing three independent
	159	// CRC instructions, each on LONG bytes. This is an optimization for
	160	// targets where the CRC instruction has a throughput of one CRC per
	161	// cycle, but a latency of three cycles.
	162	while (next.len >= LONG * 3) {
	163	var crc1: u64 = 0;
	164	var crc2: u64 = 0;
	165
	166	const start = next.len;
	167	while (true) {
	168	// Safe @alignCast(), since we've aligned the pointer to 8 bytes before this loop.
	169	const long: [*]const u64 = @ptrCast(@alignCast(next));
	170	asm volatile (
	171	\\crc32q %[out0], %[in0]
	172	\\crc32q %[out1], %[in1]
	173	\\crc32q %[out2], %[in2]
	174	: [in0] "+r" (crc0),
	175	[in1] "+r" (crc1),
	176	[in2] "+r" (crc2),
	177	: [out0] "rm" (long[0 * LONG / 8]),
	178	[out1] "rm" (long[1 * LONG / 8]),
	179	[out2] "rm" (long[2 * LONG / 8]),
	180	);
	181	next = next[8..];
	182	if (next.len <= start - LONG) break;
	183	}
	184
	185	crc0 = shift(&long_lookup_table, @truncate(crc0)) ^ crc1;
	186	crc0 = shift(&long_lookup_table, @truncate(crc0)) ^ crc2;
	187	next = next[LONG * 2 ..];
	188	}
	189
	190	// Same thing as above, but for smaller chunks of SHORT bytes.
	191	while (next.len >= SHORT * 3) {
	192	var crc1: u64 = 0;
	193	var crc2: u64 = 0;
	194
	195	const start = next.len;
	196	while (true) {
	197	const long: [*]const u64 = @ptrCast(@alignCast(next));
	198	asm volatile (
	199	\\crc32q %[out0], %[in0]
	200	\\crc32q %[out1], %[in1]
	201	\\crc32q %[out2], %[in2]
	202	: [in0] "+r" (crc0),
	203	[in1] "+r" (crc1),
	204	[in2] "+r" (crc2),
	205	: [out0] "rm" (long[0 * SHORT / 8]),
	206	[out1] "rm" (long[1 * SHORT / 8]),
	207	[out2] "rm" (long[2 * SHORT / 8]),
	208	);
	209	next = next[8..];
	210	if (next.len <= start - SHORT) break;
	211	}
	212
	213	crc0 = shift(&short_lookup_table, @truncate(crc0)) ^ crc1;
	214	crc0 = shift(&short_lookup_table, @truncate(crc0)) ^ crc2;
	215	next = next[SHORT * 2 ..];
	216	}
	217
	218	// Compute via 8-byte chunks, until we're left with less than 8 bytes.
	219	while (next.len >= 8) {
	220	const long: [*]const u64 = @ptrCast(@alignCast(next));
	221	asm volatile ("crc32q %[out], %[in]"
	222	: [in] "+r" (crc0),
	223	: [out] "rm" (long[0]),
	224	);
	225	next = next[8..];
	226	}
	227
	228	// Finish the last bytes with just single instructions.
	229	while (next.len > 0) {
	230	asm volatile ("crc32b %[out], %[in]"
	231	: [in] "+r" (crc0),
	232	: [out] "rm" (next[0]),
	233	);
	234	next = next[1..];
	235	}
	236
	237	return @truncate(~crc0);
	238	}

lib/std/hash/crc/impl.zig+14-13

...	...	@@ -23,12 +23,7 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
23	23	const I = if (@bitSizeOf(W) < 8) u8 else W;
24	24	const lookup_table = blk: {
25	25	@setEvalBranchQuota(2500);
26
27		const poly = if (algorithm.reflect_input)
28		@bitReverse(@as(I, algorithm.polynomial)) >> (@bitSizeOf(I) - @bitSizeOf(W))
29		else
30		@as(I, algorithm.polynomial) << (@bitSizeOf(I) - @bitSizeOf(W));
31
	26	const poly = reflect(algorithm.polynomial);
32	27	var table: [256]I = undefined;
33	28	for (&table, 0..) \|*e, i\| {
34	29	var crc: I = i;
...	...	@@ -52,15 +47,13 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
52	47	crc: I,
53	48
54	49	pub fn init() Self {
55		const initial = if (algorithm.reflect_input)
56		@bitReverse(@as(I, algorithm.initial)) >> (@bitSizeOf(I) - @bitSizeOf(W))
57		else
58		@as(I, algorithm.initial) << (@bitSizeOf(I) - @bitSizeOf(W));
59		return Self{ .crc = initial };
	50	const initial = reflect(algorithm.initial);
	51	return .{ .crc = initial };
60	52	}
61	53
62	54	inline fn tableEntry(index: I) I {
63		return lookup_table[@as(u8, @intCast(index & 0xFF))];
	55	const short: u8 = @truncate(index);
	56	return lookup_table[short];
64	57	}
65	58
66	59	pub fn update(self: *Self, bytes: []const u8) void {
...	...	@@ -90,7 +83,7 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
90	83	if (!algorithm.reflect_output) {
91	84	c >>= @bitSizeOf(I) - @bitSizeOf(W);
92	85	}
93		return @as(W, @intCast(c ^ algorithm.xor_output));
	86	return @intCast(c ^ algorithm.xor_output);
94	87	}
95	88
96	89	pub fn hash(bytes: []const u8) W {
...	...	@@ -98,5 +91,13 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
98	91	c.update(bytes);
99	92	return c.final();
100	93	}
	94
	95	fn reflect(x: I) I {
	96	const offset = @bitSizeOf(I) - @bitSizeOf(W);
	97	if (algorithm.reflect_input)
	98	return @bitReverse(x) >> offset
	99	else
	100	return x << offset;
	101	}
101	102	};
102	103	}

lib/std/hash/crc/test.zig+11-11

...	...	@@ -26,6 +26,17 @@ test "crc32 koopman regression" {
26	26	try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);
27	27	}
28	28
	29	test "CRC-32/ISCSI" {
	30	const Crc32Iscsi = crc.Crc32Iscsi;
	31
	32	try testing.expectEqual(@as(u32, 0xe3069283), Crc32Iscsi.hash("123456789"));
	33
	34	var c = Crc32Iscsi.init();
	35	c.update("1234");
	36	c.update("56789");
	37	try testing.expectEqual(@as(u32, 0xe3069283), c.final());
	38	}
	39
29	40	test "CRC-3/GSM" {
30	41	const Crc3Gsm = crc.Crc3Gsm;
31	42
...	...	@@ -1104,17 +1115,6 @@ test "CRC-32/CKSUM" {
1104	1115	try testing.expectEqual(@as(u32, 0x765e7680), c.final());
1105	1116	}
1106	1117
1107		test "CRC-32/ISCSI" {
1108		const Crc32Iscsi = crc.Crc32Iscsi;
1109
1110		try testing.expectEqual(@as(u32, 0xe3069283), Crc32Iscsi.hash("123456789"));
1111
1112		var c = Crc32Iscsi.init();
1113		c.update("1234");
1114		c.update("56789");
1115		try testing.expectEqual(@as(u32, 0xe3069283), c.final());
1116		}
1117
1118	1118	test "CRC-32/ISO-HDLC" {
1119	1119	const Crc32IsoHdlc = crc.Crc32IsoHdlc;
1120	1120

src/codegen/c.zig+2-1

...	...	@@ -4952,7 +4952,8 @@ fn airAsm(f: *Function, inst: Air.Inst.Index) !CValue {
4952	4952	while (it.next()) \|output\| {
4953	4953	const constraint = output.constraint;
4954	4954
4955		if (constraint.len < 2 or constraint[0] != '=' or
	4955	if (constraint.len < 2 or
	4956	(constraint[0] != '=' and constraint[0] != '+') or
4956	4957	(constraint[1] == '{' and constraint[constraint.len - 1] != '}'))
4957	4958	{
4958	4959	return f.fail("CBE: constraint not supported: '{s}'", .{constraint});

src/codegen/x86_64/CodeGen.zig+4

...	...	@@ -178027,6 +178027,10 @@ fn airAsm(self: *CodeGen, inst: Air.Inst.Index) !void {
178027	178027	fixed_mnem_size: {
178028	178028	const fixed_mnem_size: Memory.Size = switch (mnem_tag) {
178029	178029	.clflush => .byte,
	178030	.crc32 => {
	178031	mnem_size.op_has_size.unset(1);
	178032	break :fixed_mnem_size;
	178033	},
178030	178034	.fldcw, .fnstcw, .fstcw, .fnstsw, .fstsw => .word,
178031	178035	.fldenv, .fnstenv, .fstenv => .none,
178032	178036	.frstor, .fsave, .fnsave, .fxrstor, .fxrstor64, .fxsave, .fxsave64 => .none,

tools/crc/catalog.txt+2-1

...	...	@@ -97,7 +97,8 @@ width=32 poly=0xa833982b init=0xffffffff refin=true refout=true xorout=0xff
97	97	width=32 poly=0x04c11db7 init=0xffffffff refin=false refout=false xorout=0xffffffff check=0xfc891918 residue=0xc704dd7b name="CRC-32/BZIP2"
98	98	width=32 poly=0x8001801b init=0x00000000 refin=true refout=true xorout=0x00000000 check=0x6ec2edc4 residue=0x00000000 name="CRC-32/CD-ROM-EDC"
99	99	width=32 poly=0x04c11db7 init=0x00000000 refin=false refout=false xorout=0xffffffff check=0x765e7680 residue=0xc704dd7b name="CRC-32/CKSUM"
100		width=32 poly=0x1edc6f41 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xe3069283 residue=0xb798b438 name="CRC-32/ISCSI"
	100	# CRC-32C implementation is defined manually, since it has an accelerated variant.
	101	# width=32 poly=0x1edc6f41 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xe3069283 residue=0xb798b438 name="CRC-32/ISCSI"
101	102	width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xcbf43926 residue=0xdebb20e3 name="CRC-32/ISO-HDLC"
102	103	width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0x00000000 check=0x340bc6d9 residue=0x00000000 name="CRC-32/JAMCRC"
103	104	width=32 poly=0x741b8cd7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0x2d3dd0ae residue=0x00000000 name="CRC-32/KOOPMAN"

tools/update_crc_catalog.zig+24-1

...	...	@@ -39,6 +39,7 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)
39	39	try code_writer.writeAll(
40	40	\\//! This file is auto-generated by tools/update_crc_catalog.zig.
41	41	\\
	42	\\const builtin = @import("builtin");
42	43	\\const impl = @import("crc/impl.zig");
43	44	\\
44	45	\\pub const Crc = impl.Crc;
...	...	@@ -52,6 +53,17 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)
52	53	\\ _ = @import("crc/test.zig");
53	54	\\}
54	55	\\
	56	\\pub const Crc32Iscsi = switch (builtin.cpu.hasAll(.x86, &.{ .@"64bit", .crc32 })) {
	57	\\ true => @import("crc/Crc32c.zig"),
	58	\\ else => Crc(u32, .{
	59	\\ .polynomial = 0x1edc6f41,
	60	\\ .initial = 0xffffffff,
	61	\\ .reflect_input = true,
	62	\\ .reflect_output = true,
	63	\\ .xor_output = 0xffffffff,
	64	\\ }),
	65	\\};
	66	\\
55	67	);
56	68
57	69	var zig_test_file = try crc_target_dir.createFile(io, "test.zig", .{});
...	...	@@ -83,12 +95,23 @@ fn @"i like cheese"(arena: std.mem.Allocator, io: Io, args: []const []const u8)
83	95	\\}
84	96	\\
85	97	\\test "crc32 koopman regression" {
86		\\ const crc32 = crc.Koopman;
	98	\\ const crc32 = crc.Crc32Koopman;
87	99	\\ try testing.expectEqual(crc32.hash(""), 0x00000000);
88	100	\\ try testing.expectEqual(crc32.hash("a"), 0x0da2aa8a);
89	101	\\ try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);
90	102	\\}
91	103	\\
	104	\\test "CRC-32/ISCSI" {
	105	\\ const Crc32Iscsi = crc.Crc32Iscsi;
	106	\\
	107	\\ try testing.expectEqual(@as(u32, 0xe3069283), Crc32Iscsi.hash("123456789"));
	108	\\
	109	\\ var c = Crc32Iscsi.init();
	110	\\ c.update("1234");
	111	\\ c.update("56789");
	112	\\ try testing.expectEqual(@as(u32, 0xe3069283), c.final());
	113	\\}
	114	\\
92	115	);
93	116
94	117	var reader: std.Io.Reader = .fixed(catalog_txt);