authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-04-29 00:45:17-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-04-29 00:45:17-07:00
logfc45e5b39c0e09dcf69cc65a92375bef0c9593cb
tree3b95615e2ef511c59e7dfc617edb77f176b72e9c
parent497f37ce92ba13aa609acce23135fd6b06ed410f
parent5f0ecafa0dddd9b630220d1c6cf4be2e42987faf
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #19784 from tiehuis/simplify-hash-crc32

std.hash.crc: simplify api

7 files changed, 65 insertions(+), 174 deletions(-)

lib/std/debug.zig+1-1
......@@ -1150,7 +1150,7 @@ pub fn readElfDebugInfo(
11501150 };
11511151
11521152 const mapped_mem = try mapWholeFile(elf_file);
1153 if (expected_crc) |crc| if (crc != std.hash.crc.Crc32SmallWithPoly(.IEEE).hash(mapped_mem)) return error.InvalidDebugInfo;
1153 if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo;
11541154
11551155 const hdr: *const elf.Ehdr = @ptrCast(&mapped_mem[0]);
11561156 if (!mem.eql(u8, hdr.e_ident[0..4], elf.MAGIC)) return error.InvalidElfMagic;
lib/std/hash/benchmark.zig+2-6
......@@ -56,12 +56,8 @@ const hashes = [_]Hash{
5656 .name = "adler32",
5757 },
5858 Hash{
59 .ty = hash.crc.Crc32WithPoly(.IEEE),
60 .name = "crc32-slicing-by-8",
61 },
62 Hash{
63 .ty = hash.crc.Crc32SmallWithPoly(.IEEE),
64 .name = "crc32-half-byte-lookup",
59 .ty = hash.crc.Crc32,
60 .name = "crc32",
6561 },
6662 Hash{
6763 .ty = hash.CityHash32,
lib/std/hash/crc.zig+9-2
......@@ -7,8 +7,7 @@ pub const Polynomial = impl.Polynomial;
77pub const Crc32WithPoly = impl.Crc32WithPoly;
88pub const Crc32SmallWithPoly = impl.Crc32SmallWithPoly;
99
10// IEEE is by far the most common CRC and so is aliased by default.
11pub const Crc32 = Crc32WithPoly(.IEEE);
10pub const Crc32 = Crc32IsoHdlc;
1211
1312test {
1413 _ = @import("crc/test.zig");
......@@ -822,6 +821,14 @@ pub const Crc32Jamcrc = Crc(u32, .{
822821 .xor_output = 0x00000000,
823822});
824823
824pub const Crc32Koopman = Crc(u32, .{
825 .polynomial = 0x741b8cd7,
826 .initial = 0xffffffff,
827 .reflect_input = true,
828 .reflect_output = true,
829 .xor_output = 0xffffffff,
830});
831
825832pub const Crc32Mef = Crc(u32, .{
826833 .polynomial = 0x741b8cd7,
827834 .initial = 0xffffffff,
lib/std/hash/crc/impl.zig+6-135
......@@ -1,11 +1,5 @@
11// There is a generic CRC implementation "Crc()" which can be paramterized via
2// the Algorithm struct for a plethora of uses, along with two implementations
3// of CRC32 implemented with the following key characteristics:
4//
5// - Crc32WithPoly uses 8Kb of tables but is ~10x faster than the small method.
6//
7// - Crc32SmallWithPoly uses only 64 bytes of memory but is slower. Be aware that this is
8// still moderately fast just slow relative to the slicing approach.
2// the Algorithm struct for a plethora of uses.
93//
104// The primary interface for all of the standard CRC algorithms is the
115// generated file "crc.zig", which uses the implementation code here to define
......@@ -108,134 +102,11 @@ pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
108102}
109103
110104pub const Polynomial = enum(u32) {
111 IEEE = 0xedb88320,
112 Castagnoli = 0x82f63b78,
113 Koopman = 0xeb31d82e,
105 IEEE = @compileError("use Crc with algorithm .Crc32IsoHdlc"),
106 Castagnoli = @compileError("use Crc with algorithm .Crc32Iscsi"),
107 Koopman = @compileError("use Crc with algorithm .Crc32Koopman"),
114108 _,
115109};
116110
117// slicing-by-8 crc32 implementation.
118pub fn Crc32WithPoly(comptime poly: Polynomial) type {
119 return struct {
120 const Self = @This();
121 const lookup_tables = block: {
122 @setEvalBranchQuota(20000);
123 var tables: [8][256]u32 = undefined;
124
125 for (&tables[0], 0..) |*e, i| {
126 var crc = @as(u32, @intCast(i));
127 var j: usize = 0;
128 while (j < 8) : (j += 1) {
129 if (crc & 1 == 1) {
130 crc = (crc >> 1) ^ @intFromEnum(poly);
131 } else {
132 crc = (crc >> 1);
133 }
134 }
135 e.* = crc;
136 }
137
138 var i: usize = 0;
139 while (i < 256) : (i += 1) {
140 var crc = tables[0][i];
141 var j: usize = 1;
142 while (j < 8) : (j += 1) {
143 const index: u8 = @truncate(crc);
144 crc = tables[0][index] ^ (crc >> 8);
145 tables[j][i] = crc;
146 }
147 }
148
149 break :block tables;
150 };
151
152 crc: u32,
153
154 pub fn init() Self {
155 return Self{ .crc = 0xffffffff };
156 }
157
158 pub fn update(self: *Self, input: []const u8) void {
159 var i: usize = 0;
160 while (i + 8 <= input.len) : (i += 8) {
161 const p = input[i..][0..8];
162
163 // Unrolling this way gives ~50Mb/s increase
164 self.crc ^= std.mem.readInt(u32, p[0..4], .little);
165
166 self.crc =
167 lookup_tables[0][p[7]] ^
168 lookup_tables[1][p[6]] ^
169 lookup_tables[2][p[5]] ^
170 lookup_tables[3][p[4]] ^
171 lookup_tables[4][@as(u8, @truncate(self.crc >> 24))] ^
172 lookup_tables[5][@as(u8, @truncate(self.crc >> 16))] ^
173 lookup_tables[6][@as(u8, @truncate(self.crc >> 8))] ^
174 lookup_tables[7][@as(u8, @truncate(self.crc >> 0))];
175 }
176
177 while (i < input.len) : (i += 1) {
178 const index = @as(u8, @truncate(self.crc)) ^ input[i];
179 self.crc = (self.crc >> 8) ^ lookup_tables[0][index];
180 }
181 }
182
183 pub fn final(self: *Self) u32 {
184 return ~self.crc;
185 }
186
187 pub fn hash(input: []const u8) u32 {
188 var c = Self.init();
189 c.update(input);
190 return c.final();
191 }
192 };
193}
194
195// half-byte lookup table implementation.
196pub fn Crc32SmallWithPoly(comptime poly: Polynomial) type {
197 return struct {
198 const Self = @This();
199 const lookup_table = block: {
200 var table: [16]u32 = undefined;
201
202 for (&table, 0..) |*e, i| {
203 var crc = @as(u32, @intCast(i * 16));
204 var j: usize = 0;
205 while (j < 8) : (j += 1) {
206 if (crc & 1 == 1) {
207 crc = (crc >> 1) ^ @intFromEnum(poly);
208 } else {
209 crc = (crc >> 1);
210 }
211 }
212 e.* = crc;
213 }
214
215 break :block table;
216 };
217
218 crc: u32,
219
220 pub fn init() Self {
221 return Self{ .crc = 0xffffffff };
222 }
223
224 pub fn update(self: *Self, input: []const u8) void {
225 for (input) |b| {
226 self.crc = lookup_table[@as(u4, @truncate(self.crc ^ (b >> 0)))] ^ (self.crc >> 4);
227 self.crc = lookup_table[@as(u4, @truncate(self.crc ^ (b >> 4)))] ^ (self.crc >> 4);
228 }
229 }
230
231 pub fn final(self: *Self) u32 {
232 return ~self.crc;
233 }
234
235 pub fn hash(input: []const u8) u32 {
236 var c = Self.init();
237 c.update(input);
238 return c.final();
239 }
240 };
241}
111pub const Crc32WithPoly = @compileError("use Crc instead");
112pub const Crc32SmallWithPoly = @compileError("use Crc instead");
lib/std/hash/crc/test.zig+28-14
......@@ -5,22 +5,25 @@ const testing = std.testing;
55const verify = @import("../verify.zig");
66const crc = @import("../crc.zig");
77
8test "crc32 ieee" {
9 inline for ([2]type{ crc.Crc32WithPoly(.IEEE), crc.Crc32SmallWithPoly(.IEEE) }) |ieee| {
10 try testing.expect(ieee.hash("") == 0x00000000);
11 try testing.expect(ieee.hash("a") == 0xe8b7be43);
12 try testing.expect(ieee.hash("abc") == 0x352441c2);
13 try verify.iterativeApi(ieee);
14 }
8test "crc32 ieee regression" {
9 const crc32 = crc.Crc32IsoHdlc;
10 try testing.expectEqual(crc32.hash(""), 0x00000000);
11 try testing.expectEqual(crc32.hash("a"), 0xe8b7be43);
12 try testing.expectEqual(crc32.hash("abc"), 0x352441c2);
1513}
1614
17test "crc32 castagnoli" {
18 inline for ([2]type{ crc.Crc32WithPoly(.Castagnoli), crc.Crc32SmallWithPoly(.Castagnoli) }) |casta| {
19 try testing.expect(casta.hash("") == 0x00000000);
20 try testing.expect(casta.hash("a") == 0xc1d04330);
21 try testing.expect(casta.hash("abc") == 0x364b3fb7);
22 try verify.iterativeApi(casta);
23 }
15test "crc32 castagnoli regression" {
16 const crc32 = crc.Crc32Iscsi;
17 try testing.expectEqual(crc32.hash(""), 0x00000000);
18 try testing.expectEqual(crc32.hash("a"), 0xc1d04330);
19 try testing.expectEqual(crc32.hash("abc"), 0x364b3fb7);
20}
21
22test "crc32 koopman regression" {
23 const crc32 = crc.Crc32Koopman;
24 try testing.expectEqual(crc32.hash(""), 0x00000000);
25 try testing.expectEqual(crc32.hash("a"), 0x0da2aa8a);
26 try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);
2427}
2528
2629test "CRC-3/GSM" {
......@@ -1134,6 +1137,17 @@ test "CRC-32/JAMCRC" {
11341137 try testing.expectEqual(@as(u32, 0x340bc6d9), c.final());
11351138}
11361139
1140test "CRC-32/KOOPMAN" {
1141 const Crc32Koopman = crc.Crc32Koopman;
1142
1143 try testing.expectEqual(@as(u32, 0x2d3dd0ae), Crc32Koopman.hash("123456789"));
1144
1145 var c = Crc32Koopman.init();
1146 c.update("1234");
1147 c.update("56789");
1148 try testing.expectEqual(@as(u32, 0x2d3dd0ae), c.final());
1149}
1150
11371151test "CRC-32/MEF" {
11381152 const Crc32Mef = crc.Crc32Mef;
11391153
tools/crc/catalog.txt+1
......@@ -100,6 +100,7 @@ width=32 poly=0x04c11db7 init=0x00000000 refin=false refout=false xorout=0x
100100width=32 poly=0x1edc6f41 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xe3069283 residue=0xb798b438 name="CRC-32/ISCSI"
101101width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0xcbf43926 residue=0xdebb20e3 name="CRC-32/ISO-HDLC"
102102width=32 poly=0x04c11db7 init=0xffffffff refin=true refout=true xorout=0x00000000 check=0x340bc6d9 residue=0x00000000 name="CRC-32/JAMCRC"
103width=32 poly=0x741b8cd7 init=0xffffffff refin=true refout=true xorout=0xffffffff check=0x2d3dd0ae residue=0x00000000 name="CRC-32/KOOPMAN"
103104width=32 poly=0x741b8cd7 init=0xffffffff refin=true refout=true xorout=0x00000000 check=0xd2c22f51 residue=0x00000000 name="CRC-32/MEF"
104105width=32 poly=0x04c11db7 init=0xffffffff refin=false refout=false xorout=0x00000000 check=0x0376e6e7 residue=0x00000000 name="CRC-32/MPEG-2"
105106width=32 poly=0x000000af init=0x00000000 refin=false refout=false xorout=0x00000000 check=0xbd0be338 residue=0x00000000 name="CRC-32/XFER"
tools/update_crc_catalog.zig+18-16
......@@ -48,8 +48,7 @@ pub fn main() anyerror!void {
4848 \\pub const Crc32WithPoly = impl.Crc32WithPoly;
4949 \\pub const Crc32SmallWithPoly = impl.Crc32SmallWithPoly;
5050 \\
51 \\// IEEE is by far the most common CRC and so is aliased by default.
52 \\pub const Crc32 = Crc32WithPoly(.IEEE);
51 \\pub const Crc32 = Crc32IsoHdlc;
5352 \\
5453 \\test {
5554 \\ _ = @import("crc/test.zig");
......@@ -72,22 +71,25 @@ pub fn main() anyerror!void {
7271 \\const verify = @import("../verify.zig");
7372 \\const crc = @import("../crc.zig");
7473 \\
75 \\test "crc32 ieee" {
76 \\ inline for ([2]type{ crc.Crc32WithPoly(.IEEE), crc.Crc32SmallWithPoly(.IEEE) }) |ieee| {
77 \\ try testing.expect(ieee.hash("") == 0x00000000);
78 \\ try testing.expect(ieee.hash("a") == 0xe8b7be43);
79 \\ try testing.expect(ieee.hash("abc") == 0x352441c2);
80 \\ try verify.iterativeApi(ieee);
81 \\ }
74 \\test "crc32 ieee regression" {
75 \\ const crc32 = crc.Crc32IsoHdlc;
76 \\ try testing.expectEqual(crc32.hash(""), 0x00000000);
77 \\ try testing.expectEqual(crc32.hash("a"), 0xe8b7be43);
78 \\ try testing.expectEqual(crc32.hash("abc"), 0x352441c2);
8279 \\}
8380 \\
84 \\test "crc32 castagnoli" {
85 \\ inline for ([2]type{ crc.Crc32WithPoly(.Castagnoli), crc.Crc32SmallWithPoly(.Castagnoli) }) |casta| {
86 \\ try testing.expect(casta.hash("") == 0x00000000);
87 \\ try testing.expect(casta.hash("a") == 0xc1d04330);
88 \\ try testing.expect(casta.hash("abc") == 0x364b3fb7);
89 \\ try verify.iterativeApi(casta);
90 \\ }
81 \\test "crc32 castagnoli regression" {
82 \\ const crc32 = crc.Crc32Iscsi;
83 \\ try testing.expectEqual(crc32.hash(""), 0x00000000);
84 \\ try testing.expectEqual(crc32.hash("a"), 0xc1d04330);
85 \\ try testing.expectEqual(crc32.hash("abc"), 0x364b3fb7);
86 \\}
87 \\
88 \\test "crc32 koopman regression" {
89 \\ const crc32 = crc.Koopman;
90 \\ try testing.expectEqual(crc32.hash(""), 0x00000000);
91 \\ try testing.expectEqual(crc32.hash("a"), 0x0da2aa8a);
92 \\ try testing.expectEqual(crc32.hash("abc"), 0xba2322ac);
9193 \\}
9294 \\
9395 );