authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-01-29 14:12:19+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-01-29 14:30:23-08:00
log4dfca01de4fda9a195048011b3339686dce4e936
treecad8ca1b8575524f097fcdd71620131661f454b8
parent27d2d8e81f03264e951ef3a7e4bb230d223f2ac2

gzip: implement compression

8 files changed, 222 insertions(+), 57 deletions(-)

lib/std/compress.zig+31-1
......@@ -21,7 +21,7 @@ pub fn HashedReader(
2121
2222 pub fn read(self: *@This(), buf: []u8) Error!usize {
2323 const amt = try self.child_reader.read(buf);
24 self.hasher.update(buf);
24 self.hasher.update(buf[0..amt]);
2525 return amt;
2626 }
2727
......@@ -38,6 +38,36 @@ pub fn hashedReader(
3838 return .{ .child_reader = reader, .hasher = hasher };
3939}
4040
41pub fn HashedWriter(
42 comptime WriterType: anytype,
43 comptime HasherType: anytype,
44) type {
45 return struct {
46 child_writer: WriterType,
47 hasher: HasherType,
48
49 pub const Error = WriterType.Error;
50 pub const Writer = std.io.Writer(*@This(), Error, write);
51
52 pub fn write(self: *@This(), buf: []const u8) Error!usize {
53 const amt = try self.child_writer.write(buf);
54 self.hasher.update(buf[0..amt]);
55 return amt;
56 }
57
58 pub fn writer(self: *@This()) Writer {
59 return .{ .context = self };
60 }
61 };
62}
63
64pub fn hashedWriter(
65 writer: anytype,
66 hasher: anytype,
67) HashedWriter(@TypeOf(writer), @TypeOf(hasher)) {
68 return .{ .child_writer = writer, .hasher = hasher };
69}
70
4171test {
4272 _ = deflate;
4373 _ = gzip;
lib/std/compress/deflate/compressor.zig+3-3
......@@ -733,7 +733,7 @@ pub fn Compressor(comptime WriterType: anytype) type {
733733 }
734734
735735 /// Writes the compressed form of `input` to the underlying writer.
736 pub fn write(self: *Self, input: []const u8) !usize {
736 pub fn write(self: *Self, input: []const u8) Error!usize {
737737 var buf = input;
738738
739739 // writes data to hm_bw, which will eventually write the
......@@ -756,7 +756,7 @@ pub fn Compressor(comptime WriterType: anytype) type {
756756 /// If the underlying writer returns an error, `flush()` returns that error.
757757 ///
758758 /// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
759 pub fn flush(self: *Self) !void {
759 pub fn flush(self: *Self) Error!void {
760760 self.sync = true;
761761 try self.step();
762762 try self.hm_bw.writeStoredHeader(0, false);
......@@ -956,7 +956,7 @@ pub fn Compressor(comptime WriterType: anytype) type {
956956 }
957957
958958 /// Writes any pending data to the underlying writer.
959 pub fn close(self: *Self) !void {
959 pub fn close(self: *Self) Error!void {
960960 self.sync = true;
961961 try self.step();
962962 try self.hm_bw.writeStoredHeader(0, true);
lib/std/compress/deflate/compressor_test.zig+4-4
......@@ -86,7 +86,7 @@ fn testSync(level: deflate.Compression, input: []const u8) !void {
8686 read = try decomp.reader().readAll(&final);
8787 try testing.expectEqual(@as(usize, 0), read); // expect ended stream to return 0 bytes
8888
89 _ = decomp.close();
89 try decomp.close();
9090 }
9191 }
9292
......@@ -102,7 +102,7 @@ fn testSync(level: deflate.Compression, input: []const u8) !void {
102102 defer testing.allocator.free(decompressed);
103103
104104 _ = try decomp.reader().readAll(decompressed);
105 _ = decomp.close();
105 try decomp.close();
106106
107107 try testing.expectEqualSlices(u8, input, decompressed);
108108}
......@@ -477,7 +477,7 @@ test "inflate reset" {
477477 .readAllAlloc(testing.allocator, math.maxInt(usize));
478478 defer testing.allocator.free(decompressed_1);
479479
480 _ = decomp.close();
480 try decomp.close();
481481
482482 try testing.expectEqualSlices(u8, strings[0], decompressed_0);
483483 try testing.expectEqualSlices(u8, strings[1], decompressed_1);
......@@ -524,7 +524,7 @@ test "inflate reset dictionary" {
524524 .readAllAlloc(testing.allocator, math.maxInt(usize));
525525 defer testing.allocator.free(decompressed_1);
526526
527 _ = decomp.close();
527 try decomp.close();
528528
529529 try testing.expectEqualSlices(u8, strings[0], decompressed_0);
530530 try testing.expectEqualSlices(u8, strings[1], decompressed_1);
lib/std/compress/deflate/decompressor.zig+5-8
......@@ -477,11 +477,10 @@ pub fn Decompressor(comptime ReaderType: type) type {
477477 }
478478 }
479479
480 pub fn close(self: *Self) ?Error {
481 if (self.err == @as(?Error, error.EndOfStreamWithNoError)) {
482 return null;
480 pub fn close(self: *Self) Error!void {
481 if (self.err) |err| {
482 if (err != error.EndOfStreamWithNoError) return err;
483483 }
484 return self.err;
485484 }
486485
487486 // RFC 1951 section 3.2.7.
......@@ -880,7 +879,7 @@ pub fn Decompressor(comptime ReaderType: type) type {
880879
881880 /// Replaces the inner reader and dictionary with new_reader and new_dict.
882881 /// new_reader must be of the same type as the reader being replaced.
883 pub fn reset(s: *Self, new_reader: ReaderType, new_dict: ?[]const u8) !void {
882 pub fn reset(s: *Self, new_reader: ReaderType, new_dict: ?[]const u8) Error!void {
884883 s.inner_reader = new_reader;
885884 s.step = nextBlock;
886885 s.err = null;
......@@ -920,9 +919,7 @@ test "confirm decompressor resets" {
920919 const buf = try decomp.reader().readAllAlloc(std.testing.allocator, 1024 * 100);
921920 defer std.testing.allocator.free(buf);
922921
923 if (decomp.close()) |err| {
924 return err;
925 }
922 try decomp.close();
926923
927924 try decomp.reset(stream.reader(), null);
928925 }
lib/std/compress/deflate/deflate_fast_test.zig+2-2
......@@ -83,7 +83,7 @@ test "best speed" {
8383 defer decomp.deinit();
8484
8585 const read = try decomp.reader().readAll(decompressed);
86 _ = decomp.close();
86 try decomp.close();
8787
8888 try testing.expectEqual(want.items.len, read);
8989 try testing.expectEqualSlices(u8, want.items, decompressed);
......@@ -150,7 +150,7 @@ test "best speed max match offset" {
150150 var decomp = try inflate.decompressor(testing.allocator, fib.reader(), null);
151151 defer decomp.deinit();
152152 const read = try decomp.reader().readAll(decompressed);
153 _ = decomp.close();
153 try decomp.close();
154154
155155 try testing.expectEqual(src.len, read);
156156 try testing.expectEqualSlices(u8, src, decompressed);
lib/std/compress/deflate/huffman_bit_writer.zig+2-1
......@@ -124,7 +124,8 @@ pub fn HuffmanBitWriter(comptime WriterType: type) type {
124124 if (self.err) {
125125 return;
126126 }
127 self.bytes_written += try self.inner_writer.write(b);
127 try self.inner_writer.writeAll(b);
128 self.bytes_written += b.len;
128129 }
129130
130131 fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
lib/std/compress/gzip.zig+175-38
......@@ -1,5 +1,5 @@
11//
2// Decompressor for GZIP data streams (RFC1952)
2// Compressor/Decompressor for GZIP data streams (RFC1952)
33
44const std = @import("../std.zig");
55const io = std.io;
......@@ -8,6 +8,8 @@ const testing = std.testing;
88const mem = std.mem;
99const deflate = std.compress.deflate;
1010
11const magic = &[2]u8{ 0x1f, 0x8b };
12
1113// Flags for the FLG field in the header
1214const FTEXT = 1 << 0;
1315const FHCRC = 1 << 1;
......@@ -17,6 +19,14 @@ const FCOMMENT = 1 << 4;
1719
1820const max_string_len = 1024;
1921
22pub const Header = struct {
23 extra: ?[]const u8 = null,
24 filename: ?[]const u8 = null,
25 comment: ?[]const u8 = null,
26 modification_time: u32 = 0,
27 operating_system: u8 = 255,
28};
29
2030pub fn Decompress(comptime ReaderType: type) type {
2131 return struct {
2232 const Self = @This();
......@@ -30,25 +40,19 @@ pub fn Decompress(comptime ReaderType: type) type {
3040 inflater: deflate.Decompressor(ReaderType),
3141 in_reader: ReaderType,
3242 hasher: std.hash.Crc32,
33 read_amt: usize,
34
35 info: struct {
36 extra: ?[]const u8,
37 filename: ?[]const u8,
38 comment: ?[]const u8,
39 modification_time: u32,
40 operating_system: u8,
41 },
43 read_amt: u32,
44
45 info: Header,
4246
43 fn init(allocator: mem.Allocator, source: ReaderType) !Self {
44 var hasher = std.compress.hashedReader(source, std.hash.Crc32.init());
47 fn init(allocator: mem.Allocator, in_reader: ReaderType) !Self {
48 var hasher = std.compress.hashedReader(in_reader, std.hash.Crc32.init());
4549 const hashed_reader = hasher.reader();
4650
4751 // gzip header format is specified in RFC1952
4852 const header = try hashed_reader.readBytesNoEof(10);
4953
5054 // Check the ID1/ID2 fields
51 if (header[0] != 0x1f or header[1] != 0x8b)
55 if (!std.mem.eql(u8, header[0..2], magic))
5256 return error.BadHeader;
5357
5458 const CM = header[2];
......@@ -88,15 +92,15 @@ pub fn Decompress(comptime ReaderType: type) type {
8892 errdefer if (comment) |p| allocator.free(p);
8993
9094 if (FLG & FHCRC != 0) {
91 const hash = try source.readInt(u16, .little);
95 const hash = try in_reader.readInt(u16, .little);
9296 if (hash != @as(u16, @truncate(hasher.hasher.final())))
9397 return error.WrongChecksum;
9498 }
9599
96 return Self{
100 return .{
97101 .allocator = allocator,
98 .inflater = try deflate.decompressor(allocator, source, null),
99 .in_reader = source,
102 .inflater = try deflate.decompressor(allocator, in_reader, null),
103 .in_reader = in_reader,
100104 .hasher = std.hash.Crc32.init(),
101105 .info = .{
102106 .filename = filename,
......@@ -119,7 +123,7 @@ pub fn Decompress(comptime ReaderType: type) type {
119123 self.allocator.free(comment);
120124 }
121125
122 // Implements the io.Reader interface
126 /// Implements the io.Reader interface
123127 pub fn read(self: *Self, buffer: []u8) Error!usize {
124128 if (buffer.len == 0)
125129 return 0;
......@@ -128,10 +132,12 @@ pub fn Decompress(comptime ReaderType: type) type {
128132 const r = try self.inflater.read(buffer);
129133 if (r != 0) {
130134 self.hasher.update(buffer[0..r]);
131 self.read_amt += r;
135 self.read_amt +%= @truncate(r);
132136 return r;
133137 }
134138
139 try self.inflater.close();
140
135141 // We've reached the end of stream, check if the checksum matches
136142 const hash = try self.in_reader.readInt(u32, .little);
137143 if (hash != self.hasher.final())
......@@ -139,7 +145,7 @@ pub fn Decompress(comptime ReaderType: type) type {
139145
140146 // The ISIZE field is the size of the uncompressed input modulo 2^32
141147 const input_size = try self.in_reader.readInt(u32, .little);
142 if (self.read_amt & 0xffffffff != input_size)
148 if (self.read_amt != input_size)
143149 return error.CorruptedData;
144150
145151 return 0;
......@@ -155,7 +161,117 @@ pub fn decompress(allocator: mem.Allocator, reader: anytype) !Decompress(@TypeOf
155161 return Decompress(@TypeOf(reader)).init(allocator, reader);
156162}
157163
158fn testReader(data: []const u8, comptime expected: []const u8) !void {
164pub const CompressOptions = struct {
165 header: Header = .{},
166 hash_header: bool = true,
167 level: deflate.Compression = .default_compression,
168};
169
170pub fn Compress(comptime WriterType: type) type {
171 return struct {
172 const Self = @This();
173
174 pub const Error = WriterType.Error ||
175 deflate.Compressor(WriterType).Error;
176 pub const Writer = io.Writer(*Self, Error, write);
177
178 allocator: mem.Allocator,
179 deflater: deflate.Compressor(WriterType),
180 out_writer: WriterType,
181 hasher: std.hash.Crc32,
182 write_amt: u32,
183
184 fn init(allocator: mem.Allocator, out_writer: WriterType, options: CompressOptions) !Self {
185 var hasher = std.compress.hashedWriter(out_writer, std.hash.Crc32.init());
186 const hashed_writer = hasher.writer();
187
188 // ID1/ID2
189 try hashed_writer.writeAll(magic);
190 // CM
191 try hashed_writer.writeByte(8);
192 // Flags
193 try hashed_writer.writeByte(
194 @as(u8, if (options.hash_header) FHCRC else 0) |
195 @as(u8, if (options.header.extra) |_| FEXTRA else 0) |
196 @as(u8, if (options.header.filename) |_| FNAME else 0) |
197 @as(u8, if (options.header.comment) |_| FCOMMENT else 0),
198 );
199 // Modification time
200 try hashed_writer.writeInt(u32, options.header.modification_time, .little);
201 // Extra flags
202 try hashed_writer.writeByte(0);
203 // Operating system
204 try hashed_writer.writeByte(options.header.operating_system);
205
206 if (options.header.extra) |extra| {
207 try hashed_writer.writeInt(u16, @intCast(extra.len), .little);
208 try hashed_writer.writeAll(extra);
209 }
210
211 if (options.header.filename) |filename| {
212 try hashed_writer.writeAll(filename);
213 try hashed_writer.writeByte(0);
214 }
215
216 if (options.header.comment) |comment| {
217 try hashed_writer.writeAll(comment);
218 try hashed_writer.writeByte(0);
219 }
220
221 if (options.hash_header) {
222 try out_writer.writeInt(
223 u16,
224 @truncate(hasher.hasher.final()),
225 .little,
226 );
227 }
228
229 return .{
230 .allocator = allocator,
231 .deflater = try deflate.compressor(allocator, out_writer, .{ .level = options.level }),
232 .out_writer = out_writer,
233 .hasher = std.hash.Crc32.init(),
234 .write_amt = 0,
235 };
236 }
237
238 pub fn deinit(self: *Self) void {
239 self.deflater.deinit();
240 }
241
242 /// Implements the io.Writer interface
243 pub fn write(self: *Self, buffer: []const u8) Error!usize {
244 if (buffer.len == 0)
245 return 0;
246
247 // Write to the compressed stream and update the computed checksum
248 const r = try self.deflater.write(buffer);
249 self.hasher.update(buffer[0..r]);
250 self.write_amt +%= @truncate(r);
251 return r;
252 }
253
254 pub fn writer(self: *Self) Writer {
255 return .{ .context = self };
256 }
257
258 pub fn flush(self: *Self) Error!void {
259 try self.deflater.flush();
260 }
261
262 pub fn close(self: *Self) Error!void {
263 try self.deflater.close();
264 try self.out_writer.writeInt(u32, self.hasher.final(), .little);
265 try self.out_writer.writeInt(u32, self.write_amt, .little);
266 }
267 };
268}
269
270pub fn compress(allocator: mem.Allocator, writer: anytype, options: CompressOptions) !Compress(@TypeOf(writer)) {
271 return Compress(@TypeOf(writer)).init(allocator, writer, options);
272}
273
274fn testReader(expected: []const u8, data: []const u8) !void {
159275 var in_stream = io.fixedBufferStream(data);
160276
161277 var gzip_stream = try decompress(testing.allocator, in_stream.reader());
......@@ -169,70 +285,91 @@ fn testReader(data: []const u8, comptime expected: []const u8) !void {
169285 try testing.expectEqualSlices(u8, expected, buf);
170286}
171287
288fn testWriter(expected: []const u8, data: []const u8, options: CompressOptions) !void {
289 var actual = std.ArrayList(u8).init(testing.allocator);
290 defer actual.deinit();
291
292 var gzip_stream = try compress(testing.allocator, actual.writer(), options);
293 defer gzip_stream.deinit();
294
295 // Write and compress the whole file
296 try gzip_stream.writer().writeAll(data);
297 try gzip_stream.close();
298
299 // Check against the reference
300 try testing.expectEqualSlices(u8, expected, actual.items);
301}
302
172303// All the test cases are obtained by compressing the RFC1952 text
173304//
174305// https://tools.ietf.org/rfc/rfc1952.txt length=25037 bytes
175306// SHA256=164ef0897b4cbec63abf1b57f069f3599bd0fb7c72c2a4dee21bd7e03ec9af67
176307test "compressed data" {
177 try testReader(
178 @embedFile("testdata/rfc1952.txt.gz"),
179 @embedFile("testdata/rfc1952.txt"),
180 );
308 const plain = @embedFile("testdata/rfc1952.txt");
309 const compressed = @embedFile("testdata/rfc1952.txt.gz");
310 try testReader(plain, compressed);
311 try testWriter(compressed, plain, .{
312 .header = .{
313 .filename = "rfc1952.txt",
314 .modification_time = 1706533053,
315 .operating_system = 3,
316 },
317 });
181318}
182319
183320test "sanity checks" {
184321 // Truncated header
185322 try testing.expectError(
186323 error.EndOfStream,
187 testReader(&[_]u8{ 0x1f, 0x8B }, ""),
324 testReader(undefined, &[_]u8{ 0x1f, 0x8B }),
188325 );
189326 // Wrong CM
190327 try testing.expectError(
191328 error.InvalidCompression,
192 testReader(&[_]u8{
329 testReader(undefined, &[_]u8{
193330 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
194331 0x00, 0x03,
195 }, ""),
332 }),
196333 );
197334 // Wrong checksum
198335 try testing.expectError(
199336 error.WrongChecksum,
200 testReader(&[_]u8{
337 testReader(undefined, &[_]u8{
201338 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
202339 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
203340 0x00, 0x00, 0x00, 0x00,
204 }, ""),
341 }),
205342 );
206343 // Truncated checksum
207344 try testing.expectError(
208345 error.EndOfStream,
209 testReader(&[_]u8{
346 testReader(undefined, &[_]u8{
210347 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
211348 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
212 }, ""),
349 }),
213350 );
214351 // Wrong initial size
215352 try testing.expectError(
216353 error.CorruptedData,
217 testReader(&[_]u8{
354 testReader(undefined, &[_]u8{
218355 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
219356 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
220357 0x00, 0x00, 0x00, 0x01,
221 }, ""),
358 }),
222359 );
223360 // Truncated initial size field
224361 try testing.expectError(
225362 error.EndOfStream,
226 testReader(&[_]u8{
363 testReader(undefined, &[_]u8{
227364 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
228365 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
229366 0x00, 0x00, 0x00,
230 }, ""),
367 }),
231368 );
232369}
233370
234371test "header checksum" {
235 try testReader(&[_]u8{
372 try testReader("", &[_]u8{
236373 // GZIP header
237374 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
238375
......@@ -241,5 +378,5 @@ test "header checksum" {
241378
242379 // GZIP data
243380 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
244 }, "");
381 });
245382}
lib/std/compress/testdata/rfc1952.txt.gz
Binary files a/lib/std/compress/testdata/rfc1952.txt.gz and b/lib/std/compress/testdata/rfc1952.txt.gz differ