authorgravatar for xavierb@gmail.comXavier Bouchoux <xavierb@gmail.com> 2023-03-16 23:25:48+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-06-17 14:08:05-07:00
log67b3e07260ce5b41039968d35e957945e4661ffa
treefbbaadb4fe95b9f29b1939825fd190654e09ac0c
parent8b923543969c8eceb90f493875b4bb8540f54a71

zlib: naming convention

Adress review comments from https://github.com/ziglang/zig/pull/13977 by using the same naming convention as zstd. And by using `finish()` instead of `close()` for the finalisation of the compressed stream. rationale: - it is not the same as how close() is usually used, since it must be called to flush and write the final bytes. And as such it may fail. - it is not the same `flush` in the deflate code, which allows to keep writting more bytes later, and doesn't write the final checksum. - it is the same name as used in the original zlib library (Z_FINISH) Also, use a packed struct for the header, which seems a better fit.

3 files changed, 64 insertions(+), 55 deletions(-)

lib/std/compress/zlib.zig+60-51
......@@ -8,7 +8,19 @@ const testing = std.testing;
88const mem = std.mem;
99const deflate = std.compress.deflate;
1010
11pub fn ZlibStreamReader(comptime ReaderType: type) type {
11// Zlib header format as specified in RFC1950
12const ZLibHeader = packed struct {
13 checksum: u5,
14 preset_dict: u1,
15 compression_level: u2,
16 compression_method: u4,
17 compression_info: u4,
18
19 const DEFLATE = 8;
20 const WINDOW_32K = 7;
21};
22
23pub fn DecompressStream(comptime ReaderType: type) type {
1224 return struct {
1325 const Self = @This();
1426
......@@ -24,26 +36,24 @@ pub fn ZlibStreamReader(comptime ReaderType: type) type {
2436
2537 fn init(allocator: mem.Allocator, source: ReaderType) !Self {
2638 // Zlib header format is specified in RFC1950
27 const header = try source.readBytesNoEof(2);
39 const header_u16 = try source.readIntBig(u16);
2840
29 const CM = @truncate(u4, header[0]);
30 const CINFO = @truncate(u4, header[0] >> 4);
31 const FCHECK = @truncate(u5, header[1]);
32 _ = FCHECK;
33 const FDICT = @truncate(u1, header[1] >> 5);
34
35 if ((@as(u16, header[0]) << 8 | header[1]) % 31 != 0)
41 // verify the header checksum
42 if (header_u16 % 31 != 0)
3643 return error.BadHeader;
44 const header = @bitCast(ZLibHeader, header_u16);
3745
3846 // The CM field must be 8 to indicate the use of DEFLATE
39 if (CM != 8) return error.InvalidCompression;
47 if (header.compression_method != ZLibHeader.DEFLATE)
48 return error.InvalidCompression;
4049 // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.
4150 // Values above 7 are unspecified and therefore rejected.
42 if (CINFO > 7) return error.InvalidWindowSize;
51 if (header.compression_info > ZLibHeader.WINDOW_32K)
52 return error.InvalidWindowSize;
4353
4454 const dictionary = null;
4555 // TODO: Support this case
46 if (FDICT != 0)
56 if (header.preset_dict != 0)
4757 return error.Unsupported;
4858
4959 return Self{
......@@ -84,8 +94,8 @@ pub fn ZlibStreamReader(comptime ReaderType: type) type {
8494 };
8595}
8696
87pub fn zlibStreamReader(allocator: mem.Allocator, reader: anytype) !ZlibStreamReader(@TypeOf(reader)) {
88 return ZlibStreamReader(@TypeOf(reader)).init(allocator, reader);
97pub fn decompressStream(allocator: mem.Allocator, reader: anytype) !DecompressStream(@TypeOf(reader)) {
98 return DecompressStream(@TypeOf(reader)).init(allocator, reader);
8999}
90100
91101pub const CompressionLevel = enum(u2) {
......@@ -95,11 +105,11 @@ pub const CompressionLevel = enum(u2) {
95105 maximum = 3,
96106};
97107
98pub const CompressionOptions = struct {
108pub const CompressStreamOptions = struct {
99109 level: CompressionLevel = .default,
100110};
101111
102pub fn ZlibStreamWriter(comptime WriterType: type) type {
112pub fn CompressStream(comptime WriterType: type) type {
103113 return struct {
104114 const Self = @This();
105115
......@@ -112,17 +122,17 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
112122 in_writer: WriterType,
113123 hasher: std.hash.Adler32,
114124
115 fn init(allocator: mem.Allocator, dest: WriterType, options: CompressionOptions) !Self {
116 // Zlib header format is specified in RFC1950
117 const CM: u4 = 8; // DEFLATE
118 const CINFO: u4 = 7; // 32K window
119 const CMF: u8 = (@as(u8, CINFO) << 4) | CM;
125 fn init(allocator: mem.Allocator, dest: WriterType, options: CompressStreamOptions) !Self {
126 var header = ZLibHeader{
127 .compression_info = ZLibHeader.WINDOW_32K,
128 .compression_method = ZLibHeader.DEFLATE,
129 .compression_level = @enumToInt(options.level),
130 .preset_dict = 0,
131 .checksum = 0,
132 };
133 header.checksum = @truncate(u5, 31 - @bitCast(u16, header) % 31);
120134
121 const FLEVEL: u2 = @enumToInt(options.level);
122 const FDICT: u1 = 0; // No preset dictionary support
123 const FLG_temp = (@as(u8, FLEVEL) << 6) | (@as(u8, FDICT) << 5);
124 const FCHECK: u5 = 31 - ((@as(u16, CMF) * 256 + FLG_temp) % 31);
125 const FLG = FLG_temp | FCHECK;
135 try dest.writeIntBig(u16, @bitCast(u16, header));
126136
127137 const compression_level: deflate.Compression = switch (options.level) {
128138 .no_compression => .no_compression,
......@@ -131,8 +141,6 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
131141 .maximum => .best_compression,
132142 };
133143
134 try dest.writeAll(&.{ CMF, FLG });
135
136144 return Self{
137145 .allocator = allocator,
138146 .deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }),
......@@ -160,7 +168,7 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
160168 self.deflator.deinit();
161169 }
162170
163 pub fn close(self: *Self) !void {
171 pub fn finish(self: *Self) !void {
164172 const hash = self.hasher.final();
165173 try self.deflator.close();
166174 try self.in_writer.writeIntBig(u32, hash);
......@@ -168,15 +176,14 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
168176 };
169177}
170178
171pub fn zlibStreamWriter(allocator: mem.Allocator, writer: anytype, options: CompressionOptions) !ZlibStreamWriter(@TypeOf(writer)) {
172 return ZlibStreamWriter(@TypeOf(writer)).init(allocator, writer, options);
179pub fn compressStream(allocator: mem.Allocator, writer: anytype, options: CompressStreamOptions) !CompressStream(@TypeOf(writer)) {
180 return CompressStream(@TypeOf(writer)).init(allocator, writer, options);
173181}
174182
175
176fn testReader(data: []const u8, expected: []const u8) !void {
183fn testDecompress(data: []const u8, expected: []const u8) !void {
177184 var in_stream = io.fixedBufferStream(data);
178185
179 var zlib_stream = try zlibStreamReader(testing.allocator, in_stream.reader());
186 var zlib_stream = try decompressStream(testing.allocator, in_stream.reader());
180187 defer zlib_stream.deinit();
181188
182189 // Read and decompress the whole file
......@@ -195,24 +202,24 @@ test "compressed data" {
195202 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
196203
197204 // Compressed with compression level = 0
198 try testReader(
205 try testDecompress(
199206 @embedFile("testdata/rfc1951.txt.z.0"),
200207 rfc1951_txt,
201208 );
202209 // Compressed with compression level = 9
203 try testReader(
210 try testDecompress(
204211 @embedFile("testdata/rfc1951.txt.z.9"),
205212 rfc1951_txt,
206213 );
207214 // Compressed with compression level = 9 and fixed Huffman codes
208 try testReader(
215 try testDecompress(
209216 @embedFile("testdata/rfc1951.txt.fixed.z.9"),
210217 rfc1951_txt,
211218 );
212219}
213220
214221test "don't read past deflate stream's end" {
215 try testReader(&[_]u8{
222 try testDecompress(&[_]u8{
216223 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
217224 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
218225 0x83, 0x95, 0x0b, 0xf5,
......@@ -227,32 +234,32 @@ test "sanity checks" {
227234 // Truncated header
228235 try testing.expectError(
229236 error.EndOfStream,
230 testReader(&[_]u8{0x78}, ""),
237 testDecompress(&[_]u8{0x78}, ""),
231238 );
232239 // Failed FCHECK check
233240 try testing.expectError(
234241 error.BadHeader,
235 testReader(&[_]u8{ 0x78, 0x9D }, ""),
242 testDecompress(&[_]u8{ 0x78, 0x9D }, ""),
236243 );
237244 // Wrong CM
238245 try testing.expectError(
239246 error.InvalidCompression,
240 testReader(&[_]u8{ 0x79, 0x94 }, ""),
247 testDecompress(&[_]u8{ 0x79, 0x94 }, ""),
241248 );
242249 // Wrong CINFO
243250 try testing.expectError(
244251 error.InvalidWindowSize,
245 testReader(&[_]u8{ 0x88, 0x98 }, ""),
252 testDecompress(&[_]u8{ 0x88, 0x98 }, ""),
246253 );
247254 // Wrong checksum
248255 try testing.expectError(
249256 error.WrongChecksum,
250 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
257 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
251258 );
252259 // Truncated checksum
253260 try testing.expectError(
254261 error.EndOfStream,
255 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
262 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
256263 );
257264}
258265
......@@ -260,14 +267,16 @@ test "compress data" {
260267 const allocator = testing.allocator;
261268 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
262269
263 var compressed_data = std.ArrayList(u8).init(allocator);
264 defer compressed_data.deinit();
270 for (std.meta.tags(CompressionLevel)) |level| {
271 var compressed_data = std.ArrayList(u8).init(allocator);
272 defer compressed_data.deinit();
265273
266 var compressor = try zlibStreamWriter(allocator, compressed_data.writer(), .{});
267 defer compressor.deinit();
274 var compressor = try compressStream(allocator, compressed_data.writer(), .{ .level = level });
275 defer compressor.deinit();
268276
269 try compressor.writer().writeAll(rfc1951_txt);
270 try compressor.close();
277 try compressor.writer().writeAll(rfc1951_txt);
278 try compressor.finish();
271279
272 try testReader(compressed_data.items, rfc1951_txt);
280 try testDecompress(compressed_data.items, rfc1951_txt);
281 }
273282}
lib/std/http/Client.zig+2-2
......@@ -309,7 +309,7 @@ pub const RequestTransfer = union(enum) {
309309
310310/// The decompressor for response messages.
311311pub const Compression = union(enum) {
312 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.TransferReader);
312 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader);
313313 pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);
314314 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
315315
......@@ -722,7 +722,7 @@ pub const Request = struct {
722722 if (req.response.transfer_compression) |tc| switch (tc) {
723723 .compress => return error.CompressionNotSupported,
724724 .deflate => req.response.compression = .{
725 .deflate = std.compress.zlib.zlibStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
725 .deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
726726 },
727727 .gzip => req.response.compression = .{
728728 .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
lib/std/http/Server.zig+2-2
......@@ -155,7 +155,7 @@ pub const ResponseTransfer = union(enum) {
155155
156156/// The decompressor for request messages.
157157pub const Compression = union(enum) {
158 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Response.TransferReader);
158 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Response.TransferReader);
159159 pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);
160160 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
161161
......@@ -520,7 +520,7 @@ pub const Response = struct {
520520 if (res.request.transfer_compression) |tc| switch (tc) {
521521 .compress => return error.CompressionNotSupported,
522522 .deflate => res.request.compression = .{
523 .deflate = std.compress.zlib.zlibStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
523 .deflate = std.compress.zlib.decompressStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
524524 },
525525 .gzip => res.request.compression = .{
526526 .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,