authorgravatar for xavierb@gmail.comXavier Bouchoux <xavierb@gmail.com> 2023-03-16 23:25:48+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-06-17 14:08:05-07:00
log67b3e07260ce5b41039968d35e957945e4661ffa
treefbbaadb4fe95b9f29b1939825fd190654e09ac0c
parent8b923543969c8eceb90f493875b4bb8540f54a71

zlib: naming convention

Adress review comments from https://github.com/ziglang/zig/pull/13977 by using the same naming convention as zstd. And by using `finish()` instead of `close()` for the finalisation of the compressed stream. rationale: - it is not the same as how close() is usually used, since it must be called to flush and write the final bytes. And as such it may fail. - it is not the same `flush` in the deflate code, which allows to keep writting more bytes later, and doesn't write the final checksum. - it is the same name as used in the original zlib library (Z_FINISH) Also, use a packed struct for the header, which seems a better fit.

3 files changed, 64 insertions(+), 55 deletions(-)

lib/std/compress/zlib.zig+60-51
...@@ -8,7 +8,19 @@ const testing = std.testing;...@@ -8,7 +8,19 @@ const testing = std.testing;
8const mem = std.mem;8const mem = std.mem;
9const deflate = std.compress.deflate;9const deflate = std.compress.deflate;
1010
11pub fn ZlibStreamReader(comptime ReaderType: type) type {11// Zlib header format as specified in RFC1950
12const ZLibHeader = packed struct {
13 checksum: u5,
14 preset_dict: u1,
15 compression_level: u2,
16 compression_method: u4,
17 compression_info: u4,
18
19 const DEFLATE = 8;
20 const WINDOW_32K = 7;
21};
22
23pub fn DecompressStream(comptime ReaderType: type) type {
12 return struct {24 return struct {
13 const Self = @This();25 const Self = @This();
1426
...@@ -24,26 +36,24 @@ pub fn ZlibStreamReader(comptime ReaderType: type) type {...@@ -24,26 +36,24 @@ pub fn ZlibStreamReader(comptime ReaderType: type) type {
2436
25 fn init(allocator: mem.Allocator, source: ReaderType) !Self {37 fn init(allocator: mem.Allocator, source: ReaderType) !Self {
26 // Zlib header format is specified in RFC195038 // Zlib header format is specified in RFC1950
27 const header = try source.readBytesNoEof(2);39 const header_u16 = try source.readIntBig(u16);
2840
29 const CM = @truncate(u4, header[0]);41 // verify the header checksum
30 const CINFO = @truncate(u4, header[0] >> 4);42 if (header_u16 % 31 != 0)
31 const FCHECK = @truncate(u5, header[1]);
32 _ = FCHECK;
33 const FDICT = @truncate(u1, header[1] >> 5);
34
35 if ((@as(u16, header[0]) << 8 | header[1]) % 31 != 0)
36 return error.BadHeader;43 return error.BadHeader;
44 const header = @bitCast(ZLibHeader, header_u16);
3745
38 // The CM field must be 8 to indicate the use of DEFLATE46 // The CM field must be 8 to indicate the use of DEFLATE
39 if (CM != 8) return error.InvalidCompression;47 if (header.compression_method != ZLibHeader.DEFLATE)
48 return error.InvalidCompression;
40 // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.49 // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.
41 // Values above 7 are unspecified and therefore rejected.50 // Values above 7 are unspecified and therefore rejected.
42 if (CINFO > 7) return error.InvalidWindowSize;51 if (header.compression_info > ZLibHeader.WINDOW_32K)
52 return error.InvalidWindowSize;
4353
44 const dictionary = null;54 const dictionary = null;
45 // TODO: Support this case55 // TODO: Support this case
46 if (FDICT != 0)56 if (header.preset_dict != 0)
47 return error.Unsupported;57 return error.Unsupported;
4858
49 return Self{59 return Self{
...@@ -84,8 +94,8 @@ pub fn ZlibStreamReader(comptime ReaderType: type) type {...@@ -84,8 +94,8 @@ pub fn ZlibStreamReader(comptime ReaderType: type) type {
84 };94 };
85}95}
8696
87pub fn zlibStreamReader(allocator: mem.Allocator, reader: anytype) !ZlibStreamReader(@TypeOf(reader)) {97pub fn decompressStream(allocator: mem.Allocator, reader: anytype) !DecompressStream(@TypeOf(reader)) {
88 return ZlibStreamReader(@TypeOf(reader)).init(allocator, reader);98 return DecompressStream(@TypeOf(reader)).init(allocator, reader);
89}99}
90100
91pub const CompressionLevel = enum(u2) {101pub const CompressionLevel = enum(u2) {
...@@ -95,11 +105,11 @@ pub const CompressionLevel = enum(u2) {...@@ -95,11 +105,11 @@ pub const CompressionLevel = enum(u2) {
95 maximum = 3,105 maximum = 3,
96};106};
97107
98pub const CompressionOptions = struct {108pub const CompressStreamOptions = struct {
99 level: CompressionLevel = .default,109 level: CompressionLevel = .default,
100};110};
101111
102pub fn ZlibStreamWriter(comptime WriterType: type) type {112pub fn CompressStream(comptime WriterType: type) type {
103 return struct {113 return struct {
104 const Self = @This();114 const Self = @This();
105115
...@@ -112,17 +122,17 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {...@@ -112,17 +122,17 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
112 in_writer: WriterType,122 in_writer: WriterType,
113 hasher: std.hash.Adler32,123 hasher: std.hash.Adler32,
114124
115 fn init(allocator: mem.Allocator, dest: WriterType, options: CompressionOptions) !Self {125 fn init(allocator: mem.Allocator, dest: WriterType, options: CompressStreamOptions) !Self {
116 // Zlib header format is specified in RFC1950126 var header = ZLibHeader{
117 const CM: u4 = 8; // DEFLATE127 .compression_info = ZLibHeader.WINDOW_32K,
118 const CINFO: u4 = 7; // 32K window128 .compression_method = ZLibHeader.DEFLATE,
119 const CMF: u8 = (@as(u8, CINFO) << 4) | CM;129 .compression_level = @enumToInt(options.level),
130 .preset_dict = 0,
131 .checksum = 0,
132 };
133 header.checksum = @truncate(u5, 31 - @bitCast(u16, header) % 31);
120134
121 const FLEVEL: u2 = @enumToInt(options.level);135 try dest.writeIntBig(u16, @bitCast(u16, header));
122 const FDICT: u1 = 0; // No preset dictionary support
123 const FLG_temp = (@as(u8, FLEVEL) << 6) | (@as(u8, FDICT) << 5);
124 const FCHECK: u5 = 31 - ((@as(u16, CMF) * 256 + FLG_temp) % 31);
125 const FLG = FLG_temp | FCHECK;
126136
127 const compression_level: deflate.Compression = switch (options.level) {137 const compression_level: deflate.Compression = switch (options.level) {
128 .no_compression => .no_compression,138 .no_compression => .no_compression,
...@@ -131,8 +141,6 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {...@@ -131,8 +141,6 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
131 .maximum => .best_compression,141 .maximum => .best_compression,
132 };142 };
133143
134 try dest.writeAll(&.{ CMF, FLG });
135
136 return Self{144 return Self{
137 .allocator = allocator,145 .allocator = allocator,
138 .deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }),146 .deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }),
...@@ -160,7 +168,7 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {...@@ -160,7 +168,7 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
160 self.deflator.deinit();168 self.deflator.deinit();
161 }169 }
162170
163 pub fn close(self: *Self) !void {171 pub fn finish(self: *Self) !void {
164 const hash = self.hasher.final();172 const hash = self.hasher.final();
165 try self.deflator.close();173 try self.deflator.close();
166 try self.in_writer.writeIntBig(u32, hash);174 try self.in_writer.writeIntBig(u32, hash);
...@@ -168,15 +176,14 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {...@@ -168,15 +176,14 @@ pub fn ZlibStreamWriter(comptime WriterType: type) type {
168 };176 };
169}177}
170178
171pub fn zlibStreamWriter(allocator: mem.Allocator, writer: anytype, options: CompressionOptions) !ZlibStreamWriter(@TypeOf(writer)) {179pub fn compressStream(allocator: mem.Allocator, writer: anytype, options: CompressStreamOptions) !CompressStream(@TypeOf(writer)) {
172 return ZlibStreamWriter(@TypeOf(writer)).init(allocator, writer, options);180 return CompressStream(@TypeOf(writer)).init(allocator, writer, options);
173}181}
174182
175183fn testDecompress(data: []const u8, expected: []const u8) !void {
176fn testReader(data: []const u8, expected: []const u8) !void {
177 var in_stream = io.fixedBufferStream(data);184 var in_stream = io.fixedBufferStream(data);
178185
179 var zlib_stream = try zlibStreamReader(testing.allocator, in_stream.reader());186 var zlib_stream = try decompressStream(testing.allocator, in_stream.reader());
180 defer zlib_stream.deinit();187 defer zlib_stream.deinit();
181188
182 // Read and decompress the whole file189 // Read and decompress the whole file
...@@ -195,24 +202,24 @@ test "compressed data" {...@@ -195,24 +202,24 @@ test "compressed data" {
195 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");202 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
196203
197 // Compressed with compression level = 0204 // Compressed with compression level = 0
198 try testReader(205 try testDecompress(
199 @embedFile("testdata/rfc1951.txt.z.0"),206 @embedFile("testdata/rfc1951.txt.z.0"),
200 rfc1951_txt,207 rfc1951_txt,
201 );208 );
202 // Compressed with compression level = 9209 // Compressed with compression level = 9
203 try testReader(210 try testDecompress(
204 @embedFile("testdata/rfc1951.txt.z.9"),211 @embedFile("testdata/rfc1951.txt.z.9"),
205 rfc1951_txt,212 rfc1951_txt,
206 );213 );
207 // Compressed with compression level = 9 and fixed Huffman codes214 // Compressed with compression level = 9 and fixed Huffman codes
208 try testReader(215 try testDecompress(
209 @embedFile("testdata/rfc1951.txt.fixed.z.9"),216 @embedFile("testdata/rfc1951.txt.fixed.z.9"),
210 rfc1951_txt,217 rfc1951_txt,
211 );218 );
212}219}
213220
214test "don't read past deflate stream's end" {221test "don't read past deflate stream's end" {
215 try testReader(&[_]u8{222 try testDecompress(&[_]u8{
216 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,223 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
217 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,224 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
218 0x83, 0x95, 0x0b, 0xf5,225 0x83, 0x95, 0x0b, 0xf5,
...@@ -227,32 +234,32 @@ test "sanity checks" {...@@ -227,32 +234,32 @@ test "sanity checks" {
227 // Truncated header234 // Truncated header
228 try testing.expectError(235 try testing.expectError(
229 error.EndOfStream,236 error.EndOfStream,
230 testReader(&[_]u8{0x78}, ""),237 testDecompress(&[_]u8{0x78}, ""),
231 );238 );
232 // Failed FCHECK check239 // Failed FCHECK check
233 try testing.expectError(240 try testing.expectError(
234 error.BadHeader,241 error.BadHeader,
235 testReader(&[_]u8{ 0x78, 0x9D }, ""),242 testDecompress(&[_]u8{ 0x78, 0x9D }, ""),
236 );243 );
237 // Wrong CM244 // Wrong CM
238 try testing.expectError(245 try testing.expectError(
239 error.InvalidCompression,246 error.InvalidCompression,
240 testReader(&[_]u8{ 0x79, 0x94 }, ""),247 testDecompress(&[_]u8{ 0x79, 0x94 }, ""),
241 );248 );
242 // Wrong CINFO249 // Wrong CINFO
243 try testing.expectError(250 try testing.expectError(
244 error.InvalidWindowSize,251 error.InvalidWindowSize,
245 testReader(&[_]u8{ 0x88, 0x98 }, ""),252 testDecompress(&[_]u8{ 0x88, 0x98 }, ""),
246 );253 );
247 // Wrong checksum254 // Wrong checksum
248 try testing.expectError(255 try testing.expectError(
249 error.WrongChecksum,256 error.WrongChecksum,
250 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),257 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
251 );258 );
252 // Truncated checksum259 // Truncated checksum
253 try testing.expectError(260 try testing.expectError(
254 error.EndOfStream,261 error.EndOfStream,
255 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),262 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
256 );263 );
257}264}
258265
...@@ -260,14 +267,16 @@ test "compress data" {...@@ -260,14 +267,16 @@ test "compress data" {
260 const allocator = testing.allocator;267 const allocator = testing.allocator;
261 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");268 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
262269
263 var compressed_data = std.ArrayList(u8).init(allocator);270 for (std.meta.tags(CompressionLevel)) |level| {
264 defer compressed_data.deinit();271 var compressed_data = std.ArrayList(u8).init(allocator);
272 defer compressed_data.deinit();
265273
266 var compressor = try zlibStreamWriter(allocator, compressed_data.writer(), .{});274 var compressor = try compressStream(allocator, compressed_data.writer(), .{ .level = level });
267 defer compressor.deinit();275 defer compressor.deinit();
268276
269 try compressor.writer().writeAll(rfc1951_txt);277 try compressor.writer().writeAll(rfc1951_txt);
270 try compressor.close();278 try compressor.finish();
271279
272 try testReader(compressed_data.items, rfc1951_txt);280 try testDecompress(compressed_data.items, rfc1951_txt);
281 }
273}282}
lib/std/http/Client.zig+2-2
...@@ -309,7 +309,7 @@ pub const RequestTransfer = union(enum) {...@@ -309,7 +309,7 @@ pub const RequestTransfer = union(enum) {
309309
310/// The decompressor for response messages.310/// The decompressor for response messages.
311pub const Compression = union(enum) {311pub const Compression = union(enum) {
312 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.TransferReader);312 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader);
313 pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);313 pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);
314 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});314 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
315315
...@@ -722,7 +722,7 @@ pub const Request = struct {...@@ -722,7 +722,7 @@ pub const Request = struct {
722 if (req.response.transfer_compression) |tc| switch (tc) {722 if (req.response.transfer_compression) |tc| switch (tc) {
723 .compress => return error.CompressionNotSupported,723 .compress => return error.CompressionNotSupported,
724 .deflate => req.response.compression = .{724 .deflate => req.response.compression = .{
725 .deflate = std.compress.zlib.zlibStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,725 .deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
726 },726 },
727 .gzip => req.response.compression = .{727 .gzip => req.response.compression = .{
728 .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,728 .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
lib/std/http/Server.zig+2-2
...@@ -155,7 +155,7 @@ pub const ResponseTransfer = union(enum) {...@@ -155,7 +155,7 @@ pub const ResponseTransfer = union(enum) {
155155
156/// The decompressor for request messages.156/// The decompressor for request messages.
157pub const Compression = union(enum) {157pub const Compression = union(enum) {
158 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Response.TransferReader);158 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Response.TransferReader);
159 pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);159 pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);
160 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});160 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
161161
...@@ -520,7 +520,7 @@ pub const Response = struct {...@@ -520,7 +520,7 @@ pub const Response = struct {
520 if (res.request.transfer_compression) |tc| switch (tc) {520 if (res.request.transfer_compression) |tc| switch (tc) {
521 .compress => return error.CompressionNotSupported,521 .compress => return error.CompressionNotSupported,
522 .deflate => res.request.compression = .{522 .deflate => res.request.compression = .{
523 .deflate = std.compress.zlib.zlibStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,523 .deflate = std.compress.zlib.decompressStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
524 },524 },
525 .gzip => res.request.compression = .{525 .gzip => res.request.compression = .{
526 .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,526 .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,