authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-06-17 14:14:16-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-06-17 14:14:16-07:00
log8a6c3d26c199d5c195686f43f54dee8ee9b45d98
treefbbaadb4fe95b9f29b1939825fd190654e09ac0c
parent9370fb8b81f69fb28311c00f0833883acfa93521
parent67b3e07260ce5b41039968d35e957945e4661ffa
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15010 from xxxbxxx/zlib-compress

Add zlib stream writer

3 files changed, 141 insertions(+), 31 deletions(-)

lib/std/compress/zlib.zig+137-27
......@@ -1,5 +1,5 @@
11//
2// Decompressor for ZLIB data streams (RFC1950)
2// Compressor/Decompressor for ZLIB data streams (RFC1950)
33
44const std = @import("std");
55const io = std.io;
......@@ -8,7 +8,19 @@ const testing = std.testing;
88const mem = std.mem;
99const deflate = std.compress.deflate;
1010
11pub fn ZlibStream(comptime ReaderType: type) type {
11// Zlib header format as specified in RFC1950
12const ZLibHeader = packed struct {
13 checksum: u5,
14 preset_dict: u1,
15 compression_level: u2,
16 compression_method: u4,
17 compression_info: u4,
18
19 const DEFLATE = 8;
20 const WINDOW_32K = 7;
21};
22
23pub fn DecompressStream(comptime ReaderType: type) type {
1224 return struct {
1325 const Self = @This();
1426
......@@ -24,26 +36,24 @@ pub fn ZlibStream(comptime ReaderType: type) type {
2436
2537 fn init(allocator: mem.Allocator, source: ReaderType) !Self {
2638 // Zlib header format is specified in RFC1950
27 const header = try source.readBytesNoEof(2);
28
29 const CM = @truncate(u4, header[0]);
30 const CINFO = @truncate(u4, header[0] >> 4);
31 const FCHECK = @truncate(u5, header[1]);
32 _ = FCHECK;
33 const FDICT = @truncate(u1, header[1] >> 5);
39 const header_u16 = try source.readIntBig(u16);
3440
35 if ((@as(u16, header[0]) << 8 | header[1]) % 31 != 0)
41 // verify the header checksum
42 if (header_u16 % 31 != 0)
3643 return error.BadHeader;
44 const header = @bitCast(ZLibHeader, header_u16);
3745
3846 // The CM field must be 8 to indicate the use of DEFLATE
39 if (CM != 8) return error.InvalidCompression;
47 if (header.compression_method != ZLibHeader.DEFLATE)
48 return error.InvalidCompression;
4049 // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.
4150 // Values above 7 are unspecified and therefore rejected.
42 if (CINFO > 7) return error.InvalidWindowSize;
51 if (header.compression_info > ZLibHeader.WINDOW_32K)
52 return error.InvalidWindowSize;
4353
4454 const dictionary = null;
4555 // TODO: Support this case
46 if (FDICT != 0)
56 if (header.preset_dict != 0)
4757 return error.Unsupported;
4858
4959 return Self{
......@@ -84,14 +94,96 @@ pub fn ZlibStream(comptime ReaderType: type) type {
8494 };
8595}
8696
87pub fn zlibStream(allocator: mem.Allocator, reader: anytype) !ZlibStream(@TypeOf(reader)) {
88 return ZlibStream(@TypeOf(reader)).init(allocator, reader);
97pub fn decompressStream(allocator: mem.Allocator, reader: anytype) !DecompressStream(@TypeOf(reader)) {
98 return DecompressStream(@TypeOf(reader)).init(allocator, reader);
99}
100
101pub const CompressionLevel = enum(u2) {
102 no_compression = 0,
103 fastest = 1,
104 default = 2,
105 maximum = 3,
106};
107
108pub const CompressStreamOptions = struct {
109 level: CompressionLevel = .default,
110};
111
112pub fn CompressStream(comptime WriterType: type) type {
113 return struct {
114 const Self = @This();
115
116 const Error = WriterType.Error ||
117 deflate.Compressor(WriterType).Error;
118 pub const Writer = io.Writer(*Self, Error, write);
119
120 allocator: mem.Allocator,
121 deflator: deflate.Compressor(WriterType),
122 in_writer: WriterType,
123 hasher: std.hash.Adler32,
124
125 fn init(allocator: mem.Allocator, dest: WriterType, options: CompressStreamOptions) !Self {
126 var header = ZLibHeader{
127 .compression_info = ZLibHeader.WINDOW_32K,
128 .compression_method = ZLibHeader.DEFLATE,
129 .compression_level = @enumToInt(options.level),
130 .preset_dict = 0,
131 .checksum = 0,
132 };
133 header.checksum = @truncate(u5, 31 - @bitCast(u16, header) % 31);
134
135 try dest.writeIntBig(u16, @bitCast(u16, header));
136
137 const compression_level: deflate.Compression = switch (options.level) {
138 .no_compression => .no_compression,
139 .fastest => .best_speed,
140 .default => .default_compression,
141 .maximum => .best_compression,
142 };
143
144 return Self{
145 .allocator = allocator,
146 .deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }),
147 .in_writer = dest,
148 .hasher = std.hash.Adler32.init(),
149 };
150 }
151
152 pub fn write(self: *Self, bytes: []const u8) Error!usize {
153 if (bytes.len == 0) {
154 return 0;
155 }
156
157 const w = try self.deflator.write(bytes);
158
159 self.hasher.update(bytes[0..w]);
160 return w;
161 }
162
163 pub fn writer(self: *Self) Writer {
164 return .{ .context = self };
165 }
166
167 pub fn deinit(self: *Self) void {
168 self.deflator.deinit();
169 }
170
171 pub fn finish(self: *Self) !void {
172 const hash = self.hasher.final();
173 try self.deflator.close();
174 try self.in_writer.writeIntBig(u32, hash);
175 }
176 };
177}
178
179pub fn compressStream(allocator: mem.Allocator, writer: anytype, options: CompressStreamOptions) !CompressStream(@TypeOf(writer)) {
180 return CompressStream(@TypeOf(writer)).init(allocator, writer, options);
89181}
90182
91fn testReader(data: []const u8, expected: []const u8) !void {
183fn testDecompress(data: []const u8, expected: []const u8) !void {
92184 var in_stream = io.fixedBufferStream(data);
93185
94 var zlib_stream = try zlibStream(testing.allocator, in_stream.reader());
186 var zlib_stream = try decompressStream(testing.allocator, in_stream.reader());
95187 defer zlib_stream.deinit();
96188
97189 // Read and decompress the whole file
......@@ -110,24 +202,24 @@ test "compressed data" {
110202 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
111203
112204 // Compressed with compression level = 0
113 try testReader(
205 try testDecompress(
114206 @embedFile("testdata/rfc1951.txt.z.0"),
115207 rfc1951_txt,
116208 );
117209 // Compressed with compression level = 9
118 try testReader(
210 try testDecompress(
119211 @embedFile("testdata/rfc1951.txt.z.9"),
120212 rfc1951_txt,
121213 );
122214 // Compressed with compression level = 9 and fixed Huffman codes
123 try testReader(
215 try testDecompress(
124216 @embedFile("testdata/rfc1951.txt.fixed.z.9"),
125217 rfc1951_txt,
126218 );
127219}
128220
129221test "don't read past deflate stream's end" {
130 try testReader(&[_]u8{
222 try testDecompress(&[_]u8{
131223 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
132224 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
133225 0x83, 0x95, 0x0b, 0xf5,
......@@ -142,31 +234,49 @@ test "sanity checks" {
142234 // Truncated header
143235 try testing.expectError(
144236 error.EndOfStream,
145 testReader(&[_]u8{0x78}, ""),
237 testDecompress(&[_]u8{0x78}, ""),
146238 );
147239 // Failed FCHECK check
148240 try testing.expectError(
149241 error.BadHeader,
150 testReader(&[_]u8{ 0x78, 0x9D }, ""),
242 testDecompress(&[_]u8{ 0x78, 0x9D }, ""),
151243 );
152244 // Wrong CM
153245 try testing.expectError(
154246 error.InvalidCompression,
155 testReader(&[_]u8{ 0x79, 0x94 }, ""),
247 testDecompress(&[_]u8{ 0x79, 0x94 }, ""),
156248 );
157249 // Wrong CINFO
158250 try testing.expectError(
159251 error.InvalidWindowSize,
160 testReader(&[_]u8{ 0x88, 0x98 }, ""),
252 testDecompress(&[_]u8{ 0x88, 0x98 }, ""),
161253 );
162254 // Wrong checksum
163255 try testing.expectError(
164256 error.WrongChecksum,
165 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
257 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
166258 );
167259 // Truncated checksum
168260 try testing.expectError(
169261 error.EndOfStream,
170 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
262 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
171263 );
172264}
265
266test "compress data" {
267 const allocator = testing.allocator;
268 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
269
270 for (std.meta.tags(CompressionLevel)) |level| {
271 var compressed_data = std.ArrayList(u8).init(allocator);
272 defer compressed_data.deinit();
273
274 var compressor = try compressStream(allocator, compressed_data.writer(), .{ .level = level });
275 defer compressor.deinit();
276
277 try compressor.writer().writeAll(rfc1951_txt);
278 try compressor.finish();
279
280 try testDecompress(compressed_data.items, rfc1951_txt);
281 }
282}
lib/std/http/Client.zig+2-2
......@@ -309,7 +309,7 @@ pub const RequestTransfer = union(enum) {
309309
310310/// The decompressor for response messages.
311311pub const Compression = union(enum) {
312 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.TransferReader);
312 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader);
313313 pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);
314314 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
315315
......@@ -722,7 +722,7 @@ pub const Request = struct {
722722 if (req.response.transfer_compression) |tc| switch (tc) {
723723 .compress => return error.CompressionNotSupported,
724724 .deflate => req.response.compression = .{
725 .deflate = std.compress.zlib.zlibStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
725 .deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
726726 },
727727 .gzip => req.response.compression = .{
728728 .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
lib/std/http/Server.zig+2-2
......@@ -155,7 +155,7 @@ pub const ResponseTransfer = union(enum) {
155155
156156/// The decompressor for request messages.
157157pub const Compression = union(enum) {
158 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Response.TransferReader);
158 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Response.TransferReader);
159159 pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);
160160 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
161161
......@@ -520,7 +520,7 @@ pub const Response = struct {
520520 if (res.request.transfer_compression) |tc| switch (tc) {
521521 .compress => return error.CompressionNotSupported,
522522 .deflate => res.request.compression = .{
523 .deflate = std.compress.zlib.zlibStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
523 .deflate = std.compress.zlib.decompressStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
524524 },
525525 .gzip => res.request.compression = .{
526526 .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,