authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-06-17 14:14:16-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-06-17 14:14:16-07:00
log8a6c3d26c199d5c195686f43f54dee8ee9b45d98
treefbbaadb4fe95b9f29b1939825fd190654e09ac0c
parent9370fb8b81f69fb28311c00f0833883acfa93521
parent67b3e07260ce5b41039968d35e957945e4661ffa
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #15010 from xxxbxxx/zlib-compress

Add zlib stream writer

3 files changed, 141 insertions(+), 31 deletions(-)

lib/std/compress/zlib.zig+137-27
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1//1//
2// Decompressor for ZLIB data streams (RFC1950)2// Compressor/Decompressor for ZLIB data streams (RFC1950)
33
4const std = @import("std");4const std = @import("std");
5const io = std.io;5const io = std.io;
...@@ -8,7 +8,19 @@ const testing = std.testing;...@@ -8,7 +8,19 @@ const testing = std.testing;
8const mem = std.mem;8const mem = std.mem;
9const deflate = std.compress.deflate;9const deflate = std.compress.deflate;
1010
11pub fn ZlibStream(comptime ReaderType: type) type {11// Zlib header format as specified in RFC1950
12const ZLibHeader = packed struct {
13 checksum: u5,
14 preset_dict: u1,
15 compression_level: u2,
16 compression_method: u4,
17 compression_info: u4,
18
19 const DEFLATE = 8;
20 const WINDOW_32K = 7;
21};
22
23pub fn DecompressStream(comptime ReaderType: type) type {
12 return struct {24 return struct {
13 const Self = @This();25 const Self = @This();
1426
...@@ -24,26 +36,24 @@ pub fn ZlibStream(comptime ReaderType: type) type {...@@ -24,26 +36,24 @@ pub fn ZlibStream(comptime ReaderType: type) type {
2436
25 fn init(allocator: mem.Allocator, source: ReaderType) !Self {37 fn init(allocator: mem.Allocator, source: ReaderType) !Self {
26 // Zlib header format is specified in RFC195038 // Zlib header format is specified in RFC1950
27 const header = try source.readBytesNoEof(2);39 const header_u16 = try source.readIntBig(u16);
28
29 const CM = @truncate(u4, header[0]);
30 const CINFO = @truncate(u4, header[0] >> 4);
31 const FCHECK = @truncate(u5, header[1]);
32 _ = FCHECK;
33 const FDICT = @truncate(u1, header[1] >> 5);
3440
35 if ((@as(u16, header[0]) << 8 | header[1]) % 31 != 0)41 // verify the header checksum
42 if (header_u16 % 31 != 0)
36 return error.BadHeader;43 return error.BadHeader;
44 const header = @bitCast(ZLibHeader, header_u16);
3745
38 // The CM field must be 8 to indicate the use of DEFLATE46 // The CM field must be 8 to indicate the use of DEFLATE
39 if (CM != 8) return error.InvalidCompression;47 if (header.compression_method != ZLibHeader.DEFLATE)
48 return error.InvalidCompression;
40 // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.49 // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.
41 // Values above 7 are unspecified and therefore rejected.50 // Values above 7 are unspecified and therefore rejected.
42 if (CINFO > 7) return error.InvalidWindowSize;51 if (header.compression_info > ZLibHeader.WINDOW_32K)
52 return error.InvalidWindowSize;
4353
44 const dictionary = null;54 const dictionary = null;
45 // TODO: Support this case55 // TODO: Support this case
46 if (FDICT != 0)56 if (header.preset_dict != 0)
47 return error.Unsupported;57 return error.Unsupported;
4858
49 return Self{59 return Self{
...@@ -84,14 +94,96 @@ pub fn ZlibStream(comptime ReaderType: type) type {...@@ -84,14 +94,96 @@ pub fn ZlibStream(comptime ReaderType: type) type {
84 };94 };
85}95}
8696
87pub fn zlibStream(allocator: mem.Allocator, reader: anytype) !ZlibStream(@TypeOf(reader)) {97pub fn decompressStream(allocator: mem.Allocator, reader: anytype) !DecompressStream(@TypeOf(reader)) {
88 return ZlibStream(@TypeOf(reader)).init(allocator, reader);98 return DecompressStream(@TypeOf(reader)).init(allocator, reader);
99}
100
101pub const CompressionLevel = enum(u2) {
102 no_compression = 0,
103 fastest = 1,
104 default = 2,
105 maximum = 3,
106};
107
108pub const CompressStreamOptions = struct {
109 level: CompressionLevel = .default,
110};
111
112pub fn CompressStream(comptime WriterType: type) type {
113 return struct {
114 const Self = @This();
115
116 const Error = WriterType.Error ||
117 deflate.Compressor(WriterType).Error;
118 pub const Writer = io.Writer(*Self, Error, write);
119
120 allocator: mem.Allocator,
121 deflator: deflate.Compressor(WriterType),
122 in_writer: WriterType,
123 hasher: std.hash.Adler32,
124
125 fn init(allocator: mem.Allocator, dest: WriterType, options: CompressStreamOptions) !Self {
126 var header = ZLibHeader{
127 .compression_info = ZLibHeader.WINDOW_32K,
128 .compression_method = ZLibHeader.DEFLATE,
129 .compression_level = @enumToInt(options.level),
130 .preset_dict = 0,
131 .checksum = 0,
132 };
133 header.checksum = @truncate(u5, 31 - @bitCast(u16, header) % 31);
134
135 try dest.writeIntBig(u16, @bitCast(u16, header));
136
137 const compression_level: deflate.Compression = switch (options.level) {
138 .no_compression => .no_compression,
139 .fastest => .best_speed,
140 .default => .default_compression,
141 .maximum => .best_compression,
142 };
143
144 return Self{
145 .allocator = allocator,
146 .deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }),
147 .in_writer = dest,
148 .hasher = std.hash.Adler32.init(),
149 };
150 }
151
152 pub fn write(self: *Self, bytes: []const u8) Error!usize {
153 if (bytes.len == 0) {
154 return 0;
155 }
156
157 const w = try self.deflator.write(bytes);
158
159 self.hasher.update(bytes[0..w]);
160 return w;
161 }
162
163 pub fn writer(self: *Self) Writer {
164 return .{ .context = self };
165 }
166
167 pub fn deinit(self: *Self) void {
168 self.deflator.deinit();
169 }
170
171 pub fn finish(self: *Self) !void {
172 const hash = self.hasher.final();
173 try self.deflator.close();
174 try self.in_writer.writeIntBig(u32, hash);
175 }
176 };
177}
178
179pub fn compressStream(allocator: mem.Allocator, writer: anytype, options: CompressStreamOptions) !CompressStream(@TypeOf(writer)) {
180 return CompressStream(@TypeOf(writer)).init(allocator, writer, options);
89}181}
90182
91fn testReader(data: []const u8, expected: []const u8) !void {183fn testDecompress(data: []const u8, expected: []const u8) !void {
92 var in_stream = io.fixedBufferStream(data);184 var in_stream = io.fixedBufferStream(data);
93185
94 var zlib_stream = try zlibStream(testing.allocator, in_stream.reader());186 var zlib_stream = try decompressStream(testing.allocator, in_stream.reader());
95 defer zlib_stream.deinit();187 defer zlib_stream.deinit();
96188
97 // Read and decompress the whole file189 // Read and decompress the whole file
...@@ -110,24 +202,24 @@ test "compressed data" {...@@ -110,24 +202,24 @@ test "compressed data" {
110 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");202 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
111203
112 // Compressed with compression level = 0204 // Compressed with compression level = 0
113 try testReader(205 try testDecompress(
114 @embedFile("testdata/rfc1951.txt.z.0"),206 @embedFile("testdata/rfc1951.txt.z.0"),
115 rfc1951_txt,207 rfc1951_txt,
116 );208 );
117 // Compressed with compression level = 9209 // Compressed with compression level = 9
118 try testReader(210 try testDecompress(
119 @embedFile("testdata/rfc1951.txt.z.9"),211 @embedFile("testdata/rfc1951.txt.z.9"),
120 rfc1951_txt,212 rfc1951_txt,
121 );213 );
122 // Compressed with compression level = 9 and fixed Huffman codes214 // Compressed with compression level = 9 and fixed Huffman codes
123 try testReader(215 try testDecompress(
124 @embedFile("testdata/rfc1951.txt.fixed.z.9"),216 @embedFile("testdata/rfc1951.txt.fixed.z.9"),
125 rfc1951_txt,217 rfc1951_txt,
126 );218 );
127}219}
128220
129test "don't read past deflate stream's end" {221test "don't read past deflate stream's end" {
130 try testReader(&[_]u8{222 try testDecompress(&[_]u8{
131 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,223 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
132 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,224 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
133 0x83, 0x95, 0x0b, 0xf5,225 0x83, 0x95, 0x0b, 0xf5,
...@@ -142,31 +234,49 @@ test "sanity checks" {...@@ -142,31 +234,49 @@ test "sanity checks" {
142 // Truncated header234 // Truncated header
143 try testing.expectError(235 try testing.expectError(
144 error.EndOfStream,236 error.EndOfStream,
145 testReader(&[_]u8{0x78}, ""),237 testDecompress(&[_]u8{0x78}, ""),
146 );238 );
147 // Failed FCHECK check239 // Failed FCHECK check
148 try testing.expectError(240 try testing.expectError(
149 error.BadHeader,241 error.BadHeader,
150 testReader(&[_]u8{ 0x78, 0x9D }, ""),242 testDecompress(&[_]u8{ 0x78, 0x9D }, ""),
151 );243 );
152 // Wrong CM244 // Wrong CM
153 try testing.expectError(245 try testing.expectError(
154 error.InvalidCompression,246 error.InvalidCompression,
155 testReader(&[_]u8{ 0x79, 0x94 }, ""),247 testDecompress(&[_]u8{ 0x79, 0x94 }, ""),
156 );248 );
157 // Wrong CINFO249 // Wrong CINFO
158 try testing.expectError(250 try testing.expectError(
159 error.InvalidWindowSize,251 error.InvalidWindowSize,
160 testReader(&[_]u8{ 0x88, 0x98 }, ""),252 testDecompress(&[_]u8{ 0x88, 0x98 }, ""),
161 );253 );
162 // Wrong checksum254 // Wrong checksum
163 try testing.expectError(255 try testing.expectError(
164 error.WrongChecksum,256 error.WrongChecksum,
165 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),257 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
166 );258 );
167 // Truncated checksum259 // Truncated checksum
168 try testing.expectError(260 try testing.expectError(
169 error.EndOfStream,261 error.EndOfStream,
170 testReader(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),262 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
171 );263 );
172}264}
265
266test "compress data" {
267 const allocator = testing.allocator;
268 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
269
270 for (std.meta.tags(CompressionLevel)) |level| {
271 var compressed_data = std.ArrayList(u8).init(allocator);
272 defer compressed_data.deinit();
273
274 var compressor = try compressStream(allocator, compressed_data.writer(), .{ .level = level });
275 defer compressor.deinit();
276
277 try compressor.writer().writeAll(rfc1951_txt);
278 try compressor.finish();
279
280 try testDecompress(compressed_data.items, rfc1951_txt);
281 }
282}
lib/std/http/Client.zig+2-2
...@@ -309,7 +309,7 @@ pub const RequestTransfer = union(enum) {...@@ -309,7 +309,7 @@ pub const RequestTransfer = union(enum) {
309309
310/// The decompressor for response messages.310/// The decompressor for response messages.
311pub const Compression = union(enum) {311pub const Compression = union(enum) {
312 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Request.TransferReader);312 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader);
313 pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);313 pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);
314 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});314 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
315315
...@@ -722,7 +722,7 @@ pub const Request = struct {...@@ -722,7 +722,7 @@ pub const Request = struct {
722 if (req.response.transfer_compression) |tc| switch (tc) {722 if (req.response.transfer_compression) |tc| switch (tc) {
723 .compress => return error.CompressionNotSupported,723 .compress => return error.CompressionNotSupported,
724 .deflate => req.response.compression = .{724 .deflate => req.response.compression = .{
725 .deflate = std.compress.zlib.zlibStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,725 .deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
726 },726 },
727 .gzip => req.response.compression = .{727 .gzip => req.response.compression = .{
728 .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,728 .gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,
lib/std/http/Server.zig+2-2
...@@ -155,7 +155,7 @@ pub const ResponseTransfer = union(enum) {...@@ -155,7 +155,7 @@ pub const ResponseTransfer = union(enum) {
155155
156/// The decompressor for request messages.156/// The decompressor for request messages.
157pub const Compression = union(enum) {157pub const Compression = union(enum) {
158 pub const DeflateDecompressor = std.compress.zlib.ZlibStream(Response.TransferReader);158 pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Response.TransferReader);
159 pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);159 pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);
160 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});160 pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
161161
...@@ -520,7 +520,7 @@ pub const Response = struct {...@@ -520,7 +520,7 @@ pub const Response = struct {
520 if (res.request.transfer_compression) |tc| switch (tc) {520 if (res.request.transfer_compression) |tc| switch (tc) {
521 .compress => return error.CompressionNotSupported,521 .compress => return error.CompressionNotSupported,
522 .deflate => res.request.compression = .{522 .deflate => res.request.compression = .{
523 .deflate = std.compress.zlib.zlibStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,523 .deflate = std.compress.zlib.decompressStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,
524 },524 },
525 .gzip => res.request.compression = .{525 .gzip => res.request.compression = .{
526 .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,526 .gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,