authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-27 16:05:32-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-31 22:10:11-07:00
log824c157e0c25a9337ffd036f7ad5cb811b1f18cd
treec58f974383513f7ff22e4e47a99f52793b1abff5
parent73c98ca0e6aa52b942b92135ecf0305362030733

std.compress.flate: finish reorganizing

7 files changed, 271 insertions(+), 803 deletions(-)

lib/std/compress/flate.zig+21-484
...@@ -1,7 +1,23 @@...@@ -1,7 +1,23 @@
1const builtin = @import("builtin");
2const std = @import("../std.zig");1const std = @import("../std.zig");
3const testing = std.testing;2
4const Writer = std.Io.Writer;3/// When decompressing, the output buffer is used as the history window, so
4/// less than this may result in failure to decompress streams that were
5/// compressed with a larger window.
6pub const max_window_len = history_len * 2;
7
8pub const history_len = 32768;
9
10/// Deflate is a lossless data compression file format that uses a combination
11/// of LZ77 and Huffman coding.
12pub const Compress = @import("flate/Compress.zig");
13
14/// Inflate is the decoding process that takes a Deflate bitstream for
15/// decompression and correctly produces the original full-size data or file.
16pub const Decompress = @import("flate/Decompress.zig");
17
18/// Compression without Lempel-Ziv match searching. Faster compression, less
19/// memory requirements but bigger compressed sizes.
20pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig");
521
6/// Container of the deflate bit stream body. Container adds header before22/// Container of the deflate bit stream body. Container adds header before
7/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,23/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
...@@ -13,7 +29,6 @@ const Writer = std.Io.Writer;...@@ -13,7 +29,6 @@ const Writer = std.Io.Writer;
13/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes29/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
14/// crc32 checksum and 4 bytes of uncompressed data length.30/// crc32 checksum and 4 bytes of uncompressed data length.
15///31///
16///
17/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-432/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
18/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-533/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
19pub const Container = enum {34pub const Container = enum {
...@@ -84,7 +99,7 @@ pub const Container = enum {...@@ -84,7 +99,7 @@ pub const Container = enum {
84 pub fn init(containter: Container) Hasher {99 pub fn init(containter: Container) Hasher {
85 return switch (containter) {100 return switch (containter) {
86 .gzip => .{ .gzip = .{} },101 .gzip => .{ .gzip = .{} },
87 .zlib => .{ .zlib = .init() },102 .zlib => .{ .zlib = .{} },
88 .raw => .raw,103 .raw => .raw,
89 };104 };
90 }105 }
...@@ -107,7 +122,7 @@ pub const Container = enum {...@@ -107,7 +122,7 @@ pub const Container = enum {
107 }122 }
108 }123 }
109124
110 pub fn writeFooter(hasher: *Hasher, writer: *Writer) Writer.Error!void {125 pub fn writeFooter(hasher: *Hasher, writer: *std.Io.Writer) std.Io.Writer.Error!void {
111 var bits: [4]u8 = undefined;126 var bits: [4]u8 = undefined;
112 switch (hasher.*) {127 switch (hasher.*) {
113 .gzip => |*gzip| {128 .gzip => |*gzip| {
...@@ -135,484 +150,6 @@ pub const Container = enum {...@@ -135,484 +150,6 @@ pub const Container = enum {
135 };150 };
136};151};
137152
138/// When decompressing, the output buffer is used as the history window, so
139/// less than this may result in failure to decompress streams that were
140/// compressed with a larger window.
141pub const max_window_len = 1 << 16;
142
143/// Deflate is a lossless data compression file format that uses a combination
144/// of LZ77 and Huffman coding.
145pub const Compress = @import("flate/Compress.zig");
146
147/// Inflate is the decoding process that takes a Deflate bitstream for
148/// decompression and correctly produces the original full-size data or file.
149pub const Decompress = @import("flate/Decompress.zig");
150
151/// Compression without Lempel-Ziv match searching. Faster compression, less
152/// memory requirements but bigger compressed sizes.
153pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig");
154
155test "compress/decompress" {
156 const print = std.debug.print;
157 var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer
158 var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer
159
160 const levels = [_]Compress.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
161 const cases = [_]struct {
162 data: []const u8, // uncompressed content
163 // compressed data sizes per level 4-9
164 gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len,
165 huffman_only_size: usize = 0,
166 store_size: usize = 0,
167 }{
168 .{
169 .data = @embedFile("flate/testdata/rfc1951.txt"),
170 .gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 },
171 .huffman_only_size = 20287,
172 .store_size = 36967,
173 },
174 .{
175 .data = @embedFile("flate/testdata/fuzz/roundtrip1.input"),
176 .gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 },
177 .huffman_only_size = 393,
178 .store_size = 393,
179 },
180 .{
181 .data = @embedFile("flate/testdata/fuzz/roundtrip2.input"),
182 .gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 },
183 .huffman_only_size = 394,
184 .store_size = 394,
185 },
186 .{
187 .data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"),
188 .gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 },
189 .huffman_only_size = 498,
190 .store_size = 747,
191 },
192 };
193
194 for (cases, 0..) |case, case_no| {
195 const data = case.data;
196
197 for (levels, 0..) |level, i| {
198 for (Container.list) |container| {
199 var compressed_size: usize = if (case.gzip_sizes[i] > 0)
200 case.gzip_sizes[i] - Container.gzip.size() + container.size()
201 else
202 0;
203
204 // compress original stream to compressed stream
205 {
206 var compressed: Writer = .fixed(&cmp_buf);
207 var compress: Compress = .init(&compressed, &.{}, .{ .container = .raw, .level = level });
208 try compress.writer.writeAll(data);
209 try compress.end();
210
211 if (compressed_size == 0) {
212 if (container == .gzip)
213 print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
214 compressed_size = compressed.end;
215 }
216 try testing.expectEqual(compressed_size, compressed.end);
217 }
218 // decompress compressed stream to decompressed stream
219 {
220 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
221 var decompressed: Writer = .fixed(&dcm_buf);
222 var decompress: Decompress = .init(&compressed, container, &.{});
223 _ = try decompress.reader.streamRemaining(&decompressed);
224 try testing.expectEqualSlices(u8, data, decompressed.buffered());
225 }
226
227 // compressor writer interface
228 {
229 var compressed: Writer = .fixed(&cmp_buf);
230 var cmp = try Compress.init(&compressed, &.{}, .{
231 .level = level,
232 .container = container,
233 });
234 var cmp_wrt = cmp.writer();
235 try cmp_wrt.writeAll(data);
236 try cmp.finish();
237
238 try testing.expectEqual(compressed_size, compressed.pos);
239 }
240 // decompressor reader interface
241 {
242 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
243 var decompress: Decompress = .init(&compressed, container, &.{});
244 const n = try decompress.reader.readSliceShort(&dcm_buf);
245 try testing.expectEqual(data.len, n);
246 try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);
247 }
248 }
249 }
250 // huffman only compression
251 {
252 for (Container.list) |container| {
253 var compressed_size: usize = if (case.huffman_only_size > 0)
254 case.huffman_only_size - Container.gzip.size() + container.size()
255 else
256 0;
257
258 // compress original stream to compressed stream
259 {
260 var original: std.Io.Reader = .fixed(data);
261 var compressed: Writer = .fixed(&cmp_buf);
262 var cmp = try Compress.Huffman.init(container, &compressed);
263 try cmp.compress(original.reader());
264 try cmp.finish();
265 if (compressed_size == 0) {
266 if (container == .gzip)
267 print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos });
268 compressed_size = compressed.pos;
269 }
270 try testing.expectEqual(compressed_size, compressed.pos);
271 }
272 // decompress compressed stream to decompressed stream
273 {
274 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
275 var decompress: Decompress = .init(&compressed, container, &.{});
276 var decompressed: Writer = .fixed(&dcm_buf);
277 _ = try decompress.reader.streamRemaining(&decompressed);
278 try testing.expectEqualSlices(u8, data, decompressed.buffered());
279 }
280 }
281 }
282
283 // store only
284 {
285 for (Container.list) |container| {
286 var compressed_size: usize = if (case.store_size > 0)
287 case.store_size - Container.gzip.size() + container.size()
288 else
289 0;
290
291 // compress original stream to compressed stream
292 {
293 var original: std.Io.Reader = .fixed(data);
294 var compressed: Writer = .fixed(&cmp_buf);
295 var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed);
296 try cmp.compress(original.reader());
297 try cmp.finish();
298 if (compressed_size == 0) {
299 if (container == .gzip)
300 print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos });
301 compressed_size = compressed.pos;
302 }
303
304 try testing.expectEqual(compressed_size, compressed.pos);
305 }
306 // decompress compressed stream to decompressed stream
307 {
308 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
309 var decompress: Decompress = .init(&compressed, container, &.{});
310 var decompressed: Writer = .fixed(&dcm_buf);
311 _ = try decompress.reader.streamRemaining(&decompressed);
312 try testing.expectEqualSlices(u8, data, decompressed.buffered());
313 }
314 }
315 }
316 }
317}
318
319fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
320 var in: std.Io.Reader = .fixed(compressed);
321 var aw: std.Io.Writer.Allocating = .init(testing.allocator);
322 defer aw.deinit();
323
324 var decompress: Decompress = .init(&in, container, &.{});
325 _ = try decompress.reader.streamRemaining(&aw.writer);
326 try testing.expectEqualSlices(u8, expected_plain, aw.getWritten());
327}
328
329test "don't read past deflate stream's end" {
330 try testDecompress(.zlib, &[_]u8{
331 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
332 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
333 0x83, 0x95, 0x0b, 0xf5,
334 }, &[_]u8{
335 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
336 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
337 0x00, 0x00, 0xff, 0xff, 0xff,
338 });
339}
340
341test "zlib header" {
342 // Truncated header
343 try testing.expectError(
344 error.EndOfStream,
345 testDecompress(.zlib, &[_]u8{0x78}, ""),
346 );
347 // Wrong CM
348 try testing.expectError(
349 error.BadZlibHeader,
350 testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""),
351 );
352 // Wrong CINFO
353 try testing.expectError(
354 error.BadZlibHeader,
355 testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""),
356 );
357 // Wrong checksum
358 try testing.expectError(
359 error.WrongZlibChecksum,
360 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
361 );
362 // Truncated checksum
363 try testing.expectError(
364 error.EndOfStream,
365 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
366 );
367}
368
369test "gzip header" {
370 // Truncated header
371 try testing.expectError(
372 error.EndOfStream,
373 testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined),
374 );
375 // Wrong CM
376 try testing.expectError(
377 error.BadGzipHeader,
378 testDecompress(.gzip, &[_]u8{
379 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
380 0x00, 0x03,
381 }, undefined),
382 );
383
384 // Wrong checksum
385 try testing.expectError(
386 error.WrongGzipChecksum,
387 testDecompress(.gzip, &[_]u8{
388 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
389 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
390 0x00, 0x00, 0x00, 0x00,
391 }, undefined),
392 );
393 // Truncated checksum
394 try testing.expectError(
395 error.EndOfStream,
396 testDecompress(.gzip, &[_]u8{
397 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
398 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
399 }, undefined),
400 );
401 // Wrong initial size
402 try testing.expectError(
403 error.WrongGzipSize,
404 testDecompress(.gzip, &[_]u8{
405 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
406 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
407 0x00, 0x00, 0x00, 0x01,
408 }, undefined),
409 );
410 // Truncated initial size field
411 try testing.expectError(
412 error.EndOfStream,
413 testDecompress(.gzip, &[_]u8{
414 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
415 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
416 0x00, 0x00, 0x00,
417 }, undefined),
418 );
419
420 try testDecompress(.gzip, &[_]u8{
421 // GZIP header
422 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
423 // header.FHCRC (should cover entire header)
424 0x99, 0xd6,
425 // GZIP data
426 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
427 }, "");
428}
429
430test "public interface" {
431 const plain_data_buf = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };
432
433 // deflate final stored block, header + plain (stored) data
434 const deflate_block = [_]u8{
435 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
436 } ++ plain_data_buf;
437
438 const plain_data: []const u8 = &plain_data_buf;
439 const gzip_data: []const u8 = &deflate_block;
440
441 //// gzip header/footer + deflate block
442 //const gzip_data =
443 // [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
444 // deflate_block ++
445 // [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
446
447 //// zlib header/footer + deflate block
448 //const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
449 // deflate_block ++
450 // [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
451
452 // TODO
453 //const gzip = @import("gzip.zig");
454 //const zlib = @import("zlib.zig");
455
456 var buffer1: [64]u8 = undefined;
457 var buffer2: [64]u8 = undefined;
458
459 // decompress
460 {
461 var plain: Writer = .fixed(&buffer2);
462 var in: std.Io.Reader = .fixed(gzip_data);
463 var d: Decompress = .init(&in, .raw, &.{});
464 _ = try d.reader.streamRemaining(&plain);
465 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
466 }
467
468 // compress/decompress
469 {
470 var plain: Writer = .fixed(&buffer2);
471 var compressed: Writer = .fixed(&buffer1);
472
473 var cmp: Compress = .init(&compressed, &.{}, .{});
474 try cmp.writer.writeAll(plain_data);
475 try cmp.end();
476
477 var r: std.Io.Reader = .fixed(&buffer1);
478 var d: Decompress = .init(&r, .raw, &.{});
479 _ = try d.reader.streamRemaining(&plain);
480 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
481 }
482
483 // compressor/decompressor
484 {
485 var plain: Writer = .fixed(&buffer2);
486 var compressed: Writer = .fixed(&buffer1);
487
488 var cmp: Compress = .init(&compressed, &.{}, .{});
489 try cmp.writer.writeAll(plain_data);
490 try cmp.end();
491
492 var r: std.Io.Reader = .fixed(&buffer1);
493 var dcp = Decompress(&r);
494 try dcp.decompress(&plain);
495 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
496 }
497
498 // huffman
499 {
500 // huffman compress/decompress
501 {
502 var plain: Writer = .fixed(&buffer2);
503 var compressed: Writer = .fixed(&buffer1);
504
505 var in: std.Io.Reader = .fixed(plain_data);
506 try HuffmanEncoder.compress(&in, &compressed);
507
508 var r: std.Io.Reader = .fixed(&buffer1);
509 var d: Decompress = .init(&r, .raw, &.{});
510 _ = try d.reader.streamRemaining(&plain);
511 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
512 }
513
514 // huffman compressor/decompressor
515 {
516 var plain: Writer = .fixed(&buffer2);
517 var compressed: Writer = .fixed(&buffer1);
518
519 var in: std.Io.Reader = .fixed(plain_data);
520 var cmp = try HuffmanEncoder.Compressor(&compressed);
521 try cmp.compress(&in);
522 try cmp.finish();
523
524 var r: std.Io.Reader = .fixed(&buffer1);
525 var d: Decompress = .init(&r, .raw, &.{});
526 _ = try d.reader.streamRemaining(&plain);
527 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
528 }
529 }
530
531 // TODO
532 //{
533 // // store compress/decompress
534 // {
535 // var plain: Writer = .fixed(&buffer2);
536 // var compressed: Writer = .fixed(&buffer1);
537
538 // var in: std.Io.Reader = .fixed(plain_data);
539 // try store.compress(&in, &compressed);
540
541 // var r: std.Io.Reader = .fixed(&buffer1);
542 // var d: Decompress = .init(&r, .raw, &.{});
543 // _ = try d.reader.streamRemaining(&plain);
544 // try testing.expectEqualSlices(u8, plain_data, plain.buffered());
545 // }
546
547 // // store compressor/decompressor
548 // {
549 // var plain: Writer = .fixed(&buffer2);
550 // var compressed: Writer = .fixed(&buffer1);
551
552 // var in: std.Io.Reader = .fixed(plain_data);
553 // var cmp = try store.compressor(&compressed);
554 // try cmp.compress(&in);
555 // try cmp.finish();
556
557 // var r: std.Io.Reader = .fixed(&buffer1);
558 // var d: Decompress = .init(&r, .raw, &.{});
559 // _ = try d.reader.streamRemaining(&plain);
560 // try testing.expectEqualSlices(u8, plain_data, plain.buffered());
561 // }
562 //}
563}
564
565pub const match = struct {
566 pub const base_length = 3; // smallest match length per the RFC section 3.2.5
567 pub const min_length = 4; // min length used in this algorithm
568 pub const max_length = 258;
569
570 pub const min_distance = 1;
571 pub const max_distance = 32768;
572};
573
574pub const history_len = match.max_distance;
575
576pub const lookup = struct {
577 pub const bits = 15;
578 pub const len = 1 << bits;
579 pub const shift = 32 - bits;
580};
581
582test "zlib should not overshoot" {
583 // Compressed zlib data with extra 4 bytes at the end.
584 const data = [_]u8{
585 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
586 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
587 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
588 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
589 };
590
591 var reader: std.Io.Reader = .fixed(&data);
592
593 var decompress: Decompress = .init(&reader, .zlib, &.{});
594 var out: [128]u8 = undefined;
595
596 {
597 const n = try decompress.reader.readSliceShort(out[0..]);
598
599 // Expected decompressed data
600 try std.testing.expectEqual(46, n);
601 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
602
603 // Decompressor don't overshoot underlying reader.
604 // It is leaving it at the end of compressed data chunk.
605 try std.testing.expectEqual(data.len - 4, reader.seek);
606 // TODO what was this testing, exactly?
607 //try std.testing.expectEqual(0, decompress.unreadBytes());
608 }
609
610 // 4 bytes after compressed chunk are available in reader.
611 const n = try reader.readSliceShort(out[0..]);
612 try std.testing.expectEqual(n, 4);
613 try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
614}
615
616test {153test {
617 _ = HuffmanEncoder;154 _ = HuffmanEncoder;
618 _ = Compress;155 _ = Compress;
lib/std/compress/flate/BlockWriter.zig+20-1
...@@ -31,7 +31,26 @@ fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code,...@@ -31,7 +31,26 @@ fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code,
31fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,31fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
32distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,32distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
3333
34pub fn init(bw: *BlockWriter) void {34pub fn init(output: *Writer) BlockWriter {
35 return .{
36 .output = output,
37 .codegen_freq = undefined,
38 .literal_freq = undefined,
39 .distance_freq = undefined,
40 .codegen = undefined,
41 .literal_encoding = undefined,
42 .distance_encoding = undefined,
43 .codegen_encoding = undefined,
44 .fixed_literal_encoding = undefined,
45 .fixed_distance_encoding = undefined,
46 .huff_distance = undefined,
47 .fixed_literal_codes = undefined,
48 .fixed_distance_codes = undefined,
49 .distance_codes = undefined,
50 };
51}
52
53pub fn initBuffers(bw: *BlockWriter) void {
35 bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes);54 bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes);
36 bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes);55 bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes);
37 bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes);56 bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes);
lib/std/compress/flate/Compress.zig+20-271
...@@ -122,22 +122,7 @@ pub const Options = struct {...@@ -122,22 +122,7 @@ pub const Options = struct {
122122
123pub fn init(output: *Writer, buffer: []u8, options: Options) Compress {123pub fn init(output: *Writer, buffer: []u8, options: Options) Compress {
124 return .{124 return .{
125 .block_writer = .{125 .block_writer = .init(output),
126 .output = output,
127 .codegen_freq = undefined,
128 .literal_freq = undefined,
129 .distance_freq = undefined,
130 .codegen = undefined,
131 .literal_encoding = undefined,
132 .distance_encoding = undefined,
133 .codegen_encoding = undefined,
134 .fixed_literal_encoding = undefined,
135 .fixed_distance_encoding = undefined,
136 .huff_distance = undefined,
137 .fixed_literal_codes = undefined,
138 .fixed_distance_codes = undefined,
139 .distance_codes = undefined,
140 },
141 .level = .get(options.level),126 .level = .get(options.level),
142 .hasher = .init(options.container),127 .hasher = .init(options.container),
143 .state = .header,128 .state = .header,
...@@ -188,20 +173,21 @@ fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize...@@ -188,20 +173,21 @@ fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize
188 }173 }
189174
190 const buffered = me.buffered();175 const buffered = me.buffered();
191 const min_lookahead = flate.match.min_length + flate.match.max_length;176 const min_lookahead = Token.min_length + Token.max_length;
192 const history_plus_lookahead_len = flate.history_len + min_lookahead;177 const history_plus_lookahead_len = flate.history_len + min_lookahead;
193 if (buffered.len < history_plus_lookahead_len) return 0;178 if (buffered.len < history_plus_lookahead_len) return 0;
194 const lookahead = buffered[flate.history_len..];179 const lookahead = buffered[flate.history_len..];
195180
196 _ = lookahead;
197 // TODO tokenize181 // TODO tokenize
182 _ = lookahead;
198 //c.hasher.update(lookahead[0..n]);183 //c.hasher.update(lookahead[0..n]);
199 @panic("TODO");184 @panic("TODO");
200}185}
201186
202pub fn end(c: *Compress) !void {187pub fn end(c: *Compress) !void {
203 try endUnflushed(c);188 try endUnflushed(c);
204 try c.output.flush();189 const out = c.block_writer.output;
190 try out.flush();
205}191}
206192
207pub fn endUnflushed(c: *Compress) !void {193pub fn endUnflushed(c: *Compress) !void {
...@@ -227,7 +213,7 @@ pub fn endUnflushed(c: *Compress) !void {...@@ -227,7 +213,7 @@ pub fn endUnflushed(c: *Compress) !void {
227 // Checksum value of the uncompressed data (excluding any213 // Checksum value of the uncompressed data (excluding any
228 // dictionary data) computed according to Adler-32214 // dictionary data) computed according to Adler-32
229 // algorithm.215 // algorithm.
230 std.mem.writeInt(u32, try out.writableArray(4), zlib.final, .big);216 std.mem.writeInt(u32, try out.writableArray(4), zlib.adler, .big);
231 },217 },
232 .raw => {},218 .raw => {},
233 }219 }
...@@ -243,15 +229,16 @@ pub const Simple = struct {...@@ -243,15 +229,16 @@ pub const Simple = struct {
243229
244 pub const Strategy = enum { huffman, store };230 pub const Strategy = enum { huffman, store };
245231
246 pub fn init(out: *Writer, buffer: []u8, container: Container) !Simple {232 pub fn init(output: *Writer, buffer: []u8, container: Container, strategy: Strategy) !Simple {
247 const self: Simple = .{233 const header = container.header();
234 try output.writeAll(header);
235 return .{
248 .buffer = buffer,236 .buffer = buffer,
249 .wp = 0,237 .wp = 0,
250 .block_writer = .init(out),238 .block_writer = .init(output),
251 .hasher = .init(container),239 .hasher = .init(container),
240 .strategy = strategy,
252 };241 };
253 try container.writeHeader(self.out);
254 return self;
255 }242 }
256243
257 pub fn flush(self: *Simple) !void {244 pub fn flush(self: *Simple) !void {
...@@ -263,7 +250,7 @@ pub const Simple = struct {...@@ -263,7 +250,7 @@ pub const Simple = struct {
263 pub fn finish(self: *Simple) !void {250 pub fn finish(self: *Simple) !void {
264 try self.flushBuffer(true);251 try self.flushBuffer(true);
265 try self.block_writer.flush();252 try self.block_writer.flush();
266 try self.hasher.container().writeFooter(&self.hasher, self.out);253 try self.hasher.container().writeFooter(&self.hasher, self.block_writer.output);
267 }254 }
268255
269 fn flushBuffer(self: *Simple, final: bool) !void {256 fn flushBuffer(self: *Simple, final: bool) !void {
...@@ -300,7 +287,13 @@ test "generate a Huffman code from an array of frequencies" {...@@ -300,7 +287,13 @@ test "generate a Huffman code from an array of frequencies" {
300 };287 };
301288
302 var codes: [19]HuffmanEncoder.Code = undefined;289 var codes: [19]HuffmanEncoder.Code = undefined;
303 var enc: HuffmanEncoder = .{ .codes = &codes };290 var enc: HuffmanEncoder = .{
291 .codes = &codes,
292 .freq_cache = undefined,
293 .bit_count = undefined,
294 .lns = undefined,
295 .lfs = undefined,
296 };
304 enc.generate(freqs[0..], 7);297 enc.generate(freqs[0..], 7);
305298
306 try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));299 try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
...@@ -337,247 +330,3 @@ test "generate a Huffman code from an array of frequencies" {...@@ -337,247 +330,3 @@ test "generate a Huffman code from an array of frequencies" {
337 try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);330 try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
338 try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);331 try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
339}332}
340
341test "tokenization" {
342 const L = Token.initLiteral;
343 const M = Token.initMatch;
344
345 const cases = [_]struct {
346 data: []const u8,
347 tokens: []const Token,
348 }{
349 .{
350 .data = "Blah blah blah blah blah!",
351 .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
352 },
353 .{
354 .data = "ABCDEABCD ABCDEABCD",
355 .tokens = &[_]Token{
356 L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
357 L('A'), M(10, 8),
358 },
359 },
360 };
361
362 for (cases) |c| {
363 inline for (Container.list) |container| { // for each wrapping
364
365 var cw = std.Io.countingWriter(std.Io.null_writer);
366 const cww = cw.writer();
367 var df = try Compress(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
368
369 _ = try df.write(c.data);
370 try df.flush();
371
372 // df.token_writer.show();
373 try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
374 try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
375
376 try testing.expectEqual(container.headerSize(), cw.bytes_written);
377 try df.finish();
378 try testing.expectEqual(container.size(), cw.bytes_written);
379 }
380 }
381}
382
383// Tests that tokens written are equal to expected token list.
384const TestTokenWriter = struct {
385 const Self = @This();
386
387 pos: usize = 0,
388 actual: [128]Token = undefined,
389
390 pub fn init(_: anytype) Self {
391 return .{};
392 }
393 pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
394 for (tokens) |t| {
395 self.actual[self.pos] = t;
396 self.pos += 1;
397 }
398 }
399
400 pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
401
402 pub fn get(self: *Self) []Token {
403 return self.actual[0..self.pos];
404 }
405
406 pub fn show(self: *Self) void {
407 std.debug.print("\n", .{});
408 for (self.get()) |t| {
409 t.show();
410 }
411 }
412
413 pub fn flush(_: *Self) !void {}
414};
415
416test "file tokenization" {
417 const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
418 const cases = [_]struct {
419 data: []const u8, // uncompressed content
420 // expected number of tokens producet in deflate tokenization
421 tokens_count: [levels.len]usize = .{0} ** levels.len,
422 }{
423 .{
424 .data = @embedFile("testdata/rfc1951.txt"),
425 .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
426 },
427
428 .{
429 .data = @embedFile("testdata/block_writer/huffman-null-max.input"),
430 .tokens_count = .{ 257, 257, 257, 257, 257, 257 },
431 },
432 .{
433 .data = @embedFile("testdata/block_writer/huffman-pi.input"),
434 .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
435 },
436 .{
437 .data = @embedFile("testdata/block_writer/huffman-text.input"),
438 .tokens_count = .{ 235, 234, 234, 234, 234, 234 },
439 },
440 .{
441 .data = @embedFile("testdata/fuzz/roundtrip1.input"),
442 .tokens_count = .{ 333, 331, 331, 331, 331, 331 },
443 },
444 .{
445 .data = @embedFile("testdata/fuzz/roundtrip2.input"),
446 .tokens_count = .{ 334, 334, 334, 334, 334, 334 },
447 },
448 };
449
450 for (cases) |case| { // for each case
451 const data = case.data;
452
453 for (levels, 0..) |level, i| { // for each compression level
454 var original: std.Io.Reader = .fixed(data);
455
456 // buffer for decompressed data
457 var al = std.ArrayList(u8).init(testing.allocator);
458 defer al.deinit();
459 const writer = al.writer();
460
461 // create compressor
462 const WriterType = @TypeOf(writer);
463 const TokenWriter = TokenDecoder(@TypeOf(writer));
464 var cmp = try Compress(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
465
466 // Stream uncompressed `original` data to the compressor. It will
467 // produce tokens list and pass that list to the TokenDecoder. This
468 // TokenDecoder uses CircularBuffer from inflate to convert list of
469 // tokens back to the uncompressed stream.
470 try cmp.compress(original.reader());
471 try cmp.flush();
472 const expected_count = case.tokens_count[i];
473 const actual = cmp.block_writer.tokens_count;
474 if (expected_count == 0) {
475 std.debug.print("actual token count {d}\n", .{actual});
476 } else {
477 try testing.expectEqual(expected_count, actual);
478 }
479
480 try testing.expectEqual(data.len, al.items.len);
481 try testing.expectEqualSlices(u8, data, al.items);
482 }
483 }
484}
485
486const TokenDecoder = struct {
487 output: *Writer,
488 tokens_count: usize,
489
490 pub fn init(output: *Writer) TokenDecoder {
491 return .{
492 .output = output,
493 .tokens_count = 0,
494 };
495 }
496
497 pub fn write(self: *TokenDecoder, tokens: []const Token, _: bool, _: ?[]const u8) !void {
498 self.tokens_count += tokens.len;
499 for (tokens) |t| {
500 switch (t.kind) {
501 .literal => self.hist.write(t.literal()),
502 .match => try self.hist.writeMatch(t.length(), t.distance()),
503 }
504 if (self.hist.free() < 285) try self.flushWin();
505 }
506 try self.flushWin();
507 }
508
509 fn flushWin(self: *TokenDecoder) !void {
510 while (true) {
511 const buf = self.hist.read();
512 if (buf.len == 0) break;
513 try self.output.writeAll(buf);
514 }
515 }
516};
517
518test "store simple compressor" {
519 if (true) return error.SkipZigTest;
520 //const data = "Hello world!";
521 //const expected = [_]u8{
522 // 0x1, // block type 0, final bit set
523 // 0xc, 0x0, // len = 12
524 // 0xf3, 0xff, // ~len
525 // 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
526 // //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
527 //};
528
529 //var fbs: std.Io.Reader = .fixed(data);
530 //var al = std.ArrayList(u8).init(testing.allocator);
531 //defer al.deinit();
532
533 //var cmp = try store.compressor(.raw, al.writer());
534 //try cmp.compress(&fbs);
535 //try cmp.finish();
536 //try testing.expectEqualSlices(u8, &expected, al.items);
537
538 //fbs = .fixed(data);
539 //try al.resize(0);
540
541 //// huffman only compresoor will also emit store block for this small sample
542 //var hc = try huffman.compressor(.raw, al.writer());
543 //try hc.compress(&fbs);
544 //try hc.finish();
545 //try testing.expectEqualSlices(u8, &expected, al.items);
546}
547
548test "sliding window match" {
549 const data = "Blah blah blah blah blah!";
550 var win: Writer = .{};
551 try expect(win.write(data) == data.len);
552 try expect(win.wp == data.len);
553 try expect(win.rp == 0);
554
555 // length between l symbols
556 try expect(win.match(1, 6, 0) == 18);
557 try expect(win.match(1, 11, 0) == 13);
558 try expect(win.match(1, 16, 0) == 8);
559 try expect(win.match(1, 21, 0) == 0);
560
561 // position 15 = "blah blah!"
562 // position 20 = "blah!"
563 try expect(win.match(15, 20, 0) == 4);
564 try expect(win.match(15, 20, 3) == 4);
565 try expect(win.match(15, 20, 4) == 0);
566}
567
568test "sliding window slide" {
569 var win: Writer = .{};
570 win.wp = Writer.buffer_len - 11;
571 win.rp = Writer.buffer_len - 111;
572 win.buffer[win.rp] = 0xab;
573 try expect(win.lookahead().len == 100);
574 try expect(win.tokensBuffer().?.len == win.rp);
575
576 const n = win.slide();
577 try expect(n == 32757);
578 try expect(win.buffer[win.rp] == 0xab);
579 try expect(win.rp == Writer.hist_len - 111);
580 try expect(win.wp == Writer.hist_len - 11);
581 try expect(win.lookahead().len == 100);
582 try expect(win.tokensBuffer() == null);
583}
lib/std/compress/flate/Decompress.zig+184-33
...@@ -4,8 +4,8 @@ const Container = flate.Container;...@@ -4,8 +4,8 @@ const Container = flate.Container;
4const Token = @import("Token.zig");4const Token = @import("Token.zig");
5const testing = std.testing;5const testing = std.testing;
6const Decompress = @This();6const Decompress = @This();
7const Writer = std.io.Writer;7const Writer = std.Io.Writer;
8const Reader = std.io.Reader;8const Reader = std.Io.Reader;
99
10input: *Reader,10input: *Reader,
11reader: Reader,11reader: Reader,
...@@ -129,7 +129,7 @@ fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {...@@ -129,7 +129,7 @@ fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
129 return sym;129 return sym;
130}130}
131131
132pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize {132pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
133 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));133 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
134 return readInner(d, w, limit) catch |err| switch (err) {134 return readInner(d, w, limit) catch |err| switch (err) {
135 error.EndOfStream => return error.EndOfStream,135 error.EndOfStream => return error.EndOfStream,
...@@ -143,7 +143,8 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!us...@@ -143,7 +143,8 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!us
143 };143 };
144}144}
145145
146fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.StreamError)!usize {146fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.StreamError)!usize {
147 var remaining = @intFromEnum(limit);
147 const in = d.input;148 const in = d.input;
148 sw: switch (d.state) {149 sw: switch (d.state) {
149 .protocol_header => switch (d.hasher.container()) {150 .protocol_header => switch (d.hasher.container()) {
...@@ -182,15 +183,9 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S...@@ -182,15 +183,9 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
182 continue :sw .block_header;183 continue :sw .block_header;
183 },184 },
184 .zlib => {185 .zlib => {
185 const Header = extern struct {186 const header = try in.takeArray(2);
186 cmf: packed struct(u8) {187 const cmf: packed struct(u8) { cm: u4, cinfo: u4 } = @bitCast(header[0]);
187 cm: u4,188 if (cmf.cm != 8 or cmf.cinfo > 7) return error.BadZlibHeader;
188 cinfo: u4,
189 },
190 flg: u8,
191 };
192 const header = try in.takeStruct(Header);
193 if (header.cmf.cm != 8 or header.cmf.cinfo > 7) return error.BadZlibHeader;
194 continue :sw .block_header;189 continue :sw .block_header;
195 },190 },
196 .raw => continue :sw .block_header,191 .raw => continue :sw .block_header,
...@@ -219,7 +214,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S...@@ -219,7 +214,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
219 // lengths for code lengths214 // lengths for code lengths
220 var cl_lens = [_]u4{0} ** 19;215 var cl_lens = [_]u4{0} ** 19;
221 for (0..hclen) |i| {216 for (0..hclen) |i| {
222 cl_lens[flate.huffman.codegen_order[i]] = try d.takeBits(u3);217 cl_lens[flate.HuffmanEncoder.codegen_order[i]] = try d.takeBits(u3);
223 }218 }
224 var cl_dec: CodegenDecoder = .{};219 var cl_dec: CodegenDecoder = .{};
225 try cl_dec.generate(&cl_lens);220 try cl_dec.generate(&cl_lens);
...@@ -259,52 +254,56 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S...@@ -259,52 +254,56 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
259 return n;254 return n;
260 },255 },
261 .fixed_block => {256 .fixed_block => {
262 const start = w.count;257 while (remaining > 0) {
263 while (@intFromEnum(limit) > w.count - start) {
264 const code = try d.readFixedCode();258 const code = try d.readFixedCode();
265 switch (code) {259 switch (code) {
266 0...255 => try w.writeBytePreserve(flate.history_len, @intCast(code)),260 0...255 => {
261 try w.writeBytePreserve(flate.history_len, @intCast(code));
262 remaining -= 1;
263 },
267 256 => {264 256 => {
268 d.state = if (d.final_block) .protocol_footer else .block_header;265 d.state = if (d.final_block) .protocol_footer else .block_header;
269 return w.count - start;266 return @intFromEnum(limit) - remaining;
270 },267 },
271 257...285 => {268 257...285 => {
272 // Handles fixed block non literal (length) code.269 // Handles fixed block non literal (length) code.
273 // Length code is followed by 5 bits of distance code.270 // Length code is followed by 5 bits of distance code.
274 const length = try d.decodeLength(@intCast(code - 257));271 const length = try d.decodeLength(@intCast(code - 257));
275 const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5));272 const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5));
276 try writeMatch(w, length, distance);273 remaining = try writeMatch(w, length, distance, remaining);
277 },274 },
278 else => return error.InvalidCode,275 else => return error.InvalidCode,
279 }276 }
280 }277 }
281 d.state = .fixed_block;278 d.state = .fixed_block;
282 return w.count - start;279 return @intFromEnum(limit) - remaining;
283 },280 },
284 .dynamic_block => {281 .dynamic_block => {
285 // In larger archives most blocks are usually dynamic, so decompression282 // In larger archives most blocks are usually dynamic, so
286 // performance depends on this logic.283 // decompression performance depends on this logic.
287 const start = w.count;284 while (remaining > 0) {
288 while (@intFromEnum(limit) > w.count - start) {
289 const sym = try d.decodeSymbol(&d.lit_dec);285 const sym = try d.decodeSymbol(&d.lit_dec);
290286
291 switch (sym.kind) {287 switch (sym.kind) {
292 .literal => try w.writeBytePreserve(flate.history_len, sym.symbol),288 .literal => {
289 try w.writeBytePreserve(flate.history_len, sym.symbol);
290 remaining -= 1;
291 },
293 .match => {292 .match => {
294 // Decode match backreference <length, distance>293 // Decode match backreference <length, distance>
295 const length = try d.decodeLength(sym.symbol);294 const length = try d.decodeLength(sym.symbol);
296 const dsm = try d.decodeSymbol(&d.dst_dec);295 const dsm = try d.decodeSymbol(&d.dst_dec);
297 const distance = try d.decodeDistance(dsm.symbol);296 const distance = try d.decodeDistance(dsm.symbol);
298 try writeMatch(w, length, distance);297 remaining = try writeMatch(w, length, distance, remaining);
299 },298 },
300 .end_of_block => {299 .end_of_block => {
301 d.state = if (d.final_block) .protocol_footer else .block_header;300 d.state = if (d.final_block) .protocol_footer else .block_header;
302 return w.count - start;301 return @intFromEnum(limit) - remaining;
303 },302 },
304 }303 }
305 }304 }
306 d.state = .dynamic_block;305 d.state = .dynamic_block;
307 return w.count - start;306 return @intFromEnum(limit) - remaining;
308 },307 },
309 .protocol_footer => {308 .protocol_footer => {
310 d.alignBitsToByte();309 d.alignBitsToByte();
...@@ -314,7 +313,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S...@@ -314,7 +313,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
314 if (try in.takeInt(u32, .little) != gzip.count) return error.WrongGzipSize;313 if (try in.takeInt(u32, .little) != gzip.count) return error.WrongGzipSize;
315 },314 },
316 .zlib => |*zlib| {315 .zlib => |*zlib| {
317 const chksum: u32 = @byteSwap(zlib.final());316 const chksum: u32 = @byteSwap(zlib.adler);
318 if (try in.takeInt(u32, .big) != chksum) return error.WrongZlibChecksum;317 if (try in.takeInt(u32, .big) != chksum) return error.WrongZlibChecksum;
319 },318 },
320 .raw => {},319 .raw => {},
...@@ -328,10 +327,11 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S...@@ -328,10 +327,11 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
328327
329/// Write match (back-reference to the same data slice) starting at `distance`328/// Write match (back-reference to the same data slice) starting at `distance`
330/// back from current write position, and `length` of bytes.329/// back from current write position, and `length` of bytes.
331fn writeMatch(bw: *Writer, length: u16, distance: u16) !void {330fn writeMatch(w: *Writer, length: u16, distance: u16, remaining: usize) !usize {
332 _ = bw;331 _ = w;
333 _ = length;332 _ = length;
334 _ = distance;333 _ = distance;
334 _ = remaining;
335 @panic("TODO");335 @panic("TODO");
336}336}
337337
...@@ -622,7 +622,13 @@ test "init/find" {...@@ -622,7 +622,13 @@ test "init/find" {
622test "encode/decode literals" {622test "encode/decode literals" {
623 var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined;623 var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined;
624 for (1..286) |j| { // for all different number of codes624 for (1..286) |j| { // for all different number of codes
625 var enc: flate.HuffmanEncoder = .{ .codes = &codes };625 var enc: flate.HuffmanEncoder = .{
626 .codes = &codes,
627 .freq_cache = undefined,
628 .bit_count = undefined,
629 .lns = undefined,
630 .lfs = undefined,
631 };
626 // create frequencies632 // create frequencies
627 var freq = [_]u16{0} ** 286;633 var freq = [_]u16{0} ** 286;
628 freq[256] = 1; // ensure we have end of block code634 freq[256] = 1; // ensure we have end of block code
...@@ -857,7 +863,7 @@ test "fuzzing tests" {...@@ -857,7 +863,7 @@ test "fuzzing tests" {
857 const r = &decompress.reader;863 const r = &decompress.reader;
858 if (c.err) |expected_err| {864 if (c.err) |expected_err| {
859 try testing.expectError(error.ReadFailed, r.streamRemaining(&aw.writer));865 try testing.expectError(error.ReadFailed, r.streamRemaining(&aw.writer));
860 try testing.expectError(expected_err, decompress.read_err.?);866 try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed);
861 } else {867 } else {
862 _ = try r.streamRemaining(&aw.writer);868 _ = try r.streamRemaining(&aw.writer);
863 try testing.expectEqualStrings(c.out, aw.getWritten());869 try testing.expectEqualStrings(c.out, aw.getWritten());
...@@ -891,3 +897,148 @@ test "reading into empty buffer" {...@@ -891,3 +897,148 @@ test "reading into empty buffer" {
891 var buf: [0]u8 = undefined;897 var buf: [0]u8 = undefined;
892 try testing.expectEqual(0, try r.readVec(&.{&buf}));898 try testing.expectEqual(0, try r.readVec(&.{&buf}));
893}899}
900
901test "don't read past deflate stream's end" {
902 try testDecompress(.zlib, &[_]u8{
903 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
904 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
905 0x83, 0x95, 0x0b, 0xf5,
906 }, &[_]u8{
907 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
908 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
909 0x00, 0x00, 0xff, 0xff, 0xff,
910 });
911}
912
913test "zlib header" {
914 // Truncated header
915 try testing.expectError(
916 error.EndOfStream,
917 testDecompress(.zlib, &[_]u8{0x78}, ""),
918 );
919 // Wrong CM
920 try testing.expectError(
921 error.BadZlibHeader,
922 testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""),
923 );
924 // Wrong CINFO
925 try testing.expectError(
926 error.BadZlibHeader,
927 testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""),
928 );
929 // Wrong checksum
930 try testing.expectError(
931 error.WrongZlibChecksum,
932 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
933 );
934 // Truncated checksum
935 try testing.expectError(
936 error.EndOfStream,
937 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
938 );
939}
940
941test "gzip header" {
942 // Truncated header
943 try testing.expectError(
944 error.EndOfStream,
945 testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined),
946 );
947 // Wrong CM
948 try testing.expectError(
949 error.BadGzipHeader,
950 testDecompress(.gzip, &[_]u8{
951 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
952 0x00, 0x03,
953 }, undefined),
954 );
955
956 // Wrong checksum
957 try testing.expectError(
958 error.WrongGzipChecksum,
959 testDecompress(.gzip, &[_]u8{
960 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
961 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
962 0x00, 0x00, 0x00, 0x00,
963 }, undefined),
964 );
965 // Truncated checksum
966 try testing.expectError(
967 error.EndOfStream,
968 testDecompress(.gzip, &[_]u8{
969 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
970 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
971 }, undefined),
972 );
973 // Wrong initial size
974 try testing.expectError(
975 error.WrongGzipSize,
976 testDecompress(.gzip, &[_]u8{
977 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
978 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
979 0x00, 0x00, 0x00, 0x01,
980 }, undefined),
981 );
982 // Truncated initial size field
983 try testing.expectError(
984 error.EndOfStream,
985 testDecompress(.gzip, &[_]u8{
986 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
987 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
988 0x00, 0x00, 0x00,
989 }, undefined),
990 );
991
992 try testDecompress(.gzip, &[_]u8{
993 // GZIP header
994 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
995 // header.FHCRC (should cover entire header)
996 0x99, 0xd6,
997 // GZIP data
998 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
999 }, "");
1000}
1001
1002fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
1003 var in: std.Io.Reader = .fixed(compressed);
1004 var aw: std.Io.Writer.Allocating = .init(testing.allocator);
1005 defer aw.deinit();
1006
1007 var decompress: Decompress = .init(&in, container, &.{});
1008 _ = try decompress.reader.streamRemaining(&aw.writer);
1009 try testing.expectEqualSlices(u8, expected_plain, aw.getWritten());
1010}
1011
1012test "zlib should not overshoot" {
1013 // Compressed zlib data with extra 4 bytes at the end.
1014 const data = [_]u8{
1015 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
1016 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
1017 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
1018 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
1019 };
1020
1021 var reader: std.Io.Reader = .fixed(&data);
1022
1023 var decompress: Decompress = .init(&reader, .zlib, &.{});
1024 var out: [128]u8 = undefined;
1025
1026 {
1027 const n = try decompress.reader.readSliceShort(out[0..]);
1028
1029 // Expected decompressed data
1030 try std.testing.expectEqual(46, n);
1031 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
1032
1033 // Decompressor don't overshoot underlying reader.
1034 // It is leaving it at the end of compressed data chunk.
1035 try std.testing.expectEqual(data.len - 4, reader.seek);
1036 // TODO what was this testing, exactly?
1037 //try std.testing.expectEqual(0, decompress.unreadBytes());
1038 }
1039
1040 // 4 bytes after compressed chunk are available in reader.
1041 const n = try reader.readSliceShort(out[0..]);
1042 try std.testing.expectEqual(n, 4);
1043 try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
1044}
lib/std/compress/flate/HuffmanEncoder.zig+3-2
...@@ -135,7 +135,7 @@ fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize)...@@ -135,7 +135,7 @@ fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize)
135 // of ancestors of the rightmost node at level i.135 // of ancestors of the rightmost node at level i.
136 // leaf_counts[i][j] is the number of literals at the left136 // leaf_counts[i][j] is the number of literals at the left
137 // of the level j ancestor.137 // of the level j ancestor.
138 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(0);138 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(@splat(0));
139139
140 {140 {
141 var level = @as(u32, 1);141 var level = @as(u32, 1);
...@@ -389,7 +389,8 @@ pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder...@@ -389,7 +389,8 @@ pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder
389}389}
390390
391test "generate a Huffman code for the fixed literal table specific to Deflate" {391test "generate a Huffman code for the fixed literal table specific to Deflate" {
392 const enc = fixedLiteralEncoder();392 var codes: [max_num_frequencies]Code = undefined;
393 const enc: HuffmanEncoder = .fixedLiteralEncoder(&codes);
393 for (enc.codes) |c| {394 for (enc.codes) |c| {
394 switch (c.len) {395 switch (c.len) {
395 7 => {396 7 => {
lib/std/compress/flate/Lookup.zig+10-5
...@@ -6,14 +6,19 @@ const std = @import("std");...@@ -6,14 +6,19 @@ const std = @import("std");
6const testing = std.testing;6const testing = std.testing;
7const expect = testing.expect;7const expect = testing.expect;
8const flate = @import("../flate.zig");8const flate = @import("../flate.zig");
9const Token = @import("Token.zig");
910
10const Lookup = @This();11const Lookup = @This();
1112
12const prime4 = 0x9E3779B1; // 4 bytes prime number 265443576113const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
13const chain_len = 2 * flate.history_len;14const chain_len = 2 * flate.history_len;
1415
16pub const bits = 15;
17pub const len = 1 << bits;
18pub const shift = 32 - bits;
19
15// Maps hash => first position20// Maps hash => first position
16head: [flate.lookup.len]u16 = [_]u16{0} ** flate.lookup.len,21head: [len]u16 = [_]u16{0} ** len,
17// Maps position => previous positions for the same hash value22// Maps position => previous positions for the same hash value
18chain: [chain_len]u16 = [_]u16{0} ** (chain_len),23chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
1924
...@@ -52,8 +57,8 @@ pub fn slide(self: *Lookup, n: u16) void {...@@ -52,8 +57,8 @@ pub fn slide(self: *Lookup, n: u16) void {
5257
53// Add `len` 4 bytes hashes from `data` into lookup.58// Add `len` 4 bytes hashes from `data` into lookup.
54// Position of the first byte is `pos`.59// Position of the first byte is `pos`.
55pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void {60pub fn bulkAdd(self: *Lookup, data: []const u8, length: u16, pos: u16) void {
56 if (len == 0 or data.len < flate.match.min_length) {61 if (length == 0 or data.len < Token.min_length) {
57 return;62 return;
58 }63 }
59 var hb =64 var hb =
...@@ -64,7 +69,7 @@ pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void {...@@ -64,7 +69,7 @@ pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void {
64 _ = self.set(hashu(hb), pos);69 _ = self.set(hashu(hb), pos);
6570
66 var i = pos;71 var i = pos;
67 for (4..@min(len + 3, data.len)) |j| {72 for (4..@min(length + 3, data.len)) |j| {
68 hb = (hb << 8) | @as(u32, data[j]);73 hb = (hb << 8) | @as(u32, data[j]);
69 i += 1;74 i += 1;
70 _ = self.set(hashu(hb), i);75 _ = self.set(hashu(hb), i);
...@@ -80,7 +85,7 @@ fn hash(b: *const [4]u8) u32 {...@@ -80,7 +85,7 @@ fn hash(b: *const [4]u8) u32 {
80}85}
8186
82fn hashu(v: u32) u32 {87fn hashu(v: u32) u32 {
83 return @intCast((v *% prime4) >> flate.lookup.shift);88 return @intCast((v *% prime4) >> shift);
84}89}
8590
86test add {91test add {
lib/std/compress/flate/Token.zig+13-7
...@@ -6,7 +6,6 @@ const std = @import("std");...@@ -6,7 +6,6 @@ const std = @import("std");
6const assert = std.debug.assert;6const assert = std.debug.assert;
7const print = std.debug.print;7const print = std.debug.print;
8const expect = std.testing.expect;8const expect = std.testing.expect;
9const match = std.compress.flate.match;
109
11const Token = @This();10const Token = @This();
1211
...@@ -21,16 +20,23 @@ dist: u15 = 0,...@@ -21,16 +20,23 @@ dist: u15 = 0,
21len_lit: u8 = 0,20len_lit: u8 = 0,
22kind: Kind = .literal,21kind: Kind = .literal,
2322
23pub const base_length = 3; // smallest match length per the RFC section 3.2.5
24pub const min_length = 4; // min length used in this algorithm
25pub const max_length = 258;
26
27pub const min_distance = 1;
28pub const max_distance = std.compress.flate.history_len;
29
24pub fn literal(t: Token) u8 {30pub fn literal(t: Token) u8 {
25 return t.len_lit;31 return t.len_lit;
26}32}
2733
28pub fn distance(t: Token) u16 {34pub fn distance(t: Token) u16 {
29 return @as(u16, t.dist) + match.min_distance;35 return @as(u16, t.dist) + min_distance;
30}36}
3137
32pub fn length(t: Token) u16 {38pub fn length(t: Token) u16 {
33 return @as(u16, t.len_lit) + match.base_length;39 return @as(u16, t.len_lit) + base_length;
34}40}
3541
36pub fn initLiteral(lit: u8) Token {42pub fn initLiteral(lit: u8) Token {
...@@ -40,12 +46,12 @@ pub fn initLiteral(lit: u8) Token {...@@ -40,12 +46,12 @@ pub fn initLiteral(lit: u8) Token {
40// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)46// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
41// length range 3 - 258, stored in len_lit as 0 - 255 (u8)47// length range 3 - 258, stored in len_lit as 0 - 255 (u8)
42pub fn initMatch(dist: u16, len: u16) Token {48pub fn initMatch(dist: u16, len: u16) Token {
43 assert(len >= match.min_length and len <= match.max_length);49 assert(len >= min_length and len <= max_length);
44 assert(dist >= match.min_distance and dist <= match.max_distance);50 assert(dist >= min_distance and dist <= max_distance);
45 return .{51 return .{
46 .kind = .match,52 .kind = .match,
47 .dist = @intCast(dist - match.min_distance),53 .dist = @intCast(dist - min_distance),
48 .len_lit = @intCast(len - match.base_length),54 .len_lit = @intCast(len - base_length),
49 };55 };
50}56}
5157