authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-27 16:05:32-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-31 22:10:11-07:00
log824c157e0c25a9337ffd036f7ad5cb811b1f18cd
treec58f974383513f7ff22e4e47a99f52793b1abff5
parent73c98ca0e6aa52b942b92135ecf0305362030733

std.compress.flate: finish reorganizing

7 files changed, 271 insertions(+), 803 deletions(-)

lib/std/compress/flate.zig+21-484
......@@ -1,7 +1,23 @@
1const builtin = @import("builtin");
21const std = @import("../std.zig");
3const testing = std.testing;
4const Writer = std.Io.Writer;
2
3/// When decompressing, the output buffer is used as the history window, so
4/// less than this may result in failure to decompress streams that were
5/// compressed with a larger window.
6pub const max_window_len = history_len * 2;
7
8pub const history_len = 32768;
9
10/// Deflate is a lossless data compression file format that uses a combination
11/// of LZ77 and Huffman coding.
12pub const Compress = @import("flate/Compress.zig");
13
14/// Inflate is the decoding process that takes a Deflate bitstream for
15/// decompression and correctly produces the original full-size data or file.
16pub const Decompress = @import("flate/Decompress.zig");
17
18/// Compression without Lempel-Ziv match searching. Faster compression, less
19/// memory requirements but bigger compressed sizes.
20pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig");
521
622/// Container of the deflate bit stream body. Container adds header before
723/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
......@@ -13,7 +29,6 @@ const Writer = std.Io.Writer;
1329/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
1430/// crc32 checksum and 4 bytes of uncompressed data length.
1531///
16///
1732/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
1833/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
1934pub const Container = enum {
......@@ -84,7 +99,7 @@ pub const Container = enum {
8499 pub fn init(containter: Container) Hasher {
85100 return switch (containter) {
86101 .gzip => .{ .gzip = .{} },
87 .zlib => .{ .zlib = .init() },
102 .zlib => .{ .zlib = .{} },
88103 .raw => .raw,
89104 };
90105 }
......@@ -107,7 +122,7 @@ pub const Container = enum {
107122 }
108123 }
109124
110 pub fn writeFooter(hasher: *Hasher, writer: *Writer) Writer.Error!void {
125 pub fn writeFooter(hasher: *Hasher, writer: *std.Io.Writer) std.Io.Writer.Error!void {
111126 var bits: [4]u8 = undefined;
112127 switch (hasher.*) {
113128 .gzip => |*gzip| {
......@@ -135,484 +150,6 @@ pub const Container = enum {
135150 };
136151};
137152
138/// When decompressing, the output buffer is used as the history window, so
139/// less than this may result in failure to decompress streams that were
140/// compressed with a larger window.
141pub const max_window_len = 1 << 16;
142
143/// Deflate is a lossless data compression file format that uses a combination
144/// of LZ77 and Huffman coding.
145pub const Compress = @import("flate/Compress.zig");
146
147/// Inflate is the decoding process that takes a Deflate bitstream for
148/// decompression and correctly produces the original full-size data or file.
149pub const Decompress = @import("flate/Decompress.zig");
150
151/// Compression without Lempel-Ziv match searching. Faster compression, less
152/// memory requirements but bigger compressed sizes.
153pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig");
154
155test "compress/decompress" {
156 const print = std.debug.print;
157 var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer
158 var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer
159
160 const levels = [_]Compress.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
161 const cases = [_]struct {
162 data: []const u8, // uncompressed content
163 // compressed data sizes per level 4-9
164 gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len,
165 huffman_only_size: usize = 0,
166 store_size: usize = 0,
167 }{
168 .{
169 .data = @embedFile("flate/testdata/rfc1951.txt"),
170 .gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 },
171 .huffman_only_size = 20287,
172 .store_size = 36967,
173 },
174 .{
175 .data = @embedFile("flate/testdata/fuzz/roundtrip1.input"),
176 .gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 },
177 .huffman_only_size = 393,
178 .store_size = 393,
179 },
180 .{
181 .data = @embedFile("flate/testdata/fuzz/roundtrip2.input"),
182 .gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 },
183 .huffman_only_size = 394,
184 .store_size = 394,
185 },
186 .{
187 .data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"),
188 .gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 },
189 .huffman_only_size = 498,
190 .store_size = 747,
191 },
192 };
193
194 for (cases, 0..) |case, case_no| {
195 const data = case.data;
196
197 for (levels, 0..) |level, i| {
198 for (Container.list) |container| {
199 var compressed_size: usize = if (case.gzip_sizes[i] > 0)
200 case.gzip_sizes[i] - Container.gzip.size() + container.size()
201 else
202 0;
203
204 // compress original stream to compressed stream
205 {
206 var compressed: Writer = .fixed(&cmp_buf);
207 var compress: Compress = .init(&compressed, &.{}, .{ .container = .raw, .level = level });
208 try compress.writer.writeAll(data);
209 try compress.end();
210
211 if (compressed_size == 0) {
212 if (container == .gzip)
213 print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
214 compressed_size = compressed.end;
215 }
216 try testing.expectEqual(compressed_size, compressed.end);
217 }
218 // decompress compressed stream to decompressed stream
219 {
220 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
221 var decompressed: Writer = .fixed(&dcm_buf);
222 var decompress: Decompress = .init(&compressed, container, &.{});
223 _ = try decompress.reader.streamRemaining(&decompressed);
224 try testing.expectEqualSlices(u8, data, decompressed.buffered());
225 }
226
227 // compressor writer interface
228 {
229 var compressed: Writer = .fixed(&cmp_buf);
230 var cmp = try Compress.init(&compressed, &.{}, .{
231 .level = level,
232 .container = container,
233 });
234 var cmp_wrt = cmp.writer();
235 try cmp_wrt.writeAll(data);
236 try cmp.finish();
237
238 try testing.expectEqual(compressed_size, compressed.pos);
239 }
240 // decompressor reader interface
241 {
242 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
243 var decompress: Decompress = .init(&compressed, container, &.{});
244 const n = try decompress.reader.readSliceShort(&dcm_buf);
245 try testing.expectEqual(data.len, n);
246 try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);
247 }
248 }
249 }
250 // huffman only compression
251 {
252 for (Container.list) |container| {
253 var compressed_size: usize = if (case.huffman_only_size > 0)
254 case.huffman_only_size - Container.gzip.size() + container.size()
255 else
256 0;
257
258 // compress original stream to compressed stream
259 {
260 var original: std.Io.Reader = .fixed(data);
261 var compressed: Writer = .fixed(&cmp_buf);
262 var cmp = try Compress.Huffman.init(container, &compressed);
263 try cmp.compress(original.reader());
264 try cmp.finish();
265 if (compressed_size == 0) {
266 if (container == .gzip)
267 print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos });
268 compressed_size = compressed.pos;
269 }
270 try testing.expectEqual(compressed_size, compressed.pos);
271 }
272 // decompress compressed stream to decompressed stream
273 {
274 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
275 var decompress: Decompress = .init(&compressed, container, &.{});
276 var decompressed: Writer = .fixed(&dcm_buf);
277 _ = try decompress.reader.streamRemaining(&decompressed);
278 try testing.expectEqualSlices(u8, data, decompressed.buffered());
279 }
280 }
281 }
282
283 // store only
284 {
285 for (Container.list) |container| {
286 var compressed_size: usize = if (case.store_size > 0)
287 case.store_size - Container.gzip.size() + container.size()
288 else
289 0;
290
291 // compress original stream to compressed stream
292 {
293 var original: std.Io.Reader = .fixed(data);
294 var compressed: Writer = .fixed(&cmp_buf);
295 var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed);
296 try cmp.compress(original.reader());
297 try cmp.finish();
298 if (compressed_size == 0) {
299 if (container == .gzip)
300 print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos });
301 compressed_size = compressed.pos;
302 }
303
304 try testing.expectEqual(compressed_size, compressed.pos);
305 }
306 // decompress compressed stream to decompressed stream
307 {
308 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
309 var decompress: Decompress = .init(&compressed, container, &.{});
310 var decompressed: Writer = .fixed(&dcm_buf);
311 _ = try decompress.reader.streamRemaining(&decompressed);
312 try testing.expectEqualSlices(u8, data, decompressed.buffered());
313 }
314 }
315 }
316 }
317}
318
319fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
320 var in: std.Io.Reader = .fixed(compressed);
321 var aw: std.Io.Writer.Allocating = .init(testing.allocator);
322 defer aw.deinit();
323
324 var decompress: Decompress = .init(&in, container, &.{});
325 _ = try decompress.reader.streamRemaining(&aw.writer);
326 try testing.expectEqualSlices(u8, expected_plain, aw.getWritten());
327}
328
329test "don't read past deflate stream's end" {
330 try testDecompress(.zlib, &[_]u8{
331 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
332 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
333 0x83, 0x95, 0x0b, 0xf5,
334 }, &[_]u8{
335 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
336 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
337 0x00, 0x00, 0xff, 0xff, 0xff,
338 });
339}
340
341test "zlib header" {
342 // Truncated header
343 try testing.expectError(
344 error.EndOfStream,
345 testDecompress(.zlib, &[_]u8{0x78}, ""),
346 );
347 // Wrong CM
348 try testing.expectError(
349 error.BadZlibHeader,
350 testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""),
351 );
352 // Wrong CINFO
353 try testing.expectError(
354 error.BadZlibHeader,
355 testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""),
356 );
357 // Wrong checksum
358 try testing.expectError(
359 error.WrongZlibChecksum,
360 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
361 );
362 // Truncated checksum
363 try testing.expectError(
364 error.EndOfStream,
365 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
366 );
367}
368
369test "gzip header" {
370 // Truncated header
371 try testing.expectError(
372 error.EndOfStream,
373 testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined),
374 );
375 // Wrong CM
376 try testing.expectError(
377 error.BadGzipHeader,
378 testDecompress(.gzip, &[_]u8{
379 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
380 0x00, 0x03,
381 }, undefined),
382 );
383
384 // Wrong checksum
385 try testing.expectError(
386 error.WrongGzipChecksum,
387 testDecompress(.gzip, &[_]u8{
388 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
389 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
390 0x00, 0x00, 0x00, 0x00,
391 }, undefined),
392 );
393 // Truncated checksum
394 try testing.expectError(
395 error.EndOfStream,
396 testDecompress(.gzip, &[_]u8{
397 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
398 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
399 }, undefined),
400 );
401 // Wrong initial size
402 try testing.expectError(
403 error.WrongGzipSize,
404 testDecompress(.gzip, &[_]u8{
405 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
406 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
407 0x00, 0x00, 0x00, 0x01,
408 }, undefined),
409 );
410 // Truncated initial size field
411 try testing.expectError(
412 error.EndOfStream,
413 testDecompress(.gzip, &[_]u8{
414 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
415 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
416 0x00, 0x00, 0x00,
417 }, undefined),
418 );
419
420 try testDecompress(.gzip, &[_]u8{
421 // GZIP header
422 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
423 // header.FHCRC (should cover entire header)
424 0x99, 0xd6,
425 // GZIP data
426 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
427 }, "");
428}
429
430test "public interface" {
431 const plain_data_buf = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };
432
433 // deflate final stored block, header + plain (stored) data
434 const deflate_block = [_]u8{
435 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
436 } ++ plain_data_buf;
437
438 const plain_data: []const u8 = &plain_data_buf;
439 const gzip_data: []const u8 = &deflate_block;
440
441 //// gzip header/footer + deflate block
442 //const gzip_data =
443 // [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
444 // deflate_block ++
445 // [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
446
447 //// zlib header/footer + deflate block
448 //const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
449 // deflate_block ++
450 // [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
451
452 // TODO
453 //const gzip = @import("gzip.zig");
454 //const zlib = @import("zlib.zig");
455
456 var buffer1: [64]u8 = undefined;
457 var buffer2: [64]u8 = undefined;
458
459 // decompress
460 {
461 var plain: Writer = .fixed(&buffer2);
462 var in: std.Io.Reader = .fixed(gzip_data);
463 var d: Decompress = .init(&in, .raw, &.{});
464 _ = try d.reader.streamRemaining(&plain);
465 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
466 }
467
468 // compress/decompress
469 {
470 var plain: Writer = .fixed(&buffer2);
471 var compressed: Writer = .fixed(&buffer1);
472
473 var cmp: Compress = .init(&compressed, &.{}, .{});
474 try cmp.writer.writeAll(plain_data);
475 try cmp.end();
476
477 var r: std.Io.Reader = .fixed(&buffer1);
478 var d: Decompress = .init(&r, .raw, &.{});
479 _ = try d.reader.streamRemaining(&plain);
480 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
481 }
482
483 // compressor/decompressor
484 {
485 var plain: Writer = .fixed(&buffer2);
486 var compressed: Writer = .fixed(&buffer1);
487
488 var cmp: Compress = .init(&compressed, &.{}, .{});
489 try cmp.writer.writeAll(plain_data);
490 try cmp.end();
491
492 var r: std.Io.Reader = .fixed(&buffer1);
493 var dcp = Decompress(&r);
494 try dcp.decompress(&plain);
495 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
496 }
497
498 // huffman
499 {
500 // huffman compress/decompress
501 {
502 var plain: Writer = .fixed(&buffer2);
503 var compressed: Writer = .fixed(&buffer1);
504
505 var in: std.Io.Reader = .fixed(plain_data);
506 try HuffmanEncoder.compress(&in, &compressed);
507
508 var r: std.Io.Reader = .fixed(&buffer1);
509 var d: Decompress = .init(&r, .raw, &.{});
510 _ = try d.reader.streamRemaining(&plain);
511 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
512 }
513
514 // huffman compressor/decompressor
515 {
516 var plain: Writer = .fixed(&buffer2);
517 var compressed: Writer = .fixed(&buffer1);
518
519 var in: std.Io.Reader = .fixed(plain_data);
520 var cmp = try HuffmanEncoder.Compressor(&compressed);
521 try cmp.compress(&in);
522 try cmp.finish();
523
524 var r: std.Io.Reader = .fixed(&buffer1);
525 var d: Decompress = .init(&r, .raw, &.{});
526 _ = try d.reader.streamRemaining(&plain);
527 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
528 }
529 }
530
531 // TODO
532 //{
533 // // store compress/decompress
534 // {
535 // var plain: Writer = .fixed(&buffer2);
536 // var compressed: Writer = .fixed(&buffer1);
537
538 // var in: std.Io.Reader = .fixed(plain_data);
539 // try store.compress(&in, &compressed);
540
541 // var r: std.Io.Reader = .fixed(&buffer1);
542 // var d: Decompress = .init(&r, .raw, &.{});
543 // _ = try d.reader.streamRemaining(&plain);
544 // try testing.expectEqualSlices(u8, plain_data, plain.buffered());
545 // }
546
547 // // store compressor/decompressor
548 // {
549 // var plain: Writer = .fixed(&buffer2);
550 // var compressed: Writer = .fixed(&buffer1);
551
552 // var in: std.Io.Reader = .fixed(plain_data);
553 // var cmp = try store.compressor(&compressed);
554 // try cmp.compress(&in);
555 // try cmp.finish();
556
557 // var r: std.Io.Reader = .fixed(&buffer1);
558 // var d: Decompress = .init(&r, .raw, &.{});
559 // _ = try d.reader.streamRemaining(&plain);
560 // try testing.expectEqualSlices(u8, plain_data, plain.buffered());
561 // }
562 //}
563}
564
565pub const match = struct {
566 pub const base_length = 3; // smallest match length per the RFC section 3.2.5
567 pub const min_length = 4; // min length used in this algorithm
568 pub const max_length = 258;
569
570 pub const min_distance = 1;
571 pub const max_distance = 32768;
572};
573
574pub const history_len = match.max_distance;
575
576pub const lookup = struct {
577 pub const bits = 15;
578 pub const len = 1 << bits;
579 pub const shift = 32 - bits;
580};
581
582test "zlib should not overshoot" {
583 // Compressed zlib data with extra 4 bytes at the end.
584 const data = [_]u8{
585 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
586 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
587 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
588 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
589 };
590
591 var reader: std.Io.Reader = .fixed(&data);
592
593 var decompress: Decompress = .init(&reader, .zlib, &.{});
594 var out: [128]u8 = undefined;
595
596 {
597 const n = try decompress.reader.readSliceShort(out[0..]);
598
599 // Expected decompressed data
600 try std.testing.expectEqual(46, n);
601 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
602
603 // Decompressor don't overshoot underlying reader.
604 // It is leaving it at the end of compressed data chunk.
605 try std.testing.expectEqual(data.len - 4, reader.seek);
606 // TODO what was this testing, exactly?
607 //try std.testing.expectEqual(0, decompress.unreadBytes());
608 }
609
610 // 4 bytes after compressed chunk are available in reader.
611 const n = try reader.readSliceShort(out[0..]);
612 try std.testing.expectEqual(n, 4);
613 try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
614}
615
616153test {
617154 _ = HuffmanEncoder;
618155 _ = Compress;
lib/std/compress/flate/BlockWriter.zig+20-1
......@@ -31,7 +31,26 @@ fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code,
3131fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
3232distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
3333
34pub fn init(bw: *BlockWriter) void {
34pub fn init(output: *Writer) BlockWriter {
35 return .{
36 .output = output,
37 .codegen_freq = undefined,
38 .literal_freq = undefined,
39 .distance_freq = undefined,
40 .codegen = undefined,
41 .literal_encoding = undefined,
42 .distance_encoding = undefined,
43 .codegen_encoding = undefined,
44 .fixed_literal_encoding = undefined,
45 .fixed_distance_encoding = undefined,
46 .huff_distance = undefined,
47 .fixed_literal_codes = undefined,
48 .fixed_distance_codes = undefined,
49 .distance_codes = undefined,
50 };
51}
52
53pub fn initBuffers(bw: *BlockWriter) void {
3554 bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes);
3655 bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes);
3756 bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes);
lib/std/compress/flate/Compress.zig+20-271
......@@ -122,22 +122,7 @@ pub const Options = struct {
122122
123123pub fn init(output: *Writer, buffer: []u8, options: Options) Compress {
124124 return .{
125 .block_writer = .{
126 .output = output,
127 .codegen_freq = undefined,
128 .literal_freq = undefined,
129 .distance_freq = undefined,
130 .codegen = undefined,
131 .literal_encoding = undefined,
132 .distance_encoding = undefined,
133 .codegen_encoding = undefined,
134 .fixed_literal_encoding = undefined,
135 .fixed_distance_encoding = undefined,
136 .huff_distance = undefined,
137 .fixed_literal_codes = undefined,
138 .fixed_distance_codes = undefined,
139 .distance_codes = undefined,
140 },
125 .block_writer = .init(output),
141126 .level = .get(options.level),
142127 .hasher = .init(options.container),
143128 .state = .header,
......@@ -188,20 +173,21 @@ fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize
188173 }
189174
190175 const buffered = me.buffered();
191 const min_lookahead = flate.match.min_length + flate.match.max_length;
176 const min_lookahead = Token.min_length + Token.max_length;
192177 const history_plus_lookahead_len = flate.history_len + min_lookahead;
193178 if (buffered.len < history_plus_lookahead_len) return 0;
194179 const lookahead = buffered[flate.history_len..];
195180
196 _ = lookahead;
197181 // TODO tokenize
182 _ = lookahead;
198183 //c.hasher.update(lookahead[0..n]);
199184 @panic("TODO");
200185}
201186
202187pub fn end(c: *Compress) !void {
203188 try endUnflushed(c);
204 try c.output.flush();
189 const out = c.block_writer.output;
190 try out.flush();
205191}
206192
207193pub fn endUnflushed(c: *Compress) !void {
......@@ -227,7 +213,7 @@ pub fn endUnflushed(c: *Compress) !void {
227213 // Checksum value of the uncompressed data (excluding any
228214 // dictionary data) computed according to Adler-32
229215 // algorithm.
230 std.mem.writeInt(u32, try out.writableArray(4), zlib.final, .big);
216 std.mem.writeInt(u32, try out.writableArray(4), zlib.adler, .big);
231217 },
232218 .raw => {},
233219 }
......@@ -243,15 +229,16 @@ pub const Simple = struct {
243229
244230 pub const Strategy = enum { huffman, store };
245231
246 pub fn init(out: *Writer, buffer: []u8, container: Container) !Simple {
247 const self: Simple = .{
232 pub fn init(output: *Writer, buffer: []u8, container: Container, strategy: Strategy) !Simple {
233 const header = container.header();
234 try output.writeAll(header);
235 return .{
248236 .buffer = buffer,
249237 .wp = 0,
250 .block_writer = .init(out),
238 .block_writer = .init(output),
251239 .hasher = .init(container),
240 .strategy = strategy,
252241 };
253 try container.writeHeader(self.out);
254 return self;
255242 }
256243
257244 pub fn flush(self: *Simple) !void {
......@@ -263,7 +250,7 @@ pub const Simple = struct {
263250 pub fn finish(self: *Simple) !void {
264251 try self.flushBuffer(true);
265252 try self.block_writer.flush();
266 try self.hasher.container().writeFooter(&self.hasher, self.out);
253 try self.hasher.container().writeFooter(&self.hasher, self.block_writer.output);
267254 }
268255
269256 fn flushBuffer(self: *Simple, final: bool) !void {
......@@ -300,7 +287,13 @@ test "generate a Huffman code from an array of frequencies" {
300287 };
301288
302289 var codes: [19]HuffmanEncoder.Code = undefined;
303 var enc: HuffmanEncoder = .{ .codes = &codes };
290 var enc: HuffmanEncoder = .{
291 .codes = &codes,
292 .freq_cache = undefined,
293 .bit_count = undefined,
294 .lns = undefined,
295 .lfs = undefined,
296 };
304297 enc.generate(freqs[0..], 7);
305298
306299 try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
......@@ -337,247 +330,3 @@ test "generate a Huffman code from an array of frequencies" {
337330 try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
338331 try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
339332}
340
341test "tokenization" {
342 const L = Token.initLiteral;
343 const M = Token.initMatch;
344
345 const cases = [_]struct {
346 data: []const u8,
347 tokens: []const Token,
348 }{
349 .{
350 .data = "Blah blah blah blah blah!",
351 .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
352 },
353 .{
354 .data = "ABCDEABCD ABCDEABCD",
355 .tokens = &[_]Token{
356 L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
357 L('A'), M(10, 8),
358 },
359 },
360 };
361
362 for (cases) |c| {
363 inline for (Container.list) |container| { // for each wrapping
364
365 var cw = std.Io.countingWriter(std.Io.null_writer);
366 const cww = cw.writer();
367 var df = try Compress(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
368
369 _ = try df.write(c.data);
370 try df.flush();
371
372 // df.token_writer.show();
373 try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
374 try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
375
376 try testing.expectEqual(container.headerSize(), cw.bytes_written);
377 try df.finish();
378 try testing.expectEqual(container.size(), cw.bytes_written);
379 }
380 }
381}
382
383// Tests that tokens written are equal to expected token list.
384const TestTokenWriter = struct {
385 const Self = @This();
386
387 pos: usize = 0,
388 actual: [128]Token = undefined,
389
390 pub fn init(_: anytype) Self {
391 return .{};
392 }
393 pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
394 for (tokens) |t| {
395 self.actual[self.pos] = t;
396 self.pos += 1;
397 }
398 }
399
400 pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
401
402 pub fn get(self: *Self) []Token {
403 return self.actual[0..self.pos];
404 }
405
406 pub fn show(self: *Self) void {
407 std.debug.print("\n", .{});
408 for (self.get()) |t| {
409 t.show();
410 }
411 }
412
413 pub fn flush(_: *Self) !void {}
414};
415
416test "file tokenization" {
417 const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
418 const cases = [_]struct {
419 data: []const u8, // uncompressed content
420 // expected number of tokens producet in deflate tokenization
421 tokens_count: [levels.len]usize = .{0} ** levels.len,
422 }{
423 .{
424 .data = @embedFile("testdata/rfc1951.txt"),
425 .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
426 },
427
428 .{
429 .data = @embedFile("testdata/block_writer/huffman-null-max.input"),
430 .tokens_count = .{ 257, 257, 257, 257, 257, 257 },
431 },
432 .{
433 .data = @embedFile("testdata/block_writer/huffman-pi.input"),
434 .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
435 },
436 .{
437 .data = @embedFile("testdata/block_writer/huffman-text.input"),
438 .tokens_count = .{ 235, 234, 234, 234, 234, 234 },
439 },
440 .{
441 .data = @embedFile("testdata/fuzz/roundtrip1.input"),
442 .tokens_count = .{ 333, 331, 331, 331, 331, 331 },
443 },
444 .{
445 .data = @embedFile("testdata/fuzz/roundtrip2.input"),
446 .tokens_count = .{ 334, 334, 334, 334, 334, 334 },
447 },
448 };
449
450 for (cases) |case| { // for each case
451 const data = case.data;
452
453 for (levels, 0..) |level, i| { // for each compression level
454 var original: std.Io.Reader = .fixed(data);
455
456 // buffer for decompressed data
457 var al = std.ArrayList(u8).init(testing.allocator);
458 defer al.deinit();
459 const writer = al.writer();
460
461 // create compressor
462 const WriterType = @TypeOf(writer);
463 const TokenWriter = TokenDecoder(@TypeOf(writer));
464 var cmp = try Compress(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
465
466 // Stream uncompressed `original` data to the compressor. It will
467 // produce tokens list and pass that list to the TokenDecoder. This
468 // TokenDecoder uses CircularBuffer from inflate to convert list of
469 // tokens back to the uncompressed stream.
470 try cmp.compress(original.reader());
471 try cmp.flush();
472 const expected_count = case.tokens_count[i];
473 const actual = cmp.block_writer.tokens_count;
474 if (expected_count == 0) {
475 std.debug.print("actual token count {d}\n", .{actual});
476 } else {
477 try testing.expectEqual(expected_count, actual);
478 }
479
480 try testing.expectEqual(data.len, al.items.len);
481 try testing.expectEqualSlices(u8, data, al.items);
482 }
483 }
484}
485
486const TokenDecoder = struct {
487 output: *Writer,
488 tokens_count: usize,
489
490 pub fn init(output: *Writer) TokenDecoder {
491 return .{
492 .output = output,
493 .tokens_count = 0,
494 };
495 }
496
497 pub fn write(self: *TokenDecoder, tokens: []const Token, _: bool, _: ?[]const u8) !void {
498 self.tokens_count += tokens.len;
499 for (tokens) |t| {
500 switch (t.kind) {
501 .literal => self.hist.write(t.literal()),
502 .match => try self.hist.writeMatch(t.length(), t.distance()),
503 }
504 if (self.hist.free() < 285) try self.flushWin();
505 }
506 try self.flushWin();
507 }
508
509 fn flushWin(self: *TokenDecoder) !void {
510 while (true) {
511 const buf = self.hist.read();
512 if (buf.len == 0) break;
513 try self.output.writeAll(buf);
514 }
515 }
516};
517
518test "store simple compressor" {
519 if (true) return error.SkipZigTest;
520 //const data = "Hello world!";
521 //const expected = [_]u8{
522 // 0x1, // block type 0, final bit set
523 // 0xc, 0x0, // len = 12
524 // 0xf3, 0xff, // ~len
525 // 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
526 // //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
527 //};
528
529 //var fbs: std.Io.Reader = .fixed(data);
530 //var al = std.ArrayList(u8).init(testing.allocator);
531 //defer al.deinit();
532
533 //var cmp = try store.compressor(.raw, al.writer());
534 //try cmp.compress(&fbs);
535 //try cmp.finish();
536 //try testing.expectEqualSlices(u8, &expected, al.items);
537
538 //fbs = .fixed(data);
539 //try al.resize(0);
540
541 //// huffman only compresoor will also emit store block for this small sample
542 //var hc = try huffman.compressor(.raw, al.writer());
543 //try hc.compress(&fbs);
544 //try hc.finish();
545 //try testing.expectEqualSlices(u8, &expected, al.items);
546}
547
548test "sliding window match" {
549 const data = "Blah blah blah blah blah!";
550 var win: Writer = .{};
551 try expect(win.write(data) == data.len);
552 try expect(win.wp == data.len);
553 try expect(win.rp == 0);
554
555 // length between l symbols
556 try expect(win.match(1, 6, 0) == 18);
557 try expect(win.match(1, 11, 0) == 13);
558 try expect(win.match(1, 16, 0) == 8);
559 try expect(win.match(1, 21, 0) == 0);
560
561 // position 15 = "blah blah!"
562 // position 20 = "blah!"
563 try expect(win.match(15, 20, 0) == 4);
564 try expect(win.match(15, 20, 3) == 4);
565 try expect(win.match(15, 20, 4) == 0);
566}
567
568test "sliding window slide" {
569 var win: Writer = .{};
570 win.wp = Writer.buffer_len - 11;
571 win.rp = Writer.buffer_len - 111;
572 win.buffer[win.rp] = 0xab;
573 try expect(win.lookahead().len == 100);
574 try expect(win.tokensBuffer().?.len == win.rp);
575
576 const n = win.slide();
577 try expect(n == 32757);
578 try expect(win.buffer[win.rp] == 0xab);
579 try expect(win.rp == Writer.hist_len - 111);
580 try expect(win.wp == Writer.hist_len - 11);
581 try expect(win.lookahead().len == 100);
582 try expect(win.tokensBuffer() == null);
583}
lib/std/compress/flate/Decompress.zig+184-33
......@@ -4,8 +4,8 @@ const Container = flate.Container;
44const Token = @import("Token.zig");
55const testing = std.testing;
66const Decompress = @This();
7const Writer = std.io.Writer;
8const Reader = std.io.Reader;
7const Writer = std.Io.Writer;
8const Reader = std.Io.Reader;
99
1010input: *Reader,
1111reader: Reader,
......@@ -129,7 +129,7 @@ fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
129129 return sym;
130130}
131131
132pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize {
132pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
133133 const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
134134 return readInner(d, w, limit) catch |err| switch (err) {
135135 error.EndOfStream => return error.EndOfStream,
......@@ -143,7 +143,8 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!us
143143 };
144144}
145145
146fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.StreamError)!usize {
146fn readInner(d: *Decompress, w: *Writer, limit: std.Io.Limit) (Error || Reader.StreamError)!usize {
147 var remaining = @intFromEnum(limit);
147148 const in = d.input;
148149 sw: switch (d.state) {
149150 .protocol_header => switch (d.hasher.container()) {
......@@ -182,15 +183,9 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
182183 continue :sw .block_header;
183184 },
184185 .zlib => {
185 const Header = extern struct {
186 cmf: packed struct(u8) {
187 cm: u4,
188 cinfo: u4,
189 },
190 flg: u8,
191 };
192 const header = try in.takeStruct(Header);
193 if (header.cmf.cm != 8 or header.cmf.cinfo > 7) return error.BadZlibHeader;
186 const header = try in.takeArray(2);
187 const cmf: packed struct(u8) { cm: u4, cinfo: u4 } = @bitCast(header[0]);
188 if (cmf.cm != 8 or cmf.cinfo > 7) return error.BadZlibHeader;
194189 continue :sw .block_header;
195190 },
196191 .raw => continue :sw .block_header,
......@@ -219,7 +214,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
219214 // lengths for code lengths
220215 var cl_lens = [_]u4{0} ** 19;
221216 for (0..hclen) |i| {
222 cl_lens[flate.huffman.codegen_order[i]] = try d.takeBits(u3);
217 cl_lens[flate.HuffmanEncoder.codegen_order[i]] = try d.takeBits(u3);
223218 }
224219 var cl_dec: CodegenDecoder = .{};
225220 try cl_dec.generate(&cl_lens);
......@@ -259,52 +254,56 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
259254 return n;
260255 },
261256 .fixed_block => {
262 const start = w.count;
263 while (@intFromEnum(limit) > w.count - start) {
257 while (remaining > 0) {
264258 const code = try d.readFixedCode();
265259 switch (code) {
266 0...255 => try w.writeBytePreserve(flate.history_len, @intCast(code)),
260 0...255 => {
261 try w.writeBytePreserve(flate.history_len, @intCast(code));
262 remaining -= 1;
263 },
267264 256 => {
268265 d.state = if (d.final_block) .protocol_footer else .block_header;
269 return w.count - start;
266 return @intFromEnum(limit) - remaining;
270267 },
271268 257...285 => {
272269 // Handles fixed block non literal (length) code.
273270 // Length code is followed by 5 bits of distance code.
274271 const length = try d.decodeLength(@intCast(code - 257));
275272 const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5));
276 try writeMatch(w, length, distance);
273 remaining = try writeMatch(w, length, distance, remaining);
277274 },
278275 else => return error.InvalidCode,
279276 }
280277 }
281278 d.state = .fixed_block;
282 return w.count - start;
279 return @intFromEnum(limit) - remaining;
283280 },
284281 .dynamic_block => {
285 // In larger archives most blocks are usually dynamic, so decompression
286 // performance depends on this logic.
287 const start = w.count;
288 while (@intFromEnum(limit) > w.count - start) {
282 // In larger archives most blocks are usually dynamic, so
283 // decompression performance depends on this logic.
284 while (remaining > 0) {
289285 const sym = try d.decodeSymbol(&d.lit_dec);
290286
291287 switch (sym.kind) {
292 .literal => try w.writeBytePreserve(flate.history_len, sym.symbol),
288 .literal => {
289 try w.writeBytePreserve(flate.history_len, sym.symbol);
290 remaining -= 1;
291 },
293292 .match => {
294293 // Decode match backreference <length, distance>
295294 const length = try d.decodeLength(sym.symbol);
296295 const dsm = try d.decodeSymbol(&d.dst_dec);
297296 const distance = try d.decodeDistance(dsm.symbol);
298 try writeMatch(w, length, distance);
297 remaining = try writeMatch(w, length, distance, remaining);
299298 },
300299 .end_of_block => {
301300 d.state = if (d.final_block) .protocol_footer else .block_header;
302 return w.count - start;
301 return @intFromEnum(limit) - remaining;
303302 },
304303 }
305304 }
306305 d.state = .dynamic_block;
307 return w.count - start;
306 return @intFromEnum(limit) - remaining;
308307 },
309308 .protocol_footer => {
310309 d.alignBitsToByte();
......@@ -314,7 +313,7 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
314313 if (try in.takeInt(u32, .little) != gzip.count) return error.WrongGzipSize;
315314 },
316315 .zlib => |*zlib| {
317 const chksum: u32 = @byteSwap(zlib.final());
316 const chksum: u32 = @byteSwap(zlib.adler);
318317 if (try in.takeInt(u32, .big) != chksum) return error.WrongZlibChecksum;
319318 },
320319 .raw => {},
......@@ -328,10 +327,11 @@ fn readInner(d: *Decompress, w: *Writer, limit: std.io.Limit) (Error || Reader.S
328327
329328/// Write match (back-reference to the same data slice) starting at `distance`
330329/// back from current write position, and `length` of bytes.
331fn writeMatch(bw: *Writer, length: u16, distance: u16) !void {
332 _ = bw;
330fn writeMatch(w: *Writer, length: u16, distance: u16, remaining: usize) !usize {
331 _ = w;
333332 _ = length;
334333 _ = distance;
334 _ = remaining;
335335 @panic("TODO");
336336}
337337
......@@ -622,7 +622,13 @@ test "init/find" {
622622test "encode/decode literals" {
623623 var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined;
624624 for (1..286) |j| { // for all different number of codes
625 var enc: flate.HuffmanEncoder = .{ .codes = &codes };
625 var enc: flate.HuffmanEncoder = .{
626 .codes = &codes,
627 .freq_cache = undefined,
628 .bit_count = undefined,
629 .lns = undefined,
630 .lfs = undefined,
631 };
626632 // create frequencies
627633 var freq = [_]u16{0} ** 286;
628634 freq[256] = 1; // ensure we have end of block code
......@@ -857,7 +863,7 @@ test "fuzzing tests" {
857863 const r = &decompress.reader;
858864 if (c.err) |expected_err| {
859865 try testing.expectError(error.ReadFailed, r.streamRemaining(&aw.writer));
860 try testing.expectError(expected_err, decompress.read_err.?);
866 try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed);
861867 } else {
862868 _ = try r.streamRemaining(&aw.writer);
863869 try testing.expectEqualStrings(c.out, aw.getWritten());
......@@ -891,3 +897,148 @@ test "reading into empty buffer" {
891897 var buf: [0]u8 = undefined;
892898 try testing.expectEqual(0, try r.readVec(&.{&buf}));
893899}
900
901test "don't read past deflate stream's end" {
902 try testDecompress(.zlib, &[_]u8{
903 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
904 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
905 0x83, 0x95, 0x0b, 0xf5,
906 }, &[_]u8{
907 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
908 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
909 0x00, 0x00, 0xff, 0xff, 0xff,
910 });
911}
912
913test "zlib header" {
914 // Truncated header
915 try testing.expectError(
916 error.EndOfStream,
917 testDecompress(.zlib, &[_]u8{0x78}, ""),
918 );
919 // Wrong CM
920 try testing.expectError(
921 error.BadZlibHeader,
922 testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""),
923 );
924 // Wrong CINFO
925 try testing.expectError(
926 error.BadZlibHeader,
927 testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""),
928 );
929 // Wrong checksum
930 try testing.expectError(
931 error.WrongZlibChecksum,
932 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
933 );
934 // Truncated checksum
935 try testing.expectError(
936 error.EndOfStream,
937 testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
938 );
939}
940
941test "gzip header" {
942 // Truncated header
943 try testing.expectError(
944 error.EndOfStream,
945 testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined),
946 );
947 // Wrong CM
948 try testing.expectError(
949 error.BadGzipHeader,
950 testDecompress(.gzip, &[_]u8{
951 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
952 0x00, 0x03,
953 }, undefined),
954 );
955
956 // Wrong checksum
957 try testing.expectError(
958 error.WrongGzipChecksum,
959 testDecompress(.gzip, &[_]u8{
960 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
961 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
962 0x00, 0x00, 0x00, 0x00,
963 }, undefined),
964 );
965 // Truncated checksum
966 try testing.expectError(
967 error.EndOfStream,
968 testDecompress(.gzip, &[_]u8{
969 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
970 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
971 }, undefined),
972 );
973 // Wrong initial size
974 try testing.expectError(
975 error.WrongGzipSize,
976 testDecompress(.gzip, &[_]u8{
977 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
978 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
979 0x00, 0x00, 0x00, 0x01,
980 }, undefined),
981 );
982 // Truncated initial size field
983 try testing.expectError(
984 error.EndOfStream,
985 testDecompress(.gzip, &[_]u8{
986 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
987 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
988 0x00, 0x00, 0x00,
989 }, undefined),
990 );
991
992 try testDecompress(.gzip, &[_]u8{
993 // GZIP header
994 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
995 // header.FHCRC (should cover entire header)
996 0x99, 0xd6,
997 // GZIP data
998 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
999 }, "");
1000}
1001
1002fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
1003 var in: std.Io.Reader = .fixed(compressed);
1004 var aw: std.Io.Writer.Allocating = .init(testing.allocator);
1005 defer aw.deinit();
1006
1007 var decompress: Decompress = .init(&in, container, &.{});
1008 _ = try decompress.reader.streamRemaining(&aw.writer);
1009 try testing.expectEqualSlices(u8, expected_plain, aw.getWritten());
1010}
1011
1012test "zlib should not overshoot" {
1013 // Compressed zlib data with extra 4 bytes at the end.
1014 const data = [_]u8{
1015 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
1016 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
1017 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
1018 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
1019 };
1020
1021 var reader: std.Io.Reader = .fixed(&data);
1022
1023 var decompress: Decompress = .init(&reader, .zlib, &.{});
1024 var out: [128]u8 = undefined;
1025
1026 {
1027 const n = try decompress.reader.readSliceShort(out[0..]);
1028
1029 // Expected decompressed data
1030 try std.testing.expectEqual(46, n);
1031 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
1032
1033 // Decompressor don't overshoot underlying reader.
1034 // It is leaving it at the end of compressed data chunk.
1035 try std.testing.expectEqual(data.len - 4, reader.seek);
1036 // TODO what was this testing, exactly?
1037 //try std.testing.expectEqual(0, decompress.unreadBytes());
1038 }
1039
1040 // 4 bytes after compressed chunk are available in reader.
1041 const n = try reader.readSliceShort(out[0..]);
1042 try std.testing.expectEqual(n, 4);
1043 try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
1044}
lib/std/compress/flate/HuffmanEncoder.zig+3-2
......@@ -135,7 +135,7 @@ fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize)
135135 // of ancestors of the rightmost node at level i.
136136 // leaf_counts[i][j] is the number of literals at the left
137137 // of the level j ancestor.
138 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(0);
138 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(@splat(0));
139139
140140 {
141141 var level = @as(u32, 1);
......@@ -389,7 +389,8 @@ pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder
389389}
390390
391391test "generate a Huffman code for the fixed literal table specific to Deflate" {
392 const enc = fixedLiteralEncoder();
392 var codes: [max_num_frequencies]Code = undefined;
393 const enc: HuffmanEncoder = .fixedLiteralEncoder(&codes);
393394 for (enc.codes) |c| {
394395 switch (c.len) {
395396 7 => {
lib/std/compress/flate/Lookup.zig+10-5
......@@ -6,14 +6,19 @@ const std = @import("std");
66const testing = std.testing;
77const expect = testing.expect;
88const flate = @import("../flate.zig");
9const Token = @import("Token.zig");
910
1011const Lookup = @This();
1112
1213const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
1314const chain_len = 2 * flate.history_len;
1415
16pub const bits = 15;
17pub const len = 1 << bits;
18pub const shift = 32 - bits;
19
1520// Maps hash => first position
16head: [flate.lookup.len]u16 = [_]u16{0} ** flate.lookup.len,
21head: [len]u16 = [_]u16{0} ** len,
1722// Maps position => previous positions for the same hash value
1823chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
1924
......@@ -52,8 +57,8 @@ pub fn slide(self: *Lookup, n: u16) void {
5257
5358// Add `len` 4 bytes hashes from `data` into lookup.
5459// Position of the first byte is `pos`.
55pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void {
56 if (len == 0 or data.len < flate.match.min_length) {
60pub fn bulkAdd(self: *Lookup, data: []const u8, length: u16, pos: u16) void {
61 if (length == 0 or data.len < Token.min_length) {
5762 return;
5863 }
5964 var hb =
......@@ -64,7 +69,7 @@ pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void {
6469 _ = self.set(hashu(hb), pos);
6570
6671 var i = pos;
67 for (4..@min(len + 3, data.len)) |j| {
72 for (4..@min(length + 3, data.len)) |j| {
6873 hb = (hb << 8) | @as(u32, data[j]);
6974 i += 1;
7075 _ = self.set(hashu(hb), i);
......@@ -80,7 +85,7 @@ fn hash(b: *const [4]u8) u32 {
8085}
8186
8287fn hashu(v: u32) u32 {
83 return @intCast((v *% prime4) >> flate.lookup.shift);
88 return @intCast((v *% prime4) >> shift);
8489}
8590
8691test add {
lib/std/compress/flate/Token.zig+13-7
......@@ -6,7 +6,6 @@ const std = @import("std");
66const assert = std.debug.assert;
77const print = std.debug.print;
88const expect = std.testing.expect;
9const match = std.compress.flate.match;
109
1110const Token = @This();
1211
......@@ -21,16 +20,23 @@ dist: u15 = 0,
2120len_lit: u8 = 0,
2221kind: Kind = .literal,
2322
23pub const base_length = 3; // smallest match length per the RFC section 3.2.5
24pub const min_length = 4; // min length used in this algorithm
25pub const max_length = 258;
26
27pub const min_distance = 1;
28pub const max_distance = std.compress.flate.history_len;
29
2430pub fn literal(t: Token) u8 {
2531 return t.len_lit;
2632}
2733
2834pub fn distance(t: Token) u16 {
29 return @as(u16, t.dist) + match.min_distance;
35 return @as(u16, t.dist) + min_distance;
3036}
3137
3238pub fn length(t: Token) u16 {
33 return @as(u16, t.len_lit) + match.base_length;
39 return @as(u16, t.len_lit) + base_length;
3440}
3541
3642pub fn initLiteral(lit: u8) Token {
......@@ -40,12 +46,12 @@ pub fn initLiteral(lit: u8) Token {
4046// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
4147// length range 3 - 258, stored in len_lit as 0 - 255 (u8)
4248pub fn initMatch(dist: u16, len: u16) Token {
43 assert(len >= match.min_length and len <= match.max_length);
44 assert(dist >= match.min_distance and dist <= match.max_distance);
49 assert(len >= min_length and len <= max_length);
50 assert(dist >= min_distance and dist <= max_distance);
4551 return .{
4652 .kind = .match,
47 .dist = @intCast(dist - match.min_distance),
48 .len_lit = @intCast(len - match.base_length),
53 .dist = @intCast(dist - min_distance),
54 .len_lit = @intCast(len - base_length),
4955 };
5056}
5157