authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-05-18 16:28:01-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:29-07:00
logfd4fb10722beea90ee3a1c9314a10eec36a44346
tree100717a136ff7ec07cec3141c9f4cfc30522e17f
parent06b44a0afa849556f613fcc5939db99fec38883f

std.compress.flate: API reorg and reader/writer updates

A lot of this logic disappears in the face of the new buffered reader and buffered writer interface. This is passing ast-check only; semantic analysis to be solved next.

18 files changed, 2599 insertions(+), 3694 deletions(-)

lib/std/compress.zig+1-4
......@@ -1,19 +1,16 @@
11//! Compression algorithms.
22
3/// gzip and zlib are here.
34pub const flate = @import("compress/flate.zig");
4pub const gzip = @import("compress/gzip.zig");
55pub const lzma = @import("compress/lzma.zig");
66pub const lzma2 = @import("compress/lzma2.zig");
77pub const xz = @import("compress/xz.zig");
8pub const zlib = @import("compress/zlib.zig");
98pub const zstd = @import("compress/zstd.zig");
109
1110test {
1211 _ = flate;
13 _ = gzip;
1412 _ = lzma;
1513 _ = lzma2;
1614 _ = xz;
17 _ = zlib;
1815 _ = zstd;
1916}
lib/std/compress/flate.zig+404-127
......@@ -1,4 +1,196 @@
1const builtin = @import("builtin");
12const std = @import("../std.zig");
3const testing = std.testing;
4
5/// Container of the deflate bit stream body. Container adds header before
6/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
7/// no footer, raw bit stream).
8///
9/// Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
10/// addler 32 checksum.
11///
12/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
13/// crc32 checksum and 4 bytes of uncompressed data length.
14///
15///
16/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
17/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
18pub const Container = enum {
19 raw, // no header or footer
20 gzip, // gzip header and footer
21 zlib, // zlib header and footer
22
23 pub fn size(w: Container) usize {
24 return headerSize(w) + footerSize(w);
25 }
26
27 pub fn headerSize(w: Container) usize {
28 return header(w).len;
29 }
30
31 pub fn footerSize(w: Container) usize {
32 return switch (w) {
33 .gzip => 8,
34 .zlib => 4,
35 .raw => 0,
36 };
37 }
38
39 pub const list = [_]Container{ .raw, .gzip, .zlib };
40
41 pub const Error = error{
42 BadGzipHeader,
43 BadZlibHeader,
44 WrongGzipChecksum,
45 WrongGzipSize,
46 WrongZlibChecksum,
47 };
48
49 pub fn header(container: Container) []const u8 {
50 return switch (container) {
51 // GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
52 // - ID1 (IDentification 1), always 0x1f
53 // - ID2 (IDentification 2), always 0x8b
54 // - CM (Compression Method), always 8 = deflate
55 // - FLG (Flags), all set to 0
56 // - 4 bytes, MTIME (Modification time), not used, all set to zero
57 // - XFL (eXtra FLags), all set to zero
58 // - OS (Operating System), 03 = Unix
59 .gzip => &[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 },
60 // ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
61 // 1st byte:
62 // - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
63 // - The next four bits is the CM (compression method), which is 8 for deflate.
64 // 2nd byte:
65 // - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
66 // - The next bit, FDICT, is set if a dictionary is given.
67 // - The final five FCHECK bits form a mod-31 checksum.
68 //
69 // CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
70 .zlib => &[_]u8{ 0x78, 0b10_0_11100 },
71 .raw => &{},
72 };
73 }
74
75 pub fn parseHeader(comptime wrap: Container, reader: *std.io.BufferedReader) !void {
76 switch (wrap) {
77 .gzip => try parseGzipHeader(reader),
78 .zlib => try parseZlibHeader(reader),
79 .raw => {},
80 }
81 }
82
83 fn parseGzipHeader(reader: *std.io.BufferedReader) !void {
84 const magic1 = try reader.read(u8);
85 const magic2 = try reader.read(u8);
86 const method = try reader.read(u8);
87 const flags = try reader.read(u8);
88 try reader.skipBytes(6); // mtime(4), xflags, os
89 if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
90 return error.BadGzipHeader;
91 // Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
92 if (flags != 0) {
93 if (flags & 0b0000_0100 != 0) { // FEXTRA
94 const extra_len = try reader.read(u16);
95 try reader.skipBytes(extra_len);
96 }
97 if (flags & 0b0000_1000 != 0) { // FNAME
98 try reader.skipStringZ();
99 }
100 if (flags & 0b0001_0000 != 0) { // FCOMMENT
101 try reader.skipStringZ();
102 }
103 if (flags & 0b0000_0010 != 0) { // FHCRC
104 try reader.skipBytes(2);
105 }
106 }
107 }
108
109 fn parseZlibHeader(reader: *std.io.BufferedReader) !void {
110 const cm = try reader.read(u4);
111 const cinfo = try reader.read(u4);
112 _ = try reader.read(u8);
113 if (cm != 8 or cinfo > 7) {
114 return error.BadZlibHeader;
115 }
116 }
117
118 pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: *std.io.BufferedReader) !void {
119 switch (wrap) {
120 .gzip => {
121 try reader.fill(0);
122 if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
123 if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
124 },
125 .zlib => {
126 const chksum: u32 = @byteSwap(hasher.chksum());
127 if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
128 },
129 .raw => {},
130 }
131 }
132
133 pub const Hasher = union(Container) {
134 gzip: struct {
135 crc: std.hash.Crc32 = .init(),
136 count: usize = 0,
137 },
138 zlib: std.hash.Adler32,
139 raw: void,
140
141 pub fn init(containter: Container) Hasher {
142 return switch (containter) {
143 .gzip => .{ .gzip = .{} },
144 .zlib => .{ .zlib = .init() },
145 .raw => {},
146 };
147 }
148
149 pub fn container(h: Hasher) Container {
150 return h;
151 }
152
153 pub fn update(h: *Hasher, buf: []const u8) void {
154 switch (h.*) {
155 .raw => {},
156 .gzip => |*gzip| {
157 gzip.update(buf);
158 gzip.count += buf.len;
159 },
160 .zlib => |*zlib| {
161 zlib.update(buf);
162 },
163 inline .gzip, .zlib => |*x| x.update(buf),
164 }
165 }
166
167 pub fn writeFooter(hasher: *Hasher, writer: *std.io.BufferedWriter) std.io.Writer.Error!void {
168 var bits: [4]u8 = undefined;
169 switch (hasher.*) {
170 .gzip => |*gzip| {
171 // GZIP 8 bytes footer
172 // - 4 bytes, CRC32 (CRC-32)
173 // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
174 std.mem.writeInt(u32, &bits, gzip.final(), .little);
175 try writer.writeAll(&bits);
176
177 std.mem.writeInt(u32, &bits, gzip.bytes_read, .little);
178 try writer.writeAll(&bits);
179 },
180 .zlib => |*zlib| {
181 // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
182 // 4 bytes of ADLER32 (Adler-32 checksum)
183 // Checksum value of the uncompressed data (excluding any
184 // dictionary data) computed according to Adler-32
185 // algorithm.
186 std.mem.writeInt(u32, &bits, zlib.final, .big);
187 try writer.writeAll(&bits);
188 },
189 .raw => {},
190 }
191 }
192 };
193};
2194
3195/// When decompressing, the output buffer is used as the history window, so
4196/// less than this may result in failure to decompress streams that were
......@@ -7,66 +199,48 @@ pub const max_window_len = 1 << 16;
7199
8200/// Deflate is a lossless data compression file format that uses a combination
9201/// of LZ77 and Huffman coding.
10pub const deflate = @import("flate/deflate.zig");
202pub const Compress = @import("flate/Compress.zig");
11203
12204/// Inflate is the decoding process that takes a Deflate bitstream for
13205/// decompression and correctly produces the original full-size data or file.
14pub const inflate = @import("flate/inflate.zig");
15
16/// Decompress compressed data from reader and write plain data to the writer.
17pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) std.io.Writer.Error!void {
18 try inflate.decompress(.raw, reader, writer);
19}
20
21pub const Decompressor = inflate.Decompressor(.raw);
22
23/// Compression level, trades between speed and compression size.
24pub const Options = deflate.Options;
25
26/// Compress plain data from reader and write compressed data to the writer.
27pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter, options: Options) std.io.Writer.Error!void {
28 try deflate.compress(.raw, reader, writer, options);
29}
30
31pub const Compressor = deflate.Compressor(.raw);
206pub const Decompress = @import("flate/Decompress.zig");
32207
33208/// Huffman only compression. Without Lempel-Ziv match searching. Faster
34209/// compression, less memory requirements but bigger compressed sizes.
35210pub const huffman = struct {
36 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) std.io.Writer.Error!void {
37 try deflate.huffman.compress(.raw, reader, writer);
38 }
39
40 pub const Compressor = deflate.huffman.Compressor(.raw);
211 // The odd order in which the codegen code sizes are written.
212 pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
213 // The number of codegen codes.
214 pub const codegen_code_count = 19;
215
216 // The largest distance code.
217 pub const distance_code_count = 30;
218
219 // Maximum number of literals.
220 pub const max_num_lit = 286;
221
222 // Max number of frequencies used for a Huffman Code
223 // Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
224 // The largest of these is max_num_lit.
225 pub const max_num_frequencies = max_num_lit;
226
227 // Biggest block size for uncompressed block.
228 pub const max_store_block_size = 65535;
229 // The special code used to mark the end of a block.
230 pub const end_block_marker = 256;
41231};
42232
43// No compression store only. Compressed size is slightly bigger than plain.
44pub const store = struct {
45 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) std.io.Writer.Error!void {
46 try deflate.store.compress(.raw, reader, writer);
47 }
48
49 pub const Compressor = deflate.store.Compressor(.raw);
50};
51
52const builtin = @import("builtin");
53const testing = std.testing;
54const fixedBufferStream = std.io.fixedBufferStream;
55const print = std.debug.print;
56/// Container defines header/footer around deflate bit stream. Gzip and zlib
57/// compression algorithms are containers around deflate bit stream body.
58const Container = @import("flate/container.zig").Container;
59
60233test {
61 _ = deflate;
62 _ = inflate;
234 _ = Compress;
235 _ = Decompress;
63236}
64237
65238test "compress/decompress" {
239 const print = std.debug.print;
66240 var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer
67241 var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer
68242
69 const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
243 const levels = [_]Compress.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
70244 const cases = [_]struct {
71245 data: []const u8, // uncompressed content
72246 // compressed data sizes per level 4-9
......@@ -113,9 +287,11 @@ test "compress/decompress" {
113287
114288 // compress original stream to compressed stream
115289 {
116 var original = fixedBufferStream(data);
117 var compressed = fixedBufferStream(&cmp_buf);
118 try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level });
290 var original: std.io.BufferedReader = undefined;
291 original.initFixed(data);
292 var compressed: std.io.BufferedWriter = undefined;
293 compressed.initFixed(&cmp_buf);
294 try Compress.pump(container, original.reader(), &compressed, .{ .level = level });
119295 if (compressed_size == 0) {
120296 if (container == .gzip)
121297 print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
......@@ -125,16 +301,19 @@ test "compress/decompress" {
125301 }
126302 // decompress compressed stream to decompressed stream
127303 {
128 var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
129 var decompressed = fixedBufferStream(&dcm_buf);
130 try inflate.decompress(container, compressed.reader(), decompressed.writer());
304 var compressed: std.io.BufferedReader = undefined;
305 compressed.initFixed(cmp_buf[0..compressed_size]);
306 var decompressed: std.io.BufferedWriter = undefined;
307 decompressed.initFixed(&dcm_buf);
308 try Decompress.pump(container, &compressed, &decompressed);
131309 try testing.expectEqualSlices(u8, data, decompressed.getWritten());
132310 }
133311
134312 // compressor writer interface
135313 {
136 var compressed = fixedBufferStream(&cmp_buf);
137 var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level });
314 var compressed: std.io.BufferedWriter = undefined;
315 compressed.initFixed(&cmp_buf);
316 var cmp = try Compress.init(container, &compressed, .{ .level = level });
138317 var cmp_wrt = cmp.writer();
139318 try cmp_wrt.writeAll(data);
140319 try cmp.finish();
......@@ -143,8 +322,9 @@ test "compress/decompress" {
143322 }
144323 // decompressor reader interface
145324 {
146 var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
147 var dcm = inflate.decompressor(container, compressed.reader());
325 var compressed: std.io.BufferedReader = undefined;
326 compressed.initFixed(cmp_buf[0..compressed_size]);
327 var dcm = Decompress.pump(container, &compressed);
148328 var dcm_rdr = dcm.reader();
149329 const n = try dcm_rdr.readAll(&dcm_buf);
150330 try testing.expectEqual(data.len, n);
......@@ -162,9 +342,11 @@ test "compress/decompress" {
162342
163343 // compress original stream to compressed stream
164344 {
165 var original = fixedBufferStream(data);
166 var compressed = fixedBufferStream(&cmp_buf);
167 var cmp = try deflate.huffman.compressor(container, compressed.writer());
345 var original: std.io.BufferedReader = undefined;
346 original.initFixed(data);
347 var compressed: std.io.BufferedWriter = undefined;
348 compressed.initFixed(&cmp_buf);
349 var cmp = try Compress.Huffman.init(container, &compressed);
168350 try cmp.compress(original.reader());
169351 try cmp.finish();
170352 if (compressed_size == 0) {
......@@ -176,9 +358,11 @@ test "compress/decompress" {
176358 }
177359 // decompress compressed stream to decompressed stream
178360 {
179 var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
180 var decompressed = fixedBufferStream(&dcm_buf);
181 try inflate.decompress(container, compressed.reader(), decompressed.writer());
361 var compressed: std.io.BufferedReader = undefined;
362 compressed.initFixed(cmp_buf[0..compressed_size]);
363 var decompressed: std.io.BufferedWriter = undefined;
364 decompressed.initFixed(&dcm_buf);
365 try Decompress.pump(container, &compressed, &decompressed);
182366 try testing.expectEqualSlices(u8, data, decompressed.getWritten());
183367 }
184368 }
......@@ -194,9 +378,11 @@ test "compress/decompress" {
194378
195379 // compress original stream to compressed stream
196380 {
197 var original = fixedBufferStream(data);
198 var compressed = fixedBufferStream(&cmp_buf);
199 var cmp = try deflate.store.compressor(container, compressed.writer());
381 var original: std.io.BufferedReader = undefined;
382 original.initFixed(data);
383 var compressed: std.io.BufferedWriter = undefined;
384 compressed.initFixed(&cmp_buf);
385 var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed);
200386 try cmp.compress(original.reader());
201387 try cmp.finish();
202388 if (compressed_size == 0) {
......@@ -209,9 +395,11 @@ test "compress/decompress" {
209395 }
210396 // decompress compressed stream to decompressed stream
211397 {
212 var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
213 var decompressed = fixedBufferStream(&dcm_buf);
214 try inflate.decompress(container, compressed.reader(), decompressed.writer());
398 var compressed: std.io.BufferedReader = undefined;
399 compressed.initFixed(cmp_buf[0..compressed_size]);
400 var decompressed: std.io.BufferedWriter = undefined;
401 decompressed.initFixed(&dcm_buf);
402 try Decompress.pump(container, &compressed, &decompressed);
215403 try testing.expectEqualSlices(u8, data, decompressed.getWritten());
216404 }
217405 }
......@@ -220,11 +408,13 @@ test "compress/decompress" {
220408}
221409
222410fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void {
223 var in = fixedBufferStream(compressed);
224 var out = std.ArrayList(u8).init(testing.allocator);
411 var in: std.io.BufferedReader = undefined;
412 in.initFixed(compressed);
413 var out: std.io.AllocatingWriter = undefined;
414 out.init(testing.allocator);
225415 defer out.deinit();
226416
227 try inflate.decompress(container, in.reader(), out.writer());
417 try Decompress.pump(container, &in, &out.buffered_writer);
228418 try testing.expectEqualSlices(u8, expected_plain, out.items);
229419}
230420
......@@ -337,23 +527,24 @@ test "public interface" {
337527 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
338528 } ++ plain_data;
339529
340 // gzip header/footer + deflate block
341 const gzip_data =
342 [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
343 deflate_block ++
344 [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
530 //// gzip header/footer + deflate block
531 //const gzip_data =
532 // [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
533 // deflate_block ++
534 // [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
345535
346 // zlib header/footer + deflate block
347 const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
348 deflate_block ++
349 [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
536 //// zlib header/footer + deflate block
537 //const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
538 // deflate_block ++
539 // [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
350540
351 const gzip = @import("gzip.zig");
352 const zlib = @import("zlib.zig");
541 // TODO
542 //const gzip = @import("gzip.zig");
543 //const zlib = @import("zlib.zig");
353544 const flate = @This();
354545
355 try testInterface(gzip, &gzip_data, &plain_data);
356 try testInterface(zlib, &zlib_data, &plain_data);
546 //try testInterface(gzip, &gzip_data, &plain_data);
547 //try testInterface(zlib, &zlib_data, &plain_data);
357548 try testInterface(flate, &deflate_block, &plain_data);
358549}
359550
......@@ -361,95 +552,181 @@ fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const
361552 var buffer1: [64]u8 = undefined;
362553 var buffer2: [64]u8 = undefined;
363554
364 var compressed = fixedBufferStream(&buffer1);
365 var plain = fixedBufferStream(&buffer2);
366
367555 // decompress
368556 {
369 var in = fixedBufferStream(gzip_data);
370 try pkg.decompress(in.reader(), plain.writer());
557 var plain: std.io.BufferedWriter = undefined;
558 plain.initFixed(&buffer2);
559
560 var in: std.io.BufferedReader = undefined;
561 in.initFixed(gzip_data);
562 try pkg.decompress(&in, &plain);
371563 try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
372564 }
373 plain.reset();
374 compressed.reset();
375565
376566 // compress/decompress
377567 {
378 var in = fixedBufferStream(plain_data);
379 try pkg.compress(in.reader(), compressed.writer(), .{});
380 compressed.reset();
381 try pkg.decompress(compressed.reader(), plain.writer());
568 var plain: std.io.BufferedWriter = undefined;
569 plain.initFixed(&buffer2);
570 var compressed: std.io.BufferedWriter = undefined;
571 compressed.initFixed(&buffer1);
572
573 var in: std.io.BufferedReader = undefined;
574 in.initFixed(plain_data);
575 try pkg.compress(&in, &compressed, .{});
576
577 var compressed_br: std.io.BufferedReader = undefined;
578 compressed_br.initFixed(&buffer1);
579 try pkg.decompress(&compressed_br, &plain);
382580 try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
383581 }
384 plain.reset();
385 compressed.reset();
386582
387583 // compressor/decompressor
388584 {
389 var in = fixedBufferStream(plain_data);
390 var cmp = try pkg.compressor(compressed.writer(), .{});
391 try cmp.compress(in.reader());
585 var plain: std.io.BufferedWriter = undefined;
586 plain.initFixed(&buffer2);
587 var compressed: std.io.BufferedWriter = undefined;
588 compressed.initFixed(&buffer1);
589
590 var in: std.io.BufferedReader = undefined;
591 in.initFixed(plain_data);
592 var cmp = try pkg.compressor(&compressed, .{});
593 try cmp.compress(&in);
392594 try cmp.finish();
393595
394 compressed.reset();
395 var dcp = pkg.decompressor(compressed.reader());
396 try dcp.decompress(plain.writer());
596 var compressed_br: std.io.BufferedReader = undefined;
597 compressed_br.initFixed(&buffer1);
598 var dcp = pkg.decompressor(&compressed_br);
599 try dcp.decompress(&plain);
397600 try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
398601 }
399 plain.reset();
400 compressed.reset();
401602
402603 // huffman
403604 {
404605 // huffman compress/decompress
405606 {
406 var in = fixedBufferStream(plain_data);
407 try pkg.huffman.compress(in.reader(), compressed.writer());
408 compressed.reset();
409 try pkg.decompress(compressed.reader(), plain.writer());
607 var plain: std.io.BufferedWriter = undefined;
608 plain.initFixed(&buffer2);
609 var compressed: std.io.BufferedWriter = undefined;
610 compressed.initFixed(&buffer1);
611
612 var in: std.io.BufferedReader = undefined;
613 in.initFixed(plain_data);
614 try pkg.huffman.compress(&in, &compressed);
615
616 var compressed_br: std.io.BufferedReader = undefined;
617 compressed_br.initFixed(&buffer1);
618 try pkg.decompress(&compressed_br, &plain);
410619 try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
411620 }
412 plain.reset();
413 compressed.reset();
414621
415622 // huffman compressor/decompressor
416623 {
417 var in = fixedBufferStream(plain_data);
418 var cmp = try pkg.huffman.compressor(compressed.writer());
419 try cmp.compress(in.reader());
624 var plain: std.io.BufferedWriter = undefined;
625 plain.initFixed(&buffer2);
626 var compressed: std.io.BufferedWriter = undefined;
627 compressed.initFixed(&buffer1);
628
629 var in: std.io.BufferedReader = undefined;
630 in.initFixed(plain_data);
631 var cmp = try pkg.huffman.compressor(&compressed);
632 try cmp.compress(&in);
420633 try cmp.finish();
421634
422 compressed.reset();
423 try pkg.decompress(compressed.reader(), plain.writer());
635 var compressed_br: std.io.BufferedReader = undefined;
636 compressed_br.initFixed(&buffer1);
637 try pkg.decompress(&compressed_br, &plain);
424638 try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
425639 }
426640 }
427 plain.reset();
428 compressed.reset();
429641
430642 // store
431643 {
432644 // store compress/decompress
433645 {
434 var in = fixedBufferStream(plain_data);
435 try pkg.store.compress(in.reader(), compressed.writer());
436 compressed.reset();
437 try pkg.decompress(compressed.reader(), plain.writer());
646 var plain: std.io.BufferedWriter = undefined;
647 plain.initFixed(&buffer2);
648 var compressed: std.io.BufferedWriter = undefined;
649 compressed.initFixed(&buffer1);
650
651 var in: std.io.BufferedReader = undefined;
652 in.initFixed(plain_data);
653 try pkg.store.compress(&in, &compressed);
654
655 var compressed_br: std.io.BufferedReader = undefined;
656 compressed_br.initFixed(&buffer1);
657 try pkg.decompress(&compressed_br, &plain);
438658 try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
439659 }
440 plain.reset();
441 compressed.reset();
442660
443661 // store compressor/decompressor
444662 {
445 var in = fixedBufferStream(plain_data);
446 var cmp = try pkg.store.compressor(compressed.writer());
447 try cmp.compress(in.reader());
663 var plain: std.io.BufferedWriter = undefined;
664 plain.initFixed(&buffer2);
665 var compressed: std.io.BufferedWriter = undefined;
666 compressed.initFixed(&buffer1);
667
668 var in: std.io.BufferedReader = undefined;
669 in.initFixed(plain_data);
670 var cmp = try pkg.store.compressor(&compressed);
671 try cmp.compress(&in);
448672 try cmp.finish();
449673
450 compressed.reset();
451 try pkg.decompress(compressed.reader(), plain.writer());
674 var compressed_br: std.io.BufferedReader = undefined;
675 compressed_br.initFixed(&buffer1);
676 try pkg.decompress(&compressed_br, &plain);
452677 try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
453678 }
454679 }
455680}
681
682pub const match = struct {
683 pub const base_length = 3; // smallest match length per the RFC section 3.2.5
684 pub const min_length = 4; // min length used in this algorithm
685 pub const max_length = 258;
686
687 pub const min_distance = 1;
688 pub const max_distance = 32768;
689};
690
691pub const history = struct {
692 pub const len = match.max_distance;
693};
694
695pub const lookup = struct {
696 pub const bits = 15;
697 pub const len = 1 << bits;
698 pub const shift = 32 - bits;
699};
700
701test "zlib should not overshoot" {
702 // Compressed zlib data with extra 4 bytes at the end.
703 const data = [_]u8{
704 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
705 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
706 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
707 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
708 };
709
710 var stream = std.io.fixedBufferStream(data[0..]);
711 const reader = stream.reader();
712
713 var dcp = Decompress.init(reader);
714 var out: [128]u8 = undefined;
715
716 // Decompress
717 var n = try dcp.reader().readAll(out[0..]);
718
719 // Expected decompressed data
720 try std.testing.expectEqual(46, n);
721 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
722
723 // Decompressor don't overshoot underlying reader.
724 // It is leaving it at the end of compressed data chunk.
725 try std.testing.expectEqual(data.len - 4, stream.getPos());
726 try std.testing.expectEqual(0, dcp.unreadBytes());
727
728 // 4 bytes after compressed chunk are available in reader.
729 n = try reader.readAll(out[0..]);
730 try std.testing.expectEqual(n, 4);
731 try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
732}
lib/std/compress/flate/BitWriter.zig deleted-93
......@@ -1,93 +0,0 @@
1//! Bit writer for use in deflate (compression).
2//!
3//! Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
4//! When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
5//! accumulate 240 bytes they are flushed to the underlying inner_writer.
6
7const std = @import("std");
8const assert = std.debug.assert;
9const BitWriter = @This();
10
11// buffer_flush_size indicates the buffer size
12// after which bytes are flushed to the writer.
13// Should preferably be a multiple of 6, since
14// we accumulate 6 bytes between writes to the buffer.
15const buffer_flush_size = 240;
16
17// buffer_size is the actual output byte buffer size.
18// It must have additional headroom for a flush
19// which can contain up to 8 bytes.
20const buffer_size = buffer_flush_size + 8;
21
22inner_writer: *std.io.BufferedWriter,
23
24// Data waiting to be written is bytes[0 .. nbytes]
25// and then the low nbits of bits. Data is always written
26// sequentially into the bytes array.
27bits: u64 = 0,
28nbits: u32 = 0, // number of bits
29bytes: [buffer_size]u8 = undefined,
30nbytes: u32 = 0, // number of bytes
31
32const Self = @This();
33
34pub fn init(bw: *std.io.BufferedWriter) Self {
35 return .{ .inner_writer = bw };
36}
37
38pub fn setWriter(self: *Self, new_writer: *std.io.BufferedWriter) void {
39 self.inner_writer = new_writer;
40}
41
42pub fn flush(self: *Self) std.io.Writer.Error!void {
43 var n = self.nbytes;
44 while (self.nbits != 0) {
45 self.bytes[n] = @as(u8, @truncate(self.bits));
46 self.bits >>= 8;
47 if (self.nbits > 8) { // Avoid underflow
48 self.nbits -= 8;
49 } else {
50 self.nbits = 0;
51 }
52 n += 1;
53 }
54 self.bits = 0;
55 _ = try self.inner_writer.write(self.bytes[0..n]);
56 self.nbytes = 0;
57}
58
59pub fn writeBits(self: *Self, b: u32, nb: u32) std.io.Writer.Error!void {
60 self.bits |= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
61 self.nbits += nb;
62 if (self.nbits < 48)
63 return;
64
65 var n = self.nbytes;
66 std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
67 n += 6;
68 if (n >= buffer_flush_size) {
69 _ = try self.inner_writer.write(self.bytes[0..n]);
70 n = 0;
71 }
72 self.nbytes = n;
73 self.bits >>= 48;
74 self.nbits -= 48;
75}
76
77pub fn writeBytes(self: *Self, bytes: []const u8) std.io.Writer.Error!void {
78 var n = self.nbytes;
79 if (self.nbits & 7 != 0) {
80 return error.UnfinishedBits;
81 }
82 while (self.nbits != 0) {
83 self.bytes[n] = @as(u8, @truncate(self.bits));
84 self.bits >>= 8;
85 self.nbits -= 8;
86 n += 1;
87 }
88 if (n != 0) {
89 _ = try self.inner_writer.write(self.bytes[0..n]);
90 }
91 self.nbytes = 0;
92 _ = try self.inner_writer.write(bytes);
93}
lib/std/compress/flate/BlockWriter.zig+51-52
......@@ -4,35 +4,34 @@ const std = @import("std");
44const io = std.io;
55const assert = std.debug.assert;
66
7const hc = @import("huffman_encoder.zig");
8const consts = @import("consts.zig").huffman;
9const Token = @import("Token.zig");
10const BitWriter = @import("BitWriter.zig");
117const BlockWriter = @This();
8const flate = @import("../flate.zig");
9const Compress = flate.Compress;
10const huffman = flate.huffman;
11const Token = @import("Token.zig");
1212
13const codegen_order = consts.codegen_order;
13const codegen_order = huffman.codegen_order;
1414const end_code_mark = 255;
15const Self = @This();
16
17bit_writer: BitWriter,
18
19codegen_freq: [consts.codegen_code_count]u16 = undefined,
20literal_freq: [consts.max_num_lit]u16 = undefined,
21distance_freq: [consts.distance_code_count]u16 = undefined,
22codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
23literal_encoding: hc.LiteralEncoder = .{},
24distance_encoding: hc.DistanceEncoder = .{},
25codegen_encoding: hc.CodegenEncoder = .{},
26fixed_literal_encoding: hc.LiteralEncoder,
27fixed_distance_encoding: hc.DistanceEncoder,
28huff_distance: hc.DistanceEncoder,
29
30pub fn init(writer: *std.io.BufferedWriter) Self {
15
16output: *std.io.BufferedWriter,
17
18codegen_freq: [huffman.codegen_code_count]u16 = undefined,
19literal_freq: [huffman.max_num_lit]u16 = undefined,
20distance_freq: [huffman.distance_code_count]u16 = undefined,
21codegen: [huffman.max_num_lit + huffman.distance_code_count + 1]u8 = undefined,
22literal_encoding: Compress.LiteralEncoder = .{},
23distance_encoding: Compress.DistanceEncoder = .{},
24codegen_encoding: Compress.CodegenEncoder = .{},
25fixed_literal_encoding: Compress.LiteralEncoder,
26fixed_distance_encoding: Compress.DistanceEncoder,
27huff_distance: Compress.DistanceEncoder,
28
29pub fn init(output: *std.io.BufferedWriter) BlockWriter {
3130 return .{
32 .bit_writer = BitWriter.init(writer),
33 .fixed_literal_encoding = hc.fixedLiteralEncoder(),
34 .fixed_distance_encoding = hc.fixedDistanceEncoder(),
35 .huff_distance = hc.huffmanDistanceEncoder(),
31 .output = output,
32 .fixed_literal_encoding = Compress.fixedLiteralEncoder(),
33 .fixed_distance_encoding = Compress.fixedDistanceEncoder(),
34 .huff_distance = Compress.huffmanDistanceEncoder(),
3635 };
3736}
3837
......@@ -42,15 +41,15 @@ pub fn init(writer: *std.io.BufferedWriter) Self {
4241/// That is after final block; when last byte could be incomplete or
4342/// after stored block; which is aligned to the byte boundary (it has x
4443/// padding bits after first 3 bits).
45pub fn flush(self: *Self) std.io.Writer.Error!void {
44pub fn flush(self: *BlockWriter) std.io.Writer.Error!void {
4645 try self.bit_writer.flush();
4746}
4847
49pub fn setWriter(self: *Self, new_writer: *std.io.BufferedWriter) void {
48pub fn setWriter(self: *BlockWriter, new_writer: *std.io.BufferedWriter) void {
5049 self.bit_writer.setWriter(new_writer);
5150}
5251
53fn writeCode(self: *Self, c: hc.HuffCode) std.io.Writer.Error!void {
52fn writeCode(self: *BlockWriter, c: Compress.HuffCode) std.io.Writer.Error!void {
5453 try self.bit_writer.writeBits(c.code, c.len);
5554}
5655
......@@ -68,11 +67,11 @@ fn writeCode(self: *Self, c: hc.HuffCode) std.io.Writer.Error!void {
6867// lit_enc: The literal encoder to use
6968// dist_enc: The distance encoder to use
7069fn generateCodegen(
71 self: *Self,
70 self: *BlockWriter,
7271 num_literals: u32,
7372 num_distances: u32,
74 lit_enc: *hc.LiteralEncoder,
75 dist_enc: *hc.DistanceEncoder,
73 lit_enc: *Compress.LiteralEncoder,
74 dist_enc: *Compress.DistanceEncoder,
7675) void {
7776 for (self.codegen_freq, 0..) |_, i| {
7877 self.codegen_freq[i] = 0;
......@@ -169,9 +168,9 @@ const DynamicSize = struct {
169168
170169// dynamicSize returns the size of dynamically encoded data in bits.
171170fn dynamicSize(
172 self: *Self,
173 lit_enc: *hc.LiteralEncoder, // literal encoder
174 dist_enc: *hc.DistanceEncoder, // distance encoder
171 self: *BlockWriter,
172 lit_enc: *Compress.LiteralEncoder, // literal encoder
173 dist_enc: *Compress.DistanceEncoder, // distance encoder
175174 extra_bits: u32,
176175) DynamicSize {
177176 var num_codegens = self.codegen_freq.len;
......@@ -195,7 +194,7 @@ fn dynamicSize(
195194}
196195
197196// fixedSize returns the size of dynamically encoded data in bits.
198fn fixedSize(self: *Self, extra_bits: u32) u32 {
197fn fixedSize(self: *BlockWriter, extra_bits: u32) u32 {
199198 return 3 +
200199 self.fixed_literal_encoding.bitLength(&self.literal_freq) +
201200 self.fixed_distance_encoding.bitLength(&self.distance_freq) +
......@@ -214,7 +213,7 @@ fn storedSizeFits(in: ?[]const u8) StoredSize {
214213 if (in == null) {
215214 return .{ .size = 0, .storable = false };
216215 }
217 if (in.?.len <= consts.max_store_block_size) {
216 if (in.?.len <= huffman.max_store_block_size) {
218217 return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
219218 }
220219 return .{ .size = 0, .storable = false };
......@@ -227,7 +226,7 @@ fn storedSizeFits(in: ?[]const u8) StoredSize {
227226// num_codegens: The number of codegens used in codegen
228227// eof: Is it the end-of-file? (end of stream)
229228fn dynamicHeader(
230 self: *Self,
229 self: *BlockWriter,
231230 num_literals: u32,
232231 num_distances: u32,
233232 num_codegens: u32,
......@@ -272,7 +271,7 @@ fn dynamicHeader(
272271 }
273272}
274273
275fn storedHeader(self: *Self, length: usize, eof: bool) std.io.Writer.Error!void {
274fn storedHeader(self: *BlockWriter, length: usize, eof: bool) std.io.Writer.Error!void {
276275 assert(length <= 65535);
277276 const flag: u32 = if (eof) 1 else 0;
278277 try self.bit_writer.writeBits(flag, 3);
......@@ -282,7 +281,7 @@ fn storedHeader(self: *Self, length: usize, eof: bool) std.io.Writer.Error!void
282281 try self.bit_writer.writeBits(~l, 16);
283282}
284283
285fn fixedHeader(self: *Self, eof: bool) std.io.Writer.Error!void {
284fn fixedHeader(self: *BlockWriter, eof: bool) std.io.Writer.Error!void {
286285 // Indicate that we are a fixed Huffman block
287286 var value: u32 = 2;
288287 if (eof) {
......@@ -296,7 +295,7 @@ fn fixedHeader(self: *Self, eof: bool) std.io.Writer.Error!void {
296295// is larger than the original bytes, the data will be written as a
297296// stored block.
298297// If the input is null, the tokens will always be Huffman encoded.
299pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) std.io.Writer.Error!void {
298pub fn write(self: *BlockWriter, tokens: []const Token, eof: bool, input: ?[]const u8) std.io.Writer.Error!void {
300299 const lit_and_dist = self.indexTokens(tokens);
301300 const num_literals = lit_and_dist.num_literals;
302301 const num_distances = lit_and_dist.num_distances;
......@@ -374,7 +373,7 @@ pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8)
374373 try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
375374}
376375
377pub fn storedBlock(self: *Self, input: []const u8, eof: bool) std.io.Writer.Error!void {
376pub fn storedBlock(self: *BlockWriter, input: []const u8, eof: bool) std.io.Writer.Error!void {
378377 try self.storedHeader(input.len, eof);
379378 try self.bit_writer.writeBytes(input);
380379}
......@@ -385,7 +384,7 @@ pub fn storedBlock(self: *Self, input: []const u8, eof: bool) std.io.Writer.Erro
385384// If input is supplied and the compression savings are below 1/16th of the
386385// input size the block is stored.
387386fn dynamicBlock(
388 self: *Self,
387 self: *BlockWriter,
389388 tokens: []const Token,
390389 eof: bool,
391390 input: ?[]const u8,
......@@ -433,7 +432,7 @@ const TotalIndexedTokens = struct {
433432// literal_freq and distance_freq, and generates literal_encoding
434433// and distance_encoding.
435434// The number of literal and distance tokens is returned.
436fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
435fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens {
437436 var num_literals: u32 = 0;
438437 var num_distances: u32 = 0;
439438
......@@ -453,7 +452,7 @@ fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
453452 self.distance_freq[t.distanceCode()] += 1;
454453 }
455454 // add end_block_marker token at the end
456 self.literal_freq[consts.end_block_marker] += 1;
455 self.literal_freq[huffman.end_block_marker] += 1;
457456
458457 // get the number of literals
459458 num_literals = @as(u32, @intCast(self.literal_freq.len));
......@@ -482,10 +481,10 @@ fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
482481// Writes a slice of tokens to the output followed by and end_block_marker.
483482// codes for literal and distance encoding must be supplied.
484483fn writeTokens(
485 self: *Self,
484 self: *BlockWriter,
486485 tokens: []const Token,
487 le_codes: []hc.HuffCode,
488 oe_codes: []hc.HuffCode,
486 le_codes: []Compress.HuffCode,
487 oe_codes: []Compress.HuffCode,
489488) std.io.Writer.Error!void {
490489 for (tokens) |t| {
491490 if (t.kind == Token.Kind.literal) {
......@@ -508,18 +507,18 @@ fn writeTokens(
508507 }
509508 }
510509 // add end_block_marker at the end
511 try self.writeCode(le_codes[consts.end_block_marker]);
510 try self.writeCode(le_codes[huffman.end_block_marker]);
512511}
513512
514513// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
515514// if the results only gains very little from compression.
516pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) std.io.Writer.Error!void {
515pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) std.io.Writer.Error!void {
517516 // Add everything as literals
518517 histogram(input, &self.literal_freq);
519518
520 self.literal_freq[consts.end_block_marker] = 1;
519 self.literal_freq[huffman.end_block_marker] = 1;
521520
522 const num_literals = consts.end_block_marker + 1;
521 const num_literals = huffman.end_block_marker + 1;
523522 self.distance_freq[0] = 1;
524523 const num_distances = 1;
525524
......@@ -560,7 +559,7 @@ pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) std.io.Writer.Err
560559 const c = encoding[t];
561560 try self.bit_writer.writeBits(c.code, c.len);
562561 }
563 try self.writeCode(encoding[consts.end_block_marker]);
562 try self.writeCode(encoding[huffman.end_block_marker]);
564563}
565564
566565// histogram accumulates a histogram of b in h.
lib/std/compress/flate/CircularBuffer.zig deleted-240
......@@ -1,240 +0,0 @@
1//! 64K buffer of uncompressed data created in inflate (decompression). Has enough
2//! history to support writing match<length, distance>; copying length of bytes
3//! from the position distance backward from current.
4//!
5//! Reads can return less than available bytes if they are spread across
6//! different circles. So reads should repeat until get required number of bytes
7//! or until returned slice is zero length.
8//!
9//! Note on deflate limits:
10//! * non-compressible block is limited to 65,535 bytes.
11//! * backward pointer is limited in distance to 32K bytes and in length to 258 bytes.
12//!
13//! Whole non-compressed block can be written without overlap. We always have
14//! history of up to 64K, more then 32K needed.
15//!
16const std = @import("std");
17const assert = std.debug.assert;
18const testing = std.testing;
19
20const consts = @import("consts.zig").match;
21
22const mask = 0xffff; // 64K - 1
23const buffer_len = mask + 1; // 64K buffer
24
25const Self = @This();
26
27buffer: [buffer_len]u8 = undefined,
28wp: usize = 0, // write position
29rp: usize = 0, // read position
30
31fn writeAll(self: *Self, buf: []const u8) void {
32 for (buf) |c| self.write(c);
33}
34
35/// Write literal.
36pub fn write(self: *Self, b: u8) void {
37 assert(self.wp - self.rp < mask);
38 self.buffer[self.wp & mask] = b;
39 self.wp += 1;
40}
41
42/// Write match (back-reference to the same data slice) starting at `distance`
43/// back from current write position, and `length` of bytes.
44pub fn writeMatch(self: *Self, length: u16, distance: u16) !void {
45 if (self.wp < distance or
46 length < consts.base_length or length > consts.max_length or
47 distance < consts.min_distance or distance > consts.max_distance)
48 {
49 return error.InvalidMatch;
50 }
51 assert(self.wp - self.rp < mask);
52
53 var from: usize = self.wp - distance & mask;
54 const from_end: usize = from + length;
55 var to: usize = self.wp & mask;
56 const to_end: usize = to + length;
57
58 self.wp += length;
59
60 // Fast path using memcpy
61 if (from_end < buffer_len and to_end < buffer_len) // start and end at the same circle
62 {
63 var cur_len = distance;
64 var remaining_len = length;
65 while (cur_len < remaining_len) {
66 @memcpy(self.buffer[to..][0..cur_len], self.buffer[from..][0..cur_len]);
67 to += cur_len;
68 remaining_len -= cur_len;
69 cur_len = cur_len * 2;
70 }
71 @memcpy(self.buffer[to..][0..remaining_len], self.buffer[from..][0..remaining_len]);
72 return;
73 }
74
75 // Slow byte by byte
76 while (to < to_end) {
77 self.buffer[to & mask] = self.buffer[from & mask];
78 to += 1;
79 from += 1;
80 }
81}
82
83/// Returns writable part of the internal buffer of size `n` at most. Advances
84/// write pointer, assumes that returned buffer will be filled with data.
85pub fn getWritable(self: *Self, n: usize) []u8 {
86 const wp = self.wp & mask;
87 const len = @min(n, buffer_len - wp);
88 self.wp += len;
89 return self.buffer[wp .. wp + len];
90}
91
92/// Read available data. Can return part of the available data if it is
93/// spread across two circles. So read until this returns zero length.
94pub fn read(self: *Self) []const u8 {
95 return self.readAtMost(buffer_len);
96}
97
98/// Read part of available data. Can return less than max even if there are
99/// more than max decoded data.
100pub fn readAtMost(self: *Self, limit: usize) []const u8 {
101 const rb = self.readBlock(if (limit == 0) buffer_len else limit);
102 defer self.rp += rb.len;
103 return self.buffer[rb.head..rb.tail];
104}
105
106const ReadBlock = struct {
107 head: usize,
108 tail: usize,
109 len: usize,
110};
111
112/// Returns position of continuous read block data.
113fn readBlock(self: *Self, max: usize) ReadBlock {
114 const r = self.rp & mask;
115 const w = self.wp & mask;
116 const n = @min(
117 max,
118 if (w >= r) w - r else buffer_len - r,
119 );
120 return .{
121 .head = r,
122 .tail = r + n,
123 .len = n,
124 };
125}
126
127/// Number of free bytes for write.
128pub fn free(self: *Self) usize {
129 return buffer_len - (self.wp - self.rp);
130}
131
132/// Full if largest match can't fit. 258 is largest match length. That much
133/// bytes can be produced in single decode step.
134pub fn full(self: *Self) bool {
135 return self.free() < 258 + 1;
136}
137
138// example from: https://youtu.be/SJPvNi4HrWQ?t=3558
139test writeMatch {
140 var cb: Self = .{};
141
142 cb.writeAll("a salad; ");
143 try cb.writeMatch(5, 9);
144 try cb.writeMatch(3, 3);
145
146 try testing.expectEqualStrings("a salad; a salsal", cb.read());
147}
148
149test "writeMatch overlap" {
150 var cb: Self = .{};
151
152 cb.writeAll("a b c ");
153 try cb.writeMatch(8, 4);
154 cb.write('d');
155
156 try testing.expectEqualStrings("a b c b c b c d", cb.read());
157}
158
159test readAtMost {
160 var cb: Self = .{};
161
162 cb.writeAll("0123456789");
163 try cb.writeMatch(50, 10);
164
165 try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]);
166 for (0..6) |i| {
167 try testing.expectEqual(i * 10, cb.rp);
168 try testing.expectEqualStrings("0123456789", cb.readAtMost(10));
169 }
170 try testing.expectEqualStrings("", cb.readAtMost(10));
171 try testing.expectEqualStrings("", cb.read());
172}
173
174test Self {
175 var cb: Self = .{};
176
177 const data = "0123456789abcdef" ** (1024 / 16);
178 cb.writeAll(data);
179 try testing.expectEqual(@as(usize, 0), cb.rp);
180 try testing.expectEqual(@as(usize, 1024), cb.wp);
181 try testing.expectEqual(@as(usize, 1024 * 63), cb.free());
182
183 for (0..62 * 4) |_|
184 try cb.writeMatch(256, 1024); // write 62K
185
186 try testing.expectEqual(@as(usize, 0), cb.rp);
187 try testing.expectEqual(@as(usize, 63 * 1024), cb.wp);
188 try testing.expectEqual(@as(usize, 1024), cb.free());
189
190 cb.writeAll(data[0..200]);
191 _ = cb.readAtMost(1024); // make some space
192 cb.writeAll(data); // overflows write position
193 try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
194 try testing.expectEqual(@as(usize, 1024), cb.rp);
195 try testing.expectEqual(@as(usize, 1024 - 200), cb.free());
196
197 const rb = cb.readBlock(Self.buffer_len);
198 try testing.expectEqual(@as(usize, 65536 - 1024), rb.len);
199 try testing.expectEqual(@as(usize, 1024), rb.head);
200 try testing.expectEqual(@as(usize, 65536), rb.tail);
201
202 try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer
203 try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
204 try testing.expectEqual(@as(usize, 65536), cb.rp);
205 try testing.expectEqual(@as(usize, 65536 - 200), cb.free());
206
207 try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest
208}
209
210test "write overlap" {
211 var cb: Self = .{};
212 cb.wp = cb.buffer.len - 15;
213 cb.rp = cb.wp;
214
215 cb.writeAll("0123456789");
216 cb.writeAll("abcdefghij");
217
218 try testing.expectEqual(cb.buffer.len + 5, cb.wp);
219 try testing.expectEqual(cb.buffer.len - 15, cb.rp);
220
221 try testing.expectEqualStrings("0123456789abcde", cb.read());
222 try testing.expectEqualStrings("fghij", cb.read());
223
224 try testing.expect(cb.wp == cb.rp);
225}
226
227test "writeMatch/read overlap" {
228 var cb: Self = .{};
229 cb.wp = cb.buffer.len - 15;
230 cb.rp = cb.wp;
231
232 cb.writeAll("0123456789");
233 try cb.writeMatch(15, 5);
234
235 try testing.expectEqualStrings("012345678956789", cb.read());
236 try testing.expectEqualStrings("5678956789", cb.read());
237
238 try cb.writeMatch(20, 25);
239 try testing.expectEqualStrings("01234567895678956789", cb.read());
240}
lib/std/compress/flate/Compress.zig created+1230
......@@ -0,0 +1,1230 @@
1//! Default compression algorithm. Has two steps: tokenization and token
2//! encoding.
3//!
4//! Tokenization takes uncompressed input stream and produces list of tokens.
5//! Each token can be literal (byte of data) or match (backrefernce to previous
6//! data with length and distance). Tokenization accumulators 32K tokens, when
7//! full or `flush` is called tokens are passed to the `block_writer`. Level
8//! defines how hard (how slow) it tries to find match.
9//!
10//! Block writer will decide which type of deflate block to write (stored, fixed,
11//! dynamic) and encode tokens to the output byte stream. Client has to call
12//! `finish` to write block with the final bit set.
13//!
14//! Container defines type of header and footer which can be gzip, zlib or raw.
15//! They all share same deflate body. Raw has no header or footer just deflate
16//! body.
17//!
18//! Compression algorithm explained in rfc-1951 (slightly edited for this case):
19//!
20//! The compressor uses a chained hash table `lookup` to find duplicated
21//! strings, using a hash function that operates on 4-byte sequences. At any
22//! given point during compression, let XYZW be the next 4 input bytes
23//! (lookahead) to be examined (not necessarily all different, of course).
24//! First, the compressor examines the hash chain for XYZW. If the chain is
25//! empty, the compressor simply writes out X as a literal byte and advances
26//! one byte in the input. If the hash chain is not empty, indicating that the
27//! sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
28//! hash function value) has occurred recently, the compressor compares all
29//! strings on the XYZW hash chain with the actual input data sequence
30//! starting at the current point, and selects the longest match.
31//!
32//! To improve overall compression, the compressor defers the selection of
33//! matches ("lazy matching"): after a match of length N has been found, the
34//! compressor searches for a longer match starting at the next input byte. If
35//! it finds a longer match, it truncates the previous match to a length of
36//! one (thus producing a single literal byte) and then emits the longer
37//! match. Otherwise, it emits the original match, and, as described above,
38//! advances N bytes before continuing.
39//!
40//!
41//! Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
42const builtin = @import("builtin");
43const std = @import("std");
44const io = std.io;
45const assert = std.debug.assert;
46const testing = std.testing;
47const expect = testing.expect;
48const mem = std.mem;
49const math = std.math;
50
51const Compress = @This();
52const Token = @import("Token.zig");
53const BlockWriter = @import("BlockWriter.zig");
54const Container = std.compress.flate.Container;
55const Lookup = @import("Lookup.zig");
56const huffman = std.compress.flate.huffman;
57
58pub const Options = struct {
59 level: Level = .default,
60 container: Container = .raw,
61};
62
63/// Trades between speed and compression size.
64/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
65/// levels 1-3 are using different algorithm to perform faster but with less
66/// compression. That is not implemented here.
67pub const Level = enum(u4) {
68 level_4 = 4,
69 level_5 = 5,
70 level_6 = 6,
71 level_7 = 7,
72 level_8 = 8,
73 level_9 = 9,
74
75 fast = 0xb,
76 default = 0xc,
77 best = 0xd,
78};
79
80// Number of tokens to accumulate in deflate before starting block encoding.
81//
82// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
83// 8 and max 9 that gives 14 or 15 bits.
84pub const n_tokens = 1 << 15;
85
86/// Algorithm knobs for each level.
87const LevelArgs = struct {
88 good: u16, // Do less lookups if we already have match of this length.
89 nice: u16, // Stop looking for better match if we found match with at least this length.
90 lazy: u16, // Don't do lazy match find if got match with at least this length.
91 chain: u16, // How many lookups for previous match to perform.
92
93 pub fn get(level: Level) LevelArgs {
94 return switch (level) {
95 .fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
96 .level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
97 .default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
98 .level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
99 .level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
100 .best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
101 };
102 }
103};
104
105lookup: Lookup = .{},
106tokens: Tokens = .{},
107output: *std.io.BufferedWriter,
108block_writer: BlockWriter,
109level: LevelArgs,
110hasher: Container.Hasher,
111
112// Match and literal at the previous position.
113// Used for lazy match finding in processWindow.
114prev_match: ?Token = null,
115prev_literal: ?u8 = null,
116
117pub fn init(output: *std.io.BufferedWriter, options: Options) std.io.Writer.Error!Compress {
118 try output.writeAll(options.container.header(output));
119 return .{
120 .output = output,
121 .block_writer = .init(output),
122 .level = .get(options.level),
123 .hasher = .init(options.container),
124 };
125}
126
127const FlushOption = enum { none, flush, final };
128
129// Process data in window and create tokens. If token buffer is full
130// flush tokens to the token writer. In the case of `flush` or `final`
131// option it will process all data from the window. In the `none` case
132// it will preserve some data for the next match.
133fn tokenize(self: *Compress, flush_opt: FlushOption) !void {
134 // flush - process all data from window
135 const should_flush = (flush_opt != .none);
136
137 // While there is data in active lookahead buffer.
138 while (self.win.activeLookahead(should_flush)) |lh| {
139 var step: u16 = 1; // 1 in the case of literal, match length otherwise
140 const pos: u16 = self.win.pos();
141 const literal = lh[0]; // literal at current position
142 const min_len: u16 = if (self.prev_match) |m| m.length() else 0;
143
144 // Try to find match at least min_len long.
145 if (self.findMatch(pos, lh, min_len)) |match| {
146 // Found better match than previous.
147 try self.addPrevLiteral();
148
149 // Is found match length good enough?
150 if (match.length() >= self.level.lazy) {
151 // Don't try to lazy find better match, use this.
152 step = try self.addMatch(match);
153 } else {
154 // Store this match.
155 self.prev_literal = literal;
156 self.prev_match = match;
157 }
158 } else {
159 // There is no better match at current pos then it was previous.
160 // Write previous match or literal.
161 if (self.prev_match) |m| {
162 // Write match from previous position.
163 step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
164 } else {
165 // No match at previous position.
166 // Write previous literal if any, and remember this literal.
167 try self.addPrevLiteral();
168 self.prev_literal = literal;
169 }
170 }
171 // Advance window and add hashes.
172 self.windowAdvance(step, lh, pos);
173 }
174
175 if (should_flush) {
176 // In the case of flushing, last few lookahead buffers were smaller then min match len.
177 // So only last literal can be unwritten.
178 assert(self.prev_match == null);
179 try self.addPrevLiteral();
180 self.prev_literal = null;
181
182 try self.flushTokens(flush_opt);
183 }
184}
185
186fn windowAdvance(self: *Compress, step: u16, lh: []const u8, pos: u16) void {
187 // current position is already added in findMatch
188 self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
189 self.win.advance(step);
190}
191
192// Add previous literal (if any) to the tokens list.
193fn addPrevLiteral(self: *Compress) !void {
194 if (self.prev_literal) |l| try self.addToken(Token.initLiteral(l));
195}
196
197// Add match to the tokens list, reset prev pointers.
198// Returns length of the added match.
199fn addMatch(self: *Compress, m: Token) !u16 {
200 try self.addToken(m);
201 self.prev_literal = null;
202 self.prev_match = null;
203 return m.length();
204}
205
206fn addToken(self: *Compress, token: Token) !void {
207 self.tokens.add(token);
208 if (self.tokens.full()) try self.flushTokens(.none);
209}
210
211// Finds largest match in the history window with the data at current pos.
212fn findMatch(self: *Compress, pos: u16, lh: []const u8, min_len: u16) ?Token {
213 var len: u16 = min_len;
214 // Previous location with the same hash (same 4 bytes).
215 var prev_pos = self.lookup.add(lh, pos);
216 // Last found match.
217 var match: ?Token = null;
218
219 // How much back-references to try, performance knob.
220 var chain: usize = self.level.chain;
221 if (len >= self.level.good) {
222 // If we've got a match that's good enough, only look in 1/4 the chain.
223 chain >>= 2;
224 }
225
226 // Hot path loop!
227 while (prev_pos > 0 and chain > 0) : (chain -= 1) {
228 const distance = pos - prev_pos;
229 if (distance > std.compress.flate.match.max_distance)
230 break;
231
232 const new_len = self.win.match(prev_pos, pos, len);
233 if (new_len > len) {
234 match = Token.initMatch(@intCast(distance), new_len);
235 if (new_len >= self.level.nice) {
236 // The match is good enough that we don't try to find a better one.
237 return match;
238 }
239 len = new_len;
240 }
241 prev_pos = self.lookup.prev(prev_pos);
242 }
243
244 return match;
245}
246
247fn flushTokens(self: *Compress, flush_opt: FlushOption) !void {
248 // Pass tokens to the token writer
249 try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
250 // Stored block ensures byte alignment.
251 // It has 3 bits (final, block_type) and then padding until byte boundary.
252 // After that everything is aligned to the boundary in the stored block.
253 // Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
254 // Last 4 bytes are byte aligned.
255 if (flush_opt == .flush) {
256 try self.block_writer.storedBlock("", false);
257 }
258 if (flush_opt != .none) {
259 // Safe to call only when byte aligned or it is OK to add
260 // padding bits (on last byte of the final block).
261 try self.block_writer.flush();
262 }
263 // Reset internal tokens store.
264 self.tokens.reset();
265 // Notify win that tokens are flushed.
266 self.win.flush();
267}
268
269// Slide win and if needed lookup tables.
270fn slide(self: *Compress) void {
271 const n = self.win.slide();
272 self.lookup.slide(n);
273}
274
275/// Compresses as much data as possible, stops when the reader becomes
276/// empty. It will introduce some output latency (reading input without
277/// producing all output) because some data are still in internal
278/// buffers.
279///
280/// It is up to the caller to call flush (if needed) or finish (required)
281/// when is need to output any pending data or complete stream.
282///
283pub fn compress(self: *Compress, reader: anytype) !void {
284 while (true) {
285 // Fill window from reader
286 const buf = self.win.writable();
287 if (buf.len == 0) {
288 try self.tokenize(.none);
289 self.slide();
290 continue;
291 }
292 const n = try reader.readAll(buf);
293 self.hasher.update(buf[0..n]);
294 self.win.written(n);
295 // Process window
296 try self.tokenize(.none);
297 // Exit when no more data in reader
298 if (n < buf.len) break;
299 }
300}
301
302/// Flushes internal buffers to the output writer. Outputs empty stored
303/// block to sync bit stream to the byte boundary, so that the
304/// decompressor can get all input data available so far.
305///
306/// It is useful mainly in compressed network protocols, to ensure that
307/// deflate bit stream can be used as byte stream. May degrade
308/// compression so it should be used only when necessary.
309///
310/// Completes the current deflate block and follows it with an empty
311/// stored block that is three zero bits plus filler bits to the next
312/// byte, followed by four bytes (00 00 ff ff).
313///
314pub fn flush(self: *Compress) !void {
315 try self.tokenize(.flush);
316}
317
318/// Completes deflate bit stream by writing any pending data as deflate
319/// final deflate block. HAS to be called once all data are written to
320/// the compressor as a signal that next block has to have final bit
321/// set.
322///
323pub fn finish(self: *Compress) !void {
324 try self.tokenize(.final);
325 try self.hasher.writeFooter(self.output);
326}
327
328/// Use another writer while preserving history. Most probably flush
329/// should be called on old writer before setting new.
330pub fn setWriter(self: *Compress, new_writer: *std.io.BufferedWriter) void {
331 self.block_writer.setWriter(new_writer);
332 self.output = new_writer;
333}
334
335// Tokens store
336const Tokens = struct {
337 list: [n_tokens]Token = undefined,
338 pos: usize = 0,
339
340 fn add(self: *Tokens, t: Token) void {
341 self.list[self.pos] = t;
342 self.pos += 1;
343 }
344
345 fn full(self: *Tokens) bool {
346 return self.pos == self.list.len;
347 }
348
349 fn reset(self: *Tokens) void {
350 self.pos = 0;
351 }
352
353 fn tokens(self: *Tokens) []const Token {
354 return self.list[0..self.pos];
355 }
356};
357
358/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
359/// only performs Huffman entropy encoding. Results in faster compression, much
360/// less memory requirements during compression but bigger compressed sizes.
361pub const Huffman = SimpleCompressor(.huffman, .raw);
362
363/// Creates store blocks only. Data are not compressed only packed into deflate
364/// store blocks. That adds 9 bytes of header for each block. Max stored block
365/// size is 64K. Block is emitted when flush is called on on finish.
366pub const store = struct {
367 pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
368 var c = try store.compressor(container, writer);
369 try c.compress(reader);
370 try c.finish();
371 }
372
373 pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
374 return SimpleCompressor(.store, container, WriterType);
375 }
376
377 pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
378 return try store.Compressor(container, @TypeOf(writer)).init(writer);
379 }
380};
381
382const SimpleCompressorKind = enum {
383 huffman,
384 store,
385};
386
387fn simpleCompressor(
388 comptime kind: SimpleCompressorKind,
389 comptime container: Container,
390 writer: anytype,
391) !SimpleCompressor(kind, container, @TypeOf(writer)) {
392 return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
393}
394
395fn SimpleCompressor(
396 comptime kind: SimpleCompressorKind,
397 comptime container: Container,
398 comptime WriterType: type,
399) type {
400 const BlockWriterType = BlockWriter(WriterType);
401 return struct {
402 buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
403 wp: usize = 0,
404
405 output: WriterType,
406 block_writer: BlockWriterType,
407 hasher: container.Hasher() = .{},
408
409 const Self = @This();
410
411 pub fn init(output: WriterType) !Self {
412 const self = Self{
413 .output = output,
414 .block_writer = BlockWriterType.init(output),
415 };
416 try container.writeHeader(self.output);
417 return self;
418 }
419
420 pub fn flush(self: *Self) !void {
421 try self.flushBuffer(false);
422 try self.block_writer.storedBlock("", false);
423 try self.block_writer.flush();
424 }
425
426 pub fn finish(self: *Self) !void {
427 try self.flushBuffer(true);
428 try self.block_writer.flush();
429 try container.writeFooter(&self.hasher, self.output);
430 }
431
432 fn flushBuffer(self: *Self, final: bool) !void {
433 const buf = self.buffer[0..self.wp];
434 switch (kind) {
435 .huffman => try self.block_writer.huffmanBlock(buf, final),
436 .store => try self.block_writer.storedBlock(buf, final),
437 }
438 self.wp = 0;
439 }
440
441 // Writes all data from the input reader of uncompressed data.
442 // It is up to the caller to call flush or finish if there is need to
443 // output compressed blocks.
444 pub fn compress(self: *Self, reader: anytype) !void {
445 while (true) {
446 // read from rdr into buffer
447 const buf = self.buffer[self.wp..];
448 if (buf.len == 0) {
449 try self.flushBuffer(false);
450 continue;
451 }
452 const n = try reader.readAll(buf);
453 self.hasher.update(buf[0..n]);
454 self.wp += n;
455 if (n < buf.len) break; // no more data in reader
456 }
457 }
458 };
459}
460
461const LiteralNode = struct {
462 literal: u16,
463 freq: u16,
464};
465
466// Describes the state of the constructed tree for a given depth.
467const LevelInfo = struct {
468 // Our level. for better printing
469 level: u32,
470
471 // The frequency of the last node at this level
472 last_freq: u32,
473
474 // The frequency of the next character to add to this level
475 next_char_freq: u32,
476
477 // The frequency of the next pair (from level below) to add to this level.
478 // Only valid if the "needed" value of the next lower level is 0.
479 next_pair_freq: u32,
480
481 // The number of chains remaining to generate for this level before moving
482 // up to the next level
483 needed: u32,
484};
485
486// hcode is a huffman code with a bit code and bit length.
487pub const HuffCode = struct {
488 code: u16 = 0,
489 len: u16 = 0,
490
491 // set sets the code and length of an hcode.
492 fn set(self: *HuffCode, code: u16, length: u16) void {
493 self.len = length;
494 self.code = code;
495 }
496};
497
498pub fn HuffmanEncoder(comptime size: usize) type {
499 return struct {
500 codes: [size]HuffCode = undefined,
501 // Reusable buffer with the longest possible frequency table.
502 freq_cache: [huffman.max_num_frequencies + 1]LiteralNode = undefined,
503 bit_count: [17]u32 = undefined,
504 lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
505 lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
506
507 const Self = @This();
508
509 // Update this Huffman Code object to be the minimum code for the specified frequency count.
510 //
511 // freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
512 // max_bits The maximum number of bits to use for any literal.
513 pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
514 var list = self.freq_cache[0 .. freq.len + 1];
515 // Number of non-zero literals
516 var count: u32 = 0;
517 // Set list to be the set of all non-zero literals and their frequencies
518 for (freq, 0..) |f, i| {
519 if (f != 0) {
520 list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
521 count += 1;
522 } else {
523 list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
524 self.codes[i].len = 0;
525 }
526 }
527 list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
528
529 list = list[0..count];
530 if (count <= 2) {
531 // Handle the small cases here, because they are awkward for the general case code. With
532 // two or fewer literals, everything has bit length 1.
533 for (list, 0..) |node, i| {
534 // "list" is in order of increasing literal value.
535 self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
536 }
537 return;
538 }
539 self.lfs = list;
540 mem.sort(LiteralNode, self.lfs, {}, byFreq);
541
542 // Get the number of literals for each bit count
543 const bit_count = self.bitCounts(list, max_bits);
544 // And do the assignment
545 self.assignEncodingAndSize(bit_count, list);
546 }
547
548 pub fn bitLength(self: *Self, freq: []u16) u32 {
549 var total: u32 = 0;
550 for (freq, 0..) |f, i| {
551 if (f != 0) {
552 total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
553 }
554 }
555 return total;
556 }
557
558 // Return the number of literals assigned to each bit size in the Huffman encoding
559 //
560 // This method is only called when list.len >= 3
561 // The cases of 0, 1, and 2 literals are handled by special case code.
562 //
563 // list: An array of the literals with non-zero frequencies
564 // and their associated frequencies. The array is in order of increasing
565 // frequency, and has as its last element a special element with frequency
566 // `math.maxInt(i32)`
567 //
568 // max_bits: The maximum number of bits that should be used to encode any literal.
569 // Must be less than 16.
570 //
571 // Returns an integer array in which array[i] indicates the number of literals
572 // that should be encoded in i bits.
573 fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
574 var max_bits = max_bits_to_use;
575 const n = list.len;
576 const max_bits_limit = 16;
577
578 assert(max_bits < max_bits_limit);
579
580 // The tree can't have greater depth than n - 1, no matter what. This
581 // saves a little bit of work in some small cases
582 max_bits = @min(max_bits, n - 1);
583
584 // Create information about each of the levels.
585 // A bogus "Level 0" whose sole purpose is so that
586 // level1.prev.needed == 0. This makes level1.next_pair_freq
587 // be a legitimate value that never gets chosen.
588 var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
589 // leaf_counts[i] counts the number of literals at the left
590 // of ancestors of the rightmost node at level i.
591 // leaf_counts[i][j] is the number of literals at the left
592 // of the level j ancestor.
593 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
594
595 {
596 var level = @as(u32, 1);
597 while (level <= max_bits) : (level += 1) {
598 // For every level, the first two items are the first two characters.
599 // We initialize the levels as if we had already figured this out.
600 levels[level] = LevelInfo{
601 .level = level,
602 .last_freq = list[1].freq,
603 .next_char_freq = list[2].freq,
604 .next_pair_freq = list[0].freq + list[1].freq,
605 .needed = 0,
606 };
607 leaf_counts[level][level] = 2;
608 if (level == 1) {
609 levels[level].next_pair_freq = math.maxInt(i32);
610 }
611 }
612 }
613
614 // We need a total of 2*n - 2 items at top level and have already generated 2.
615 levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
616
617 {
618 var level = max_bits;
619 while (true) {
620 var l = &levels[level];
621 if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
622 // We've run out of both leaves and pairs.
623 // End all calculations for this level.
624 // To make sure we never come back to this level or any lower level,
625 // set next_pair_freq impossibly large.
626 l.needed = 0;
627 levels[level + 1].next_pair_freq = math.maxInt(i32);
628 level += 1;
629 continue;
630 }
631
632 const prev_freq = l.last_freq;
633 if (l.next_char_freq < l.next_pair_freq) {
634 // The next item on this row is a leaf node.
635 const next = leaf_counts[level][level] + 1;
636 l.last_freq = l.next_char_freq;
637 // Lower leaf_counts are the same of the previous node.
638 leaf_counts[level][level] = next;
639 if (next >= list.len) {
640 l.next_char_freq = maxNode().freq;
641 } else {
642 l.next_char_freq = list[next].freq;
643 }
644 } else {
645 // The next item on this row is a pair from the previous row.
646 // next_pair_freq isn't valid until we generate two
647 // more values in the level below
648 l.last_freq = l.next_pair_freq;
649 // Take leaf counts from the lower level, except counts[level] remains the same.
650 @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
651 levels[l.level - 1].needed = 2;
652 }
653
654 l.needed -= 1;
655 if (l.needed == 0) {
656 // We've done everything we need to do for this level.
657 // Continue calculating one level up. Fill in next_pair_freq
658 // of that level with the sum of the two nodes we've just calculated on
659 // this level.
660 if (l.level == max_bits) {
661 // All done!
662 break;
663 }
664 levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
665 level += 1;
666 } else {
667 // If we stole from below, move down temporarily to replenish it.
668 while (levels[level - 1].needed > 0) {
669 level -= 1;
670 if (level == 0) {
671 break;
672 }
673 }
674 }
675 }
676 }
677
678 // Somethings is wrong if at the end, the top level is null or hasn't used
679 // all of the leaves.
680 assert(leaf_counts[max_bits][max_bits] == n);
681
682 var bit_count = self.bit_count[0 .. max_bits + 1];
683 var bits: u32 = 1;
684 const counts = &leaf_counts[max_bits];
685 {
686 var level = max_bits;
687 while (level > 0) : (level -= 1) {
688 // counts[level] gives the number of literals requiring at least "bits"
689 // bits to encode.
690 bit_count[bits] = counts[level] - counts[level - 1];
691 bits += 1;
692 if (level == 0) {
693 break;
694 }
695 }
696 }
697 return bit_count;
698 }
699
700 // Look at the leaves and assign them a bit count and an encoding as specified
701 // in RFC 1951 3.2.2
702 fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
703 var code = @as(u16, 0);
704 var list = list_arg;
705
706 for (bit_count, 0..) |bits, n| {
707 code <<= 1;
708 if (n == 0 or bits == 0) {
709 continue;
710 }
711 // The literals list[list.len-bits] .. list[list.len-bits]
712 // are encoded using "bits" bits, and get the values
713 // code, code + 1, .... The code values are
714 // assigned in literal order (not frequency order).
715 const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
716
717 self.lns = chunk;
718 mem.sort(LiteralNode, self.lns, {}, byLiteral);
719
720 for (chunk) |node| {
721 self.codes[node.literal] = HuffCode{
722 .code = bitReverse(u16, code, @as(u5, @intCast(n))),
723 .len = @as(u16, @intCast(n)),
724 };
725 code += 1;
726 }
727 list = list[0 .. list.len - @as(u32, @intCast(bits))];
728 }
729 }
730 };
731}
732
733fn maxNode() LiteralNode {
734 return LiteralNode{
735 .literal = math.maxInt(u16),
736 .freq = math.maxInt(u16),
737 };
738}
739
740pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
741 return .{};
742}
743
744pub const LiteralEncoder = HuffmanEncoder(huffman.max_num_frequencies);
745pub const DistanceEncoder = HuffmanEncoder(huffman.distance_code_count);
746pub const CodegenEncoder = HuffmanEncoder(19);
747
748// Generates a HuffmanCode corresponding to the fixed literal table
749pub fn fixedLiteralEncoder() LiteralEncoder {
750 var h: LiteralEncoder = undefined;
751 var ch: u16 = 0;
752
753 while (ch < huffman.max_num_frequencies) : (ch += 1) {
754 var bits: u16 = undefined;
755 var size: u16 = undefined;
756 switch (ch) {
757 0...143 => {
758 // size 8, 000110000 .. 10111111
759 bits = ch + 48;
760 size = 8;
761 },
762 144...255 => {
763 // size 9, 110010000 .. 111111111
764 bits = ch + 400 - 144;
765 size = 9;
766 },
767 256...279 => {
768 // size 7, 0000000 .. 0010111
769 bits = ch - 256;
770 size = 7;
771 },
772 else => {
773 // size 8, 11000000 .. 11000111
774 bits = ch + 192 - 280;
775 size = 8;
776 },
777 }
778 h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
779 }
780 return h;
781}
782
783pub fn fixedDistanceEncoder() DistanceEncoder {
784 var h: DistanceEncoder = undefined;
785 for (h.codes, 0..) |_, ch| {
786 h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
787 }
788 return h;
789}
790
791pub fn huffmanDistanceEncoder() DistanceEncoder {
792 var distance_freq = [1]u16{0} ** huffman.distance_code_count;
793 distance_freq[0] = 1;
794 // huff_distance is a static distance encoder used for huffman only encoding.
795 // It can be reused since we will not be encoding distance values.
796 var h: DistanceEncoder = .{};
797 h.generate(distance_freq[0..], 15);
798 return h;
799}
800
801fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
802 _ = context;
803 return a.literal < b.literal;
804}
805
806fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
807 _ = context;
808 if (a.freq == b.freq) {
809 return a.literal < b.literal;
810 }
811 return a.freq < b.freq;
812}
813
814test "generate a Huffman code from an array of frequencies" {
815 var freqs: [19]u16 = [_]u16{
816 8, // 0
817 1, // 1
818 1, // 2
819 2, // 3
820 5, // 4
821 10, // 5
822 9, // 6
823 1, // 7
824 0, // 8
825 0, // 9
826 0, // 10
827 0, // 11
828 0, // 12
829 0, // 13
830 0, // 14
831 0, // 15
832 1, // 16
833 3, // 17
834 5, // 18
835 };
836
837 var enc = huffmanEncoder(19);
838 enc.generate(freqs[0..], 7);
839
840 try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
841
842 try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
843 try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
844 try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
845 try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
846 try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
847 try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
848 try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
849 try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
850 try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
851 try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
852 try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
853 try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
854 try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
855 try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
856 try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
857 try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
858 try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
859 try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
860 try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
861
862 try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
863 try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
864 try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
865 try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
866 try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
867 try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
868 try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
869 try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
870 try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
871 try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
872 try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
873}
874
875test "generate a Huffman code for the fixed literal table specific to Deflate" {
876 const enc = fixedLiteralEncoder();
877 for (enc.codes) |c| {
878 switch (c.len) {
879 7 => {
880 const v = @bitReverse(@as(u7, @intCast(c.code)));
881 try testing.expect(v <= 0b0010111);
882 },
883 8 => {
884 const v = @bitReverse(@as(u8, @intCast(c.code)));
885 try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
886 (v >= 0b11000000 and v <= 11000111));
887 },
888 9 => {
889 const v = @bitReverse(@as(u9, @intCast(c.code)));
890 try testing.expect(v >= 0b110010000 and v <= 0b111111111);
891 },
892 else => unreachable,
893 }
894 }
895}
896
897test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
898 const enc = fixedDistanceEncoder();
899 for (enc.codes) |c| {
900 const v = @bitReverse(@as(u5, @intCast(c.code)));
901 try testing.expect(v <= 29);
902 try testing.expect(c.len == 5);
903 }
904}
905
906// Reverse bit-by-bit a N-bit code.
907fn bitReverse(comptime T: type, value: T, n: usize) T {
908 const r = @bitReverse(value);
909 return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
910}
911
912test bitReverse {
913 const ReverseBitsTest = struct {
914 in: u16,
915 bit_count: u5,
916 out: u16,
917 };
918
919 const reverse_bits_tests = [_]ReverseBitsTest{
920 .{ .in = 1, .bit_count = 1, .out = 1 },
921 .{ .in = 1, .bit_count = 2, .out = 2 },
922 .{ .in = 1, .bit_count = 3, .out = 4 },
923 .{ .in = 1, .bit_count = 4, .out = 8 },
924 .{ .in = 1, .bit_count = 5, .out = 16 },
925 .{ .in = 17, .bit_count = 5, .out = 17 },
926 .{ .in = 257, .bit_count = 9, .out = 257 },
927 .{ .in = 29, .bit_count = 5, .out = 23 },
928 };
929
930 for (reverse_bits_tests) |h| {
931 const v = bitReverse(u16, h.in, h.bit_count);
932 try std.testing.expectEqual(h.out, v);
933 }
934}
935
936test "fixedLiteralEncoder codes" {
937 var al = std.ArrayList(u8).init(testing.allocator);
938 defer al.deinit();
939 var bw = std.io.bitWriter(.little, al.writer());
940
941 const f = fixedLiteralEncoder();
942 for (f.codes) |c| {
943 try bw.writeBits(c.code, c.len);
944 }
945 try testing.expectEqualSlices(u8, &fixed_codes, al.items);
946}
947
948pub const fixed_codes = [_]u8{
949 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
950 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
951 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
952 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
953 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
954 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
955 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
956 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
957 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
958 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
959 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
960 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
961 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
962 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
963 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
964 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
965 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
966 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
967 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
968 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
969 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
970 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
971 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
972 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
973 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
974 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
975 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
976 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
977 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
978 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
979 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
980 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
981 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
982 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
983 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
984 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
985 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
986 0b10100011,
987};
988
989test "tokenization" {
990 const L = Token.initLiteral;
991 const M = Token.initMatch;
992
993 const cases = [_]struct {
994 data: []const u8,
995 tokens: []const Token,
996 }{
997 .{
998 .data = "Blah blah blah blah blah!",
999 .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
1000 },
1001 .{
1002 .data = "ABCDEABCD ABCDEABCD",
1003 .tokens = &[_]Token{
1004 L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
1005 L('A'), M(10, 8),
1006 },
1007 },
1008 };
1009
1010 for (cases) |c| {
1011 inline for (Container.list) |container| { // for each wrapping
1012
1013 var cw = io.countingWriter(io.null_writer);
1014 const cww = cw.writer();
1015 var df = try Compress(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
1016
1017 _ = try df.write(c.data);
1018 try df.flush();
1019
1020 // df.token_writer.show();
1021 try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
1022 try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
1023
1024 try testing.expectEqual(container.headerSize(), cw.bytes_written);
1025 try df.finish();
1026 try testing.expectEqual(container.size(), cw.bytes_written);
1027 }
1028 }
1029}
1030
1031// Tests that tokens written are equal to expected token list.
1032const TestTokenWriter = struct {
1033 const Self = @This();
1034
1035 pos: usize = 0,
1036 actual: [128]Token = undefined,
1037
1038 pub fn init(_: anytype) Self {
1039 return .{};
1040 }
1041 pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
1042 for (tokens) |t| {
1043 self.actual[self.pos] = t;
1044 self.pos += 1;
1045 }
1046 }
1047
1048 pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
1049
1050 pub fn get(self: *Self) []Token {
1051 return self.actual[0..self.pos];
1052 }
1053
1054 pub fn show(self: *Self) void {
1055 std.debug.print("\n", .{});
1056 for (self.get()) |t| {
1057 t.show();
1058 }
1059 }
1060
1061 pub fn flush(_: *Self) !void {}
1062};
1063
1064test "file tokenization" {
1065 const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
1066 const cases = [_]struct {
1067 data: []const u8, // uncompressed content
1068 // expected number of tokens producet in deflate tokenization
1069 tokens_count: [levels.len]usize = .{0} ** levels.len,
1070 }{
1071 .{
1072 .data = @embedFile("testdata/rfc1951.txt"),
1073 .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
1074 },
1075
1076 .{
1077 .data = @embedFile("testdata/block_writer/huffman-null-max.input"),
1078 .tokens_count = .{ 257, 257, 257, 257, 257, 257 },
1079 },
1080 .{
1081 .data = @embedFile("testdata/block_writer/huffman-pi.input"),
1082 .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
1083 },
1084 .{
1085 .data = @embedFile("testdata/block_writer/huffman-text.input"),
1086 .tokens_count = .{ 235, 234, 234, 234, 234, 234 },
1087 },
1088 .{
1089 .data = @embedFile("testdata/fuzz/roundtrip1.input"),
1090 .tokens_count = .{ 333, 331, 331, 331, 331, 331 },
1091 },
1092 .{
1093 .data = @embedFile("testdata/fuzz/roundtrip2.input"),
1094 .tokens_count = .{ 334, 334, 334, 334, 334, 334 },
1095 },
1096 };
1097
1098 for (cases) |case| { // for each case
1099 const data = case.data;
1100
1101 for (levels, 0..) |level, i| { // for each compression level
1102 var original = io.fixedBufferStream(data);
1103
1104 // buffer for decompressed data
1105 var al = std.ArrayList(u8).init(testing.allocator);
1106 defer al.deinit();
1107 const writer = al.writer();
1108
1109 // create compressor
1110 const WriterType = @TypeOf(writer);
1111 const TokenWriter = TokenDecoder(@TypeOf(writer));
1112 var cmp = try Compress(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
1113
1114 // Stream uncompressed `original` data to the compressor. It will
1115 // produce tokens list and pass that list to the TokenDecoder. This
1116 // TokenDecoder uses CircularBuffer from inflate to convert list of
1117 // tokens back to the uncompressed stream.
1118 try cmp.compress(original.reader());
1119 try cmp.flush();
1120 const expected_count = case.tokens_count[i];
1121 const actual = cmp.block_writer.tokens_count;
1122 if (expected_count == 0) {
1123 std.debug.print("actual token count {d}\n", .{actual});
1124 } else {
1125 try testing.expectEqual(expected_count, actual);
1126 }
1127
1128 try testing.expectEqual(data.len, al.items.len);
1129 try testing.expectEqualSlices(u8, data, al.items);
1130 }
1131 }
1132}
1133
1134const TokenDecoder = struct {
1135 output: *std.io.BufferedWriter,
1136 tokens_count: usize,
1137
1138 pub fn init(output: *std.io.BufferedWriter) TokenDecoder {
1139 return .{
1140 .output = output,
1141 .tokens_count = 0,
1142 };
1143 }
1144
1145 pub fn write(self: *TokenDecoder, tokens: []const Token, _: bool, _: ?[]const u8) !void {
1146 self.tokens_count += tokens.len;
1147 for (tokens) |t| {
1148 switch (t.kind) {
1149 .literal => self.hist.write(t.literal()),
1150 .match => try self.hist.writeMatch(t.length(), t.distance()),
1151 }
1152 if (self.hist.free() < 285) try self.flushWin();
1153 }
1154 try self.flushWin();
1155 }
1156
1157 fn flushWin(self: *TokenDecoder) !void {
1158 while (true) {
1159 const buf = self.hist.read();
1160 if (buf.len == 0) break;
1161 try self.output.writeAll(buf);
1162 }
1163 }
1164};
1165
1166test "store simple compressor" {
1167 const data = "Hello world!";
1168 const expected = [_]u8{
1169 0x1, // block type 0, final bit set
1170 0xc, 0x0, // len = 12
1171 0xf3, 0xff, // ~len
1172 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
1173 //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
1174 };
1175
1176 var fbs = std.io.fixedBufferStream(data);
1177 var al = std.ArrayList(u8).init(testing.allocator);
1178 defer al.deinit();
1179
1180 var cmp = try store.compressor(.raw, al.writer());
1181 try cmp.compress(fbs.reader());
1182 try cmp.finish();
1183 try testing.expectEqualSlices(u8, &expected, al.items);
1184
1185 fbs.reset();
1186 try al.resize(0);
1187
1188 // huffman only compresoor will also emit store block for this small sample
1189 var hc = try huffman.compressor(.raw, al.writer());
1190 try hc.compress(fbs.reader());
1191 try hc.finish();
1192 try testing.expectEqualSlices(u8, &expected, al.items);
1193}
1194
1195test "sliding window match" {
1196 const data = "Blah blah blah blah blah!";
1197 var win: std.io.BufferedWriter = .{};
1198 try expect(win.write(data) == data.len);
1199 try expect(win.wp == data.len);
1200 try expect(win.rp == 0);
1201
1202 // length between l symbols
1203 try expect(win.match(1, 6, 0) == 18);
1204 try expect(win.match(1, 11, 0) == 13);
1205 try expect(win.match(1, 16, 0) == 8);
1206 try expect(win.match(1, 21, 0) == 0);
1207
1208 // position 15 = "blah blah!"
1209 // position 20 = "blah!"
1210 try expect(win.match(15, 20, 0) == 4);
1211 try expect(win.match(15, 20, 3) == 4);
1212 try expect(win.match(15, 20, 4) == 0);
1213}
1214
1215test "sliding window slide" {
1216 var win: std.io.BufferedWriter = .{};
1217 win.wp = std.io.BufferedWriter.buffer_len - 11;
1218 win.rp = std.io.BufferedWriter.buffer_len - 111;
1219 win.buffer[win.rp] = 0xab;
1220 try expect(win.lookahead().len == 100);
1221 try expect(win.tokensBuffer().?.len == win.rp);
1222
1223 const n = win.slide();
1224 try expect(n == 32757);
1225 try expect(win.buffer[win.rp] == 0xab);
1226 try expect(win.rp == std.io.BufferedWriter.hist_len - 111);
1227 try expect(win.wp == std.io.BufferedWriter.hist_len - 11);
1228 try expect(win.lookahead().len == 100);
1229 try expect(win.tokensBuffer() == null);
1230}
lib/std/compress/flate/Decompress.zig created+891
......@@ -0,0 +1,891 @@
1//! Inflate decompresses deflate bit stream. Reads compressed data from reader
2//! provided in init. Decompressed data are stored in internal hist buffer and
3//! can be accesses iterable `next` or reader interface.
4//!
5//! Container defines header/footer wrapper around deflate bit stream. Can be
6//! gzip or zlib.
7//!
8//! Deflate bit stream consists of multiple blocks. Block can be one of three types:
9//! * stored, non compressed, max 64k in size
10//! * fixed, huffman codes are predefined
11//! * dynamic, huffman code tables are encoded at the block start
12//!
13//! `step` function runs decoder until internal `hist` buffer is full. Client than needs to read
14//! that data in order to proceed with decoding.
15//!
16//! Allocates 74.5K of internal buffers, most important are:
17//! * 64K for history (CircularBuffer)
18//! * ~10K huffman decoders (Literal and DistanceDecoder)
19
20const std = @import("../../std.zig");
21const flate = std.compress.flate;
22const Container = flate.Container;
23const Token = @import("Token.zig");
24const testing = std.testing;
25
26input: *std.io.BufferedReader,
27// Hashes, produces checksum, of uncompressed data for gzip/zlib footer.
28hasher: Container.Hasher(),
29
30// dynamic block huffman code decoders
31lit_dec: LiteralDecoder,
32dst_dec: DistanceDecoder,
33
34// current read state
35bfinal: u1,
36block_type: u2,
37state: ReadState,
38
39read_err: Error!void,
40
41const ReadState = enum {
42 protocol_header,
43 block_header,
44 block,
45 protocol_footer,
46 end,
47};
48
49const Decompress = @This();
50
51pub const Error = Container.Error || error{
52 InvalidCode,
53 InvalidMatch,
54 InvalidBlockType,
55 WrongStoredBlockNlen,
56 InvalidDynamicBlockHeader,
57 EndOfStream,
58 ReadFailed,
59 OversubscribedHuffmanTree,
60 IncompleteHuffmanTree,
61 MissingEndOfBlockCode,
62};
63
64pub fn init(input: *std.io.BufferedReader) Decompress {
65 return .{
66 .input = input,
67 .hasher = .{},
68 .lit_dec = .{},
69 .dst_dec = .{},
70 .bfinal = 0,
71 .block_type = 0b11,
72 .state = .protocol_header,
73 .read_err = {},
74 };
75}
76
77fn blockHeader(self: *Decompress) Error!void {
78 self.bfinal = try self.bits.read(u1);
79 self.block_type = try self.bits.read(u2);
80}
81
82fn storedBlock(self: *Decompress) !bool {
83 self.bits.alignToByte(); // skip padding until byte boundary
84 // everything after this is byte aligned in stored block
85 var len = try self.bits.read(u16);
86 const nlen = try self.bits.read(u16);
87 if (len != ~nlen) return error.WrongStoredBlockNlen;
88
89 while (len > 0) {
90 const buf = self.hist.getWritable(len);
91 try self.bits.readAll(buf);
92 len -= @intCast(buf.len);
93 }
94 return true;
95}
96
97fn fixedBlock(self: *Decompress) !bool {
98 while (!self.hist.full()) {
99 const code = try self.bits.readFixedCode();
100 switch (code) {
101 0...255 => self.hist.write(@intCast(code)),
102 256 => return true, // end of block
103 257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
104 else => return error.InvalidCode,
105 }
106 }
107 return false;
108}
109
110// Handles fixed block non literal (length) code.
111// Length code is followed by 5 bits of distance code.
112fn fixedDistanceCode(self: *Decompress, code: u8) !void {
113 try self.bits.fill(5 + 5 + 13);
114 const length = try self.decodeLength(code);
115 const distance = try self.decodeDistance(try self.bits.readF(u5, .{
116 .buffered = true,
117 .reverse = true,
118 }));
119 try self.hist.writeMatch(length, distance);
120}
121
122fn decodeLength(self: *Decompress, code: u8) !u16 {
123 if (code > 28) return error.InvalidCode;
124 const ml = Token.matchLength(code);
125 return if (ml.extra_bits == 0) // 0 - 5 extra bits
126 ml.base
127 else
128 ml.base + try self.bits.readN(ml.extra_bits, .{ .buffered = true });
129}
130
131fn decodeDistance(self: *Decompress, code: u8) !u16 {
132 if (code > 29) return error.InvalidCode;
133 const md = Token.matchDistance(code);
134 return if (md.extra_bits == 0) // 0 - 13 extra bits
135 md.base
136 else
137 md.base + try self.bits.readN(md.extra_bits, .{ .buffered = true });
138}
139
140fn dynamicBlockHeader(self: *Decompress) !void {
141 const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
142 const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
143 const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lengths are encoded
144
145 if (hlit > 286 or hdist > 30)
146 return error.InvalidDynamicBlockHeader;
147
148 // lengths for code lengths
149 var cl_lens = [_]u4{0} ** 19;
150 for (0..hclen) |i| {
151 cl_lens[flate.huffman.codegen_order[i]] = try self.bits.read(u3);
152 }
153 var cl_dec: CodegenDecoder = .{};
154 try cl_dec.generate(&cl_lens);
155
156 // decoded code lengths
157 var dec_lens = [_]u4{0} ** (286 + 30);
158 var pos: usize = 0;
159 while (pos < hlit + hdist) {
160 const sym = try cl_dec.find(try self.bits.peekF(u7, .{ .reverse = true }));
161 try self.bits.shift(sym.code_bits);
162 pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
163 }
164 if (pos > hlit + hdist) {
165 return error.InvalidDynamicBlockHeader;
166 }
167
168 // literal code lengths to literal decoder
169 try self.lit_dec.generate(dec_lens[0..hlit]);
170
171 // distance code lengths to distance decoder
172 try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
173}
174
175// Decode code length symbol to code length. Writes decoded length into
176// lens slice starting at position pos. Returns number of positions
177// advanced.
178fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usize {
179 if (pos >= lens.len)
180 return error.InvalidDynamicBlockHeader;
181
182 switch (code) {
183 0...15 => {
184 // Represent code lengths of 0 - 15
185 lens[pos] = @intCast(code);
186 return 1;
187 },
188 16 => {
189 // Copy the previous code length 3 - 6 times.
190 // The next 2 bits indicate repeat length
191 const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
192 if (pos == 0 or pos + n > lens.len)
193 return error.InvalidDynamicBlockHeader;
194 for (0..n) |i| {
195 lens[pos + i] = lens[pos + i - 1];
196 }
197 return n;
198 },
199 // Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
200 17 => return @as(u8, try self.bits.read(u3)) + 3,
201 // Repeat a code length of 0 for 11 - 138 times (7 bits of length)
202 18 => return @as(u8, try self.bits.read(u7)) + 11,
203 else => return error.InvalidDynamicBlockHeader,
204 }
205}
206
207// In larger archives most blocks are usually dynamic, so decompression
208// performance depends on this function.
209fn dynamicBlock(self: *Decompress) !bool {
210 // Hot path loop!
211 while (!self.hist.full()) {
212 // optimization so other bit reads can be buffered (avoiding one `if` in hot path)
213 try self.bits.fill(15);
214 const sym = try self.decodeSymbol(&self.lit_dec);
215
216 switch (sym.kind) {
217 .literal => self.hist.write(sym.symbol),
218 .match => {
219 // Decode match backreference <length, distance>
220 try self.bits.fill(5 + 15 + 13);
221 const length = try self.decodeLength(sym.symbol);
222 const dsm = try self.decodeSymbol(&self.dst_dec);
223 const distance = try self.decodeDistance(dsm.symbol);
224 try self.hist.writeMatch(length, distance);
225 },
226 .end_of_block => return true,
227 }
228 }
229 return false;
230}
231
232// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
233// decoder to find symbol for that code. We then know how many bits is
234// used. Shift bit reader for that much bits, those bits are used. And
235// return symbol.
236fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
237 const sym = try decoder.find(try self.bits.peekF(u15, .{ .buffered = true, .reverse = true }));
238 try self.bits.shift(sym.code_bits);
239 return sym;
240}
241
242fn step(self: *Decompress) !void {
243 switch (self.state) {
244 .protocol_header => {
245 try self.hasher.container().parseHeader(&self.bits);
246 self.state = .block_header;
247 },
248 .block_header => {
249 try self.blockHeader();
250 self.state = .block;
251 if (self.block_type == 2) try self.dynamicBlockHeader();
252 },
253 .block => {
254 const done = switch (self.block_type) {
255 0 => try self.storedBlock(),
256 1 => try self.fixedBlock(),
257 2 => try self.dynamicBlock(),
258 else => return error.InvalidBlockType,
259 };
260 if (done) {
261 self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
262 }
263 },
264 .protocol_footer => {
265 self.bits.alignToByte();
266 try self.hasher.container().parseFooter(&self.hasher, &self.bits);
267 self.state = .end;
268 },
269 .end => {},
270 }
271}
272
273/// Replaces the inner reader with new reader.
274pub fn setReader(self: *Decompress, new_reader: *std.io.BufferedReader) void {
275 self.bits.forward_reader = new_reader;
276 if (self.state == .end or self.state == .protocol_footer) {
277 self.state = .protocol_header;
278 }
279}
280
281// Reads all compressed data from the internal reader and outputs plain
282// (uncompressed) data to the provided writer.
283pub fn decompress(self: *Decompress, writer: *std.io.BufferedWriter) !void {
284 while (try self.next()) |buf| {
285 try writer.writeAll(buf);
286 }
287}
288
289/// Returns the number of bytes that have been read from the internal
290/// reader but not yet consumed by the decompressor.
291pub fn unreadBytes(self: Decompress) usize {
292 // There can be no error here: the denominator is not zero, and
293 // overflow is not possible since the type is unsigned.
294 return std.math.divCeil(usize, self.bits.nbits, 8) catch unreachable;
295}
296
297// Iterator interface
298
299/// Can be used in iterator like loop without memcpy to another buffer:
300/// while (try inflate.next()) |buf| { ... }
301pub fn next(self: *Decompress) Error!?[]const u8 {
302 const out = try self.get(0);
303 if (out.len == 0) return null;
304 return out;
305}
306
307/// Returns decompressed data from internal sliding window buffer.
308/// Returned buffer can be any length between 0 and `limit` bytes. 0
309/// returned bytes means end of stream reached. With limit=0 returns as
310/// much data it can. It newer will be more than 65536 bytes, which is
311/// size of internal buffer.
312/// TODO merge this logic into readerRead and readerReadVec
313pub fn get(self: *Decompress, limit: usize) Error![]const u8 {
314 while (true) {
315 const out = self.hist.readAtMost(limit);
316 if (out.len > 0) {
317 self.hasher.update(out);
318 return out;
319 }
320 if (self.state == .end) return out;
321 try self.step();
322 }
323}
324
325fn readerRead(
326 context: ?*anyopaque,
327 bw: *std.io.BufferedWriter,
328 limit: std.io.Reader.Limit,
329) std.io.Reader.RwError!usize {
330 const self: *Decompress = @alignCast(@ptrCast(context));
331 const out = try bw.writableSliceGreedy(1);
332 const in = self.get(limit.minInt(out.len)) catch |err| switch (err) {
333 error.EndOfStream => return error.EndOfStream,
334 error.ReadFailed => return error.ReadFailed,
335 else => |e| {
336 self.read_err = e;
337 return error.ReadFailed;
338 },
339 };
340 if (in.len == 0) return error.EndOfStream;
341 @memcpy(out[0..in.len], in);
342 bw.advance(in.len);
343 return in.len;
344}
345
346fn readerReadVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
347 const self: *Decompress = @alignCast(@ptrCast(context));
348 return readVec(self, data) catch |err| switch (err) {
349 error.EndOfStream => return error.EndOfStream,
350 error.ReadFailed => return error.ReadFailed,
351 else => |e| {
352 self.read_err = e;
353 return error.ReadFailed;
354 },
355 };
356}
357
358fn readerDiscard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
359 _ = context;
360 _ = limit;
361 @panic("TODO");
362}
363
364pub fn readVec(self: *Decompress, data: []const []u8) Error!usize {
365 for (data) |out| {
366 if (out.len == 0) continue;
367 const in = try self.get(out.len);
368 @memcpy(out[0..in.len], in);
369 if (in.len == 0) return error.EndOfStream;
370 return in.len;
371 }
372 return 0;
373}
374
375pub fn reader(self: *Decompress) std.io.Reader {
376 return .{
377 .context = self,
378 .vtable = &.{
379 .read = readerRead,
380 .readVec = readerReadVec,
381 .discard = readerDiscard,
382 },
383 };
384}
385
386pub fn readable(self: *Decompress, buffer: []u8) std.io.BufferedReader {
387 return reader(self).buffered(buffer);
388}
389
390pub const Symbol = packed struct {
391 pub const Kind = enum(u2) {
392 literal,
393 end_of_block,
394 match,
395 };
396
397 symbol: u8 = 0, // symbol from alphabet
398 code_bits: u4 = 0, // number of bits in code 0-15
399 kind: Kind = .literal,
400
401 code: u16 = 0, // huffman code of the symbol
402 next: u16 = 0, // pointer to the next symbol in linked list
403 // it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
404
405 // Sorting less than function.
406 pub fn asc(_: void, a: Symbol, b: Symbol) bool {
407 if (a.code_bits == b.code_bits) {
408 if (a.kind == b.kind) {
409 return a.symbol < b.symbol;
410 }
411 return @intFromEnum(a.kind) < @intFromEnum(b.kind);
412 }
413 return a.code_bits < b.code_bits;
414 }
415};
416
417pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
418pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
419pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
420
421/// Creates huffman tree codes from list of code lengths (in `build`).
422///
423/// `find` then finds symbol for code bits. Code can be any length between 1 and
424/// 15 bits. When calling `find` we don't know how many bits will be used to
425/// find symbol. When symbol is returned it has code_bits field which defines
426/// how much we should advance in bit stream.
427///
428/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
429/// many times in this table; 32K places for 286 (at most) symbols.
430/// Small lookup table is optimization for faster search.
431/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
432/// with difference that we here use statically allocated arrays.
433///
434fn HuffmanDecoder(
435 comptime alphabet_size: u16,
436 comptime max_code_bits: u4,
437 comptime lookup_bits: u4,
438) type {
439 const lookup_shift = max_code_bits - lookup_bits;
440
441 return struct {
442 // all symbols in alaphabet, sorted by code_len, symbol
443 symbols: [alphabet_size]Symbol = undefined,
444 // lookup table code -> symbol
445 lookup: [1 << lookup_bits]Symbol = undefined,
446
447 const Self = @This();
448
449 /// Generates symbols and lookup tables from list of code lens for each symbol.
450 pub fn generate(self: *Self, lens: []const u4) !void {
451 try checkCompleteness(lens);
452
453 // init alphabet with code_bits
454 for (self.symbols, 0..) |_, i| {
455 const cb: u4 = if (i < lens.len) lens[i] else 0;
456 self.symbols[i] = if (i < 256)
457 .{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
458 else if (i == 256)
459 .{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
460 else
461 .{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
462 }
463 std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
464
465 // reset lookup table
466 for (0..self.lookup.len) |i| {
467 self.lookup[i] = .{};
468 }
469
470 // assign code to symbols
471 // reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
472 var code: u16 = 0;
473 var idx: u16 = 0;
474 for (&self.symbols, 0..) |*sym, pos| {
475 if (sym.code_bits == 0) continue; // skip unused
476 sym.code = code;
477
478 const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
479 const next_idx = next_code >> lookup_shift;
480
481 if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
482 if (sym.code_bits <= lookup_bits) {
483 // fill small lookup table
484 for (idx..next_idx) |j|
485 self.lookup[j] = sym.*;
486 } else {
487 // insert into linked table starting at root
488 const root = &self.lookup[idx];
489 const root_next = root.next;
490 root.next = @intCast(pos);
491 sym.next = root_next;
492 }
493
494 idx = next_idx;
495 code = next_code;
496 }
497 }
498
499 /// Given the list of code lengths check that it represents a canonical
500 /// Huffman code for n symbols.
501 ///
502 /// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
503 fn checkCompleteness(lens: []const u4) !void {
504 if (alphabet_size == 286)
505 if (lens[256] == 0) return error.MissingEndOfBlockCode;
506
507 var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
508 var max: usize = 0;
509 for (lens) |n| {
510 if (n == 0) continue;
511 if (n > max) max = n;
512 count[n] += 1;
513 }
514 if (max == 0) // empty tree
515 return;
516
517 // check for an over-subscribed or incomplete set of lengths
518 var left: usize = 1; // one possible code of zero length
519 for (1..count.len) |len| {
520 left <<= 1; // one more bit, double codes left
521 if (count[len] > left)
522 return error.OversubscribedHuffmanTree;
523 left -= count[len]; // deduct count from possible codes
524 }
525 if (left > 0) { // left > 0 means incomplete
526 // incomplete code ok only for single length 1 code
527 if (max_code_bits > 7 and max == count[0] + count[1]) return;
528 return error.IncompleteHuffmanTree;
529 }
530 }
531
532 /// Finds symbol for lookup table code.
533 pub fn find(self: *Self, code: u16) !Symbol {
534 // try to find in lookup table
535 const idx = code >> lookup_shift;
536 const sym = self.lookup[idx];
537 if (sym.code_bits != 0) return sym;
538 // if not use linked list of symbols with same prefix
539 return self.findLinked(code, sym.next);
540 }
541
542 inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
543 var pos = start;
544 while (pos > 0) {
545 const sym = self.symbols[pos];
546 const shift = max_code_bits - sym.code_bits;
547 // compare code_bits number of upper bits
548 if ((code ^ sym.code) >> shift == 0) return sym;
549 pos = sym.next;
550 }
551 return error.InvalidCode;
552 }
553 };
554}
555
556test "init/find" {
557 // example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
558 const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
559 var h: CodegenDecoder = .{};
560 try h.generate(&code_lens);
561
562 const expected = [_]struct {
563 sym: Symbol,
564 code: u16,
565 }{
566 .{
567 .code = 0b00_00000,
568 .sym = .{ .symbol = 3, .code_bits = 2 },
569 },
570 .{
571 .code = 0b01_00000,
572 .sym = .{ .symbol = 18, .code_bits = 2 },
573 },
574 .{
575 .code = 0b100_0000,
576 .sym = .{ .symbol = 1, .code_bits = 3 },
577 },
578 .{
579 .code = 0b101_0000,
580 .sym = .{ .symbol = 4, .code_bits = 3 },
581 },
582 .{
583 .code = 0b110_0000,
584 .sym = .{ .symbol = 17, .code_bits = 3 },
585 },
586 .{
587 .code = 0b1110_000,
588 .sym = .{ .symbol = 0, .code_bits = 4 },
589 },
590 .{
591 .code = 0b1111_000,
592 .sym = .{ .symbol = 16, .code_bits = 4 },
593 },
594 };
595
596 // unused symbols
597 for (0..12) |i| {
598 try testing.expectEqual(0, h.symbols[i].code_bits);
599 }
600 // used, from index 12
601 for (expected, 12..) |e, i| {
602 try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
603 try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
604 const sym_from_code = try h.find(e.code);
605 try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
606 }
607
608 // All possible codes for each symbol.
609 // Lookup table has 126 elements, to cover all possible 7 bit codes.
610 for (0b0000_000..0b0100_000) |c| // 0..32 (32)
611 try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
612
613 for (0b0100_000..0b1000_000) |c| // 32..64 (32)
614 try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
615
616 for (0b1000_000..0b1010_000) |c| // 64..80 (16)
617 try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
618
619 for (0b1010_000..0b1100_000) |c| // 80..96 (16)
620 try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
621
622 for (0b1100_000..0b1110_000) |c| // 96..112 (16)
623 try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
624
625 for (0b1110_000..0b1111_000) |c| // 112..120 (8)
626 try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
627
628 for (0b1111_000..0b1_0000_000) |c| // 120...128 (8)
629 try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
630}
631
632test "encode/decode literals" {
633 const LiteralEncoder = std.compress.flate.Compress.LiteralEncoder;
634
635 for (1..286) |j| { // for all different number of codes
636 var enc: LiteralEncoder = .{};
637 // create frequencies
638 var freq = [_]u16{0} ** 286;
639 freq[256] = 1; // ensure we have end of block code
640 for (&freq, 1..) |*f, i| {
641 if (i % j == 0)
642 f.* = @intCast(i);
643 }
644
645 // encoder from frequencies
646 enc.generate(&freq, 15);
647
648 // get code_lens from encoder
649 var code_lens = [_]u4{0} ** 286;
650 for (code_lens, 0..) |_, i| {
651 code_lens[i] = @intCast(enc.codes[i].len);
652 }
653 // generate decoder from code lens
654 var dec: LiteralDecoder = .{};
655 try dec.generate(&code_lens);
656
657 // expect decoder code to match original encoder code
658 for (dec.symbols) |s| {
659 if (s.code_bits == 0) continue;
660 const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
661 const symbol: u16 = switch (s.kind) {
662 .literal => s.symbol,
663 .end_of_block => 256,
664 .match => @as(u16, s.symbol) + 257,
665 };
666
667 const c = enc.codes[symbol];
668 try testing.expect(c.code == c_code);
669 }
670
671 // find each symbol by code
672 for (enc.codes) |c| {
673 if (c.len == 0) continue;
674
675 const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
676 const s = try dec.find(s_code);
677 try testing.expect(s.code == s_code);
678 try testing.expect(s.code_bits == c.len);
679 }
680 }
681}
682
683test "decompress" {
684 const cases = [_]struct {
685 in: []const u8,
686 out: []const u8,
687 }{
688 // non compressed block (type 0)
689 .{
690 .in = &[_]u8{
691 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
692 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
693 },
694 .out = "Hello world\n",
695 },
696 // fixed code block (type 1)
697 .{
698 .in = &[_]u8{
699 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
700 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
701 },
702 .out = "Hello world\n",
703 },
704 // dynamic block (type 2)
705 .{
706 .in = &[_]u8{
707 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
708 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
709 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
710 },
711 .out = "ABCDEABCD ABCDEABCD",
712 },
713 };
714 for (cases) |c| {
715 var fb = std.io.fixedBufferStream(c.in);
716 var al = std.ArrayList(u8).init(testing.allocator);
717 defer al.deinit();
718
719 try decompress(.raw, fb.reader(), al.writer());
720 try testing.expectEqualStrings(c.out, al.items);
721 }
722}
723
724test "gzip decompress" {
725 const cases = [_]struct {
726 in: []const u8,
727 out: []const u8,
728 }{
729 // non compressed block (type 0)
730 .{
731 .in = &[_]u8{
732 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
733 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
734 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
735 0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
736 0x0c, 0x00, 0x00, 0x00, // gzip footer: size
737 },
738 .out = "Hello world\n",
739 },
740 // fixed code block (type 1)
741 .{
742 .in = &[_]u8{
743 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
744 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
745 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
746 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
747 },
748 .out = "Hello world\n",
749 },
750 // dynamic block (type 2)
751 .{
752 .in = &[_]u8{
753 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
754 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
755 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
756 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
757 0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
758 },
759 .out = "ABCDEABCD ABCDEABCD",
760 },
761 // gzip header with name
762 .{
763 .in = &[_]u8{
764 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
765 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
766 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
767 },
768 .out = "Hello world\n",
769 },
770 };
771 for (cases) |c| {
772 var fb = std.io.fixedBufferStream(c.in);
773 var al = std.ArrayList(u8).init(testing.allocator);
774 defer al.deinit();
775
776 try decompress(.gzip, fb.reader(), al.writer());
777 try testing.expectEqualStrings(c.out, al.items);
778 }
779}
780
781test "zlib decompress" {
782 const cases = [_]struct {
783 in: []const u8,
784 out: []const u8,
785 }{
786 // non compressed block (type 0)
787 .{
788 .in = &[_]u8{
789 0x78, 0b10_0_11100, // zlib header (2 bytes)
790 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
791 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
792 0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
793 },
794 .out = "Hello world\n",
795 },
796 };
797 for (cases) |c| {
798 var fb = std.io.fixedBufferStream(c.in);
799 var al = std.ArrayList(u8).init(testing.allocator);
800 defer al.deinit();
801
802 try decompress(.zlib, fb.reader(), al.writer());
803 try testing.expectEqualStrings(c.out, al.items);
804 }
805}
806
807test "fuzzing tests" {
808 const cases = [_]struct {
809 input: []const u8,
810 out: []const u8 = "",
811 err: ?anyerror = null,
812 }{
813 .{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
814 .{ .input = "empty-distance-alphabet01" },
815 .{ .input = "empty-distance-alphabet02" },
816 .{ .input = "end-of-stream", .err = error.EndOfStream },
817 .{ .input = "invalid-distance", .err = error.InvalidMatch },
818 .{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
819 .{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
820 .{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
821 .{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
822 .{ .input = "out-of-codes", .err = error.InvalidCode },
823 .{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
824 .{ .input = "puff02", .err = error.EndOfStream },
825 .{ .input = "puff03", .out = &[_]u8{0xa} },
826 .{ .input = "puff04", .err = error.InvalidCode },
827 .{ .input = "puff05", .err = error.EndOfStream },
828 .{ .input = "puff06", .err = error.EndOfStream },
829 .{ .input = "puff08", .err = error.InvalidCode },
830 .{ .input = "puff09", .out = "P" },
831 .{ .input = "puff10", .err = error.InvalidCode },
832 .{ .input = "puff11", .err = error.InvalidMatch },
833 .{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
834 .{ .input = "puff13", .err = error.IncompleteHuffmanTree },
835 .{ .input = "puff14", .err = error.EndOfStream },
836 .{ .input = "puff15", .err = error.IncompleteHuffmanTree },
837 .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
838 .{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
839 .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
840 .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
841 .{ .input = "fuzz3", .err = error.InvalidMatch },
842 .{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
843 .{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
844 .{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
845 .{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
846 .{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
847 .{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
848 .{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
849 .{ .input = "puff24", .err = error.IncompleteHuffmanTree },
850 .{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
851 .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
852 .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
853 };
854
855 inline for (cases, 0..) |c, case_no| {
856 var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
857 var out = std.ArrayList(u8).init(testing.allocator);
858 defer out.deinit();
859 errdefer std.debug.print("test case failed {}\n", .{case_no});
860
861 if (c.err) |expected_err| {
862 try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
863 } else {
864 try decompress(.raw, in.reader(), out.writer());
865 try testing.expectEqualStrings(c.out, out.items);
866 }
867 }
868}
869
870test "bug 18966" {
871 const input = @embedFile("testdata/fuzz/bug_18966.input");
872 const expect = @embedFile("testdata/fuzz/bug_18966.expect");
873
874 var in = std.io.fixedBufferStream(input);
875 var out = std.ArrayList(u8).init(testing.allocator);
876 defer out.deinit();
877
878 try decompress(.gzip, in.reader(), out.writer());
879 try testing.expectEqualStrings(expect, out.items);
880}
881
882test "bug 19895" {
883 const input = &[_]u8{
884 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
885 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
886 };
887 var in = std.io.fixedBufferStream(input);
888 var decomp = Decompress.init(.raw, in.reader());
889 var buf: [0]u8 = undefined;
890 try testing.expectEqual(0, try decomp.read(&buf));
891}
lib/std/compress/flate/Lookup.zig+15-15
......@@ -5,22 +5,22 @@
55const std = @import("std");
66const testing = std.testing;
77const expect = testing.expect;
8const consts = @import("consts.zig");
8const flate = @import("../flate.zig");
99
10const Self = @This();
10const Lookup = @This();
1111
1212const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
13const chain_len = 2 * consts.history.len;
13const chain_len = 2 * flate.history.len;
1414
1515// Maps hash => first position
16head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len,
16head: [flate.lookup.len]u16 = [_]u16{0} ** flate.lookup.len,
1717// Maps position => previous positions for the same hash value
1818chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
1919
2020// Calculates hash of the 4 bytes from data.
2121// Inserts `pos` position of that hash in the lookup tables.
2222// Returns previous location with the same hash value.
23pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
23pub fn add(self: *Lookup, data: []const u8, pos: u16) u16 {
2424 if (data.len < 4) return 0;
2525 const h = hash(data[0..4]);
2626 return self.set(h, pos);
......@@ -28,11 +28,11 @@ pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
2828
2929// Returns previous location with the same hash value given the current
3030// position.
31pub fn prev(self: *Self, pos: u16) u16 {
31pub fn prev(self: *Lookup, pos: u16) u16 {
3232 return self.chain[pos];
3333}
3434
35fn set(self: *Self, h: u32, pos: u16) u16 {
35fn set(self: *Lookup, h: u32, pos: u16) u16 {
3636 const p = self.head[h];
3737 self.head[h] = pos;
3838 self.chain[pos] = p;
......@@ -40,7 +40,7 @@ fn set(self: *Self, h: u32, pos: u16) u16 {
4040}
4141
4242// Slide all positions in head and chain for `n`
43pub fn slide(self: *Self, n: u16) void {
43pub fn slide(self: *Lookup, n: u16) void {
4444 for (&self.head) |*v| {
4545 v.* -|= n;
4646 }
......@@ -52,8 +52,8 @@ pub fn slide(self: *Self, n: u16) void {
5252
5353// Add `len` 4 bytes hashes from `data` into lookup.
5454// Position of the first byte is `pos`.
55pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {
56 if (len == 0 or data.len < consts.match.min_length) {
55pub fn bulkAdd(self: *Lookup, data: []const u8, len: u16, pos: u16) void {
56 if (len == 0 or data.len < flate.match.min_length) {
5757 return;
5858 }
5959 var hb =
......@@ -80,7 +80,7 @@ fn hash(b: *const [4]u8) u32 {
8080}
8181
8282fn hashu(v: u32) u32 {
83 return @intCast((v *% prime4) >> consts.lookup.shift);
83 return @intCast((v *% prime4) >> flate.lookup.shift);
8484}
8585
8686test add {
......@@ -91,7 +91,7 @@ test add {
9191 0x01, 0x02, 0x03,
9292 };
9393
94 var h: Self = .{};
94 var h: Lookup = .{};
9595 for (data, 0..) |_, i| {
9696 const p = h.add(data[i..], @intCast(i));
9797 if (i >= 8 and i < 24) {
......@@ -101,7 +101,7 @@ test add {
101101 }
102102 }
103103
104 const v = Self.hash(data[2 .. 2 + 4]);
104 const v = Lookup.hash(data[2 .. 2 + 4]);
105105 try expect(h.head[v] == 2 + 16);
106106 try expect(h.chain[2 + 16] == 2 + 8);
107107 try expect(h.chain[2 + 8] == 2);
......@@ -111,13 +111,13 @@ test bulkAdd {
111111 const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
112112
113113 // one by one
114 var h: Self = .{};
114 var h: Lookup = .{};
115115 for (data, 0..) |_, i| {
116116 _ = h.add(data[i..], @intCast(i));
117117 }
118118
119119 // in bulk
120 var bh: Self = .{};
120 var bh: Lookup = .{};
121121 bh.bulkAdd(data, data.len, 0);
122122
123123 try testing.expectEqualSlices(u16, &h.head, &bh.head);
lib/std/compress/flate/SlidingWindow.zig deleted-160
......@@ -1,160 +0,0 @@
1//! Used in deflate (compression), holds uncompressed data form which Tokens are
2//! produces. In combination with Lookup it is used to find matches in history data.
3//!
4const std = @import("std");
5const consts = @import("consts.zig");
6
7const expect = testing.expect;
8const assert = std.debug.assert;
9const testing = std.testing;
10
11const hist_len = consts.history.len;
12const buffer_len = 2 * hist_len;
13const min_lookahead = consts.match.min_length + consts.match.max_length;
14const max_rp = buffer_len - min_lookahead;
15
16const Self = @This();
17
18buffer: [buffer_len]u8 = undefined,
19wp: usize = 0, // write position
20rp: usize = 0, // read position
21fp: isize = 0, // last flush position, tokens are build from fp..rp
22
23/// Returns number of bytes written, or 0 if buffer is full and need to slide.
24pub fn write(self: *Self, buf: []const u8) usize {
25 if (self.rp >= max_rp) return 0; // need to slide
26
27 const n = @min(buf.len, buffer_len - self.wp);
28 @memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]);
29 self.wp += n;
30 return n;
31}
32
33/// Slide buffer for hist_len.
34/// Drops old history, preserves between hist_len and hist_len - min_lookahead.
35/// Returns number of bytes removed.
36pub fn slide(self: *Self) u16 {
37 assert(self.rp >= max_rp and self.wp >= self.rp);
38 const n = self.wp - hist_len;
39 @memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]);
40 self.rp -= hist_len;
41 self.wp -= hist_len;
42 self.fp -= hist_len;
43 return @intCast(n);
44}
45
46/// Data from the current position (read position). Those part of the buffer is
47/// not converted to tokens yet.
48fn lookahead(self: *Self) []const u8 {
49 assert(self.wp >= self.rp);
50 return self.buffer[self.rp..self.wp];
51}
52
53/// Returns part of the lookahead buffer. If should_flush is set no lookahead is
54/// preserved otherwise preserves enough data for the longest match. Returns
55/// null if there is not enough data.
56pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 {
57 const min: usize = if (should_flush) 0 else min_lookahead;
58 const lh = self.lookahead();
59 return if (lh.len > min) lh else null;
60}
61
62/// Advances read position, shrinks lookahead.
63pub fn advance(self: *Self, n: u16) void {
64 assert(self.wp >= self.rp + n);
65 self.rp += n;
66}
67
68/// Returns writable part of the buffer, where new uncompressed data can be
69/// written.
70pub fn writable(self: *Self) []u8 {
71 return self.buffer[self.wp..];
72}
73
74/// Notification of what part of writable buffer is filled with data.
75pub fn written(self: *Self, n: usize) void {
76 self.wp += n;
77}
78
79/// Finds match length between previous and current position.
80/// Used in hot path!
81pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 {
82 const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length);
83 // lookahead buffers from previous and current positions
84 const prev_lh = self.buffer[prev_pos..][0..max_len];
85 const curr_lh = self.buffer[curr_pos..][0..max_len];
86
87 // If we already have match (min_len > 0),
88 // test the first byte above previous len a[min_len] != b[min_len]
89 // and then all the bytes from that position to zero.
90 // That is likely positions to find difference than looping from first bytes.
91 var i: usize = min_len;
92 if (i > 0) {
93 if (max_len <= i) return 0;
94 while (true) {
95 if (prev_lh[i] != curr_lh[i]) return 0;
96 if (i == 0) break;
97 i -= 1;
98 }
99 i = min_len;
100 }
101 while (i < max_len) : (i += 1)
102 if (prev_lh[i] != curr_lh[i]) break;
103 return if (i >= consts.match.min_length) @intCast(i) else 0;
104}
105
106/// Current position of non-compressed data. Data before rp are already converted
107/// to tokens.
108pub fn pos(self: *Self) u16 {
109 return @intCast(self.rp);
110}
111
112/// Notification that token list is cleared.
113pub fn flush(self: *Self) void {
114 self.fp = @intCast(self.rp);
115}
116
117/// Part of the buffer since last flush or null if there was slide in between (so
118/// fp becomes negative).
119pub fn tokensBuffer(self: *Self) ?[]const u8 {
120 assert(self.fp <= self.rp);
121 if (self.fp < 0) return null;
122 return self.buffer[@intCast(self.fp)..self.rp];
123}
124
125test match {
126 const data = "Blah blah blah blah blah!";
127 var win: Self = .{};
128 try expect(win.write(data) == data.len);
129 try expect(win.wp == data.len);
130 try expect(win.rp == 0);
131
132 // length between l symbols
133 try expect(win.match(1, 6, 0) == 18);
134 try expect(win.match(1, 11, 0) == 13);
135 try expect(win.match(1, 16, 0) == 8);
136 try expect(win.match(1, 21, 0) == 0);
137
138 // position 15 = "blah blah!"
139 // position 20 = "blah!"
140 try expect(win.match(15, 20, 0) == 4);
141 try expect(win.match(15, 20, 3) == 4);
142 try expect(win.match(15, 20, 4) == 0);
143}
144
145test slide {
146 var win: Self = .{};
147 win.wp = Self.buffer_len - 11;
148 win.rp = Self.buffer_len - 111;
149 win.buffer[win.rp] = 0xab;
150 try expect(win.lookahead().len == 100);
151 try expect(win.tokensBuffer().?.len == win.rp);
152
153 const n = win.slide();
154 try expect(n == 32757);
155 try expect(win.buffer[win.rp] == 0xab);
156 try expect(win.rp == Self.hist_len - 111);
157 try expect(win.wp == Self.hist_len - 11);
158 try expect(win.lookahead().len == 100);
159 try expect(win.tokensBuffer() == null);
160}
lib/std/compress/flate/Token.zig+7-7
......@@ -6,7 +6,7 @@ const std = @import("std");
66const assert = std.debug.assert;
77const print = std.debug.print;
88const expect = std.testing.expect;
9const consts = @import("consts.zig").match;
9const match = std.compress.flate.match;
1010
1111const Token = @This();
1212
......@@ -26,11 +26,11 @@ pub fn literal(t: Token) u8 {
2626}
2727
2828pub fn distance(t: Token) u16 {
29 return @as(u16, t.dist) + consts.min_distance;
29 return @as(u16, t.dist) + match.min_distance;
3030}
3131
3232pub fn length(t: Token) u16 {
33 return @as(u16, t.len_lit) + consts.base_length;
33 return @as(u16, t.len_lit) + match.base_length;
3434}
3535
3636pub fn initLiteral(lit: u8) Token {
......@@ -40,12 +40,12 @@ pub fn initLiteral(lit: u8) Token {
4040// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
4141// length range 3 - 258, stored in len_lit as 0 - 255 (u8)
4242pub fn initMatch(dist: u16, len: u16) Token {
43 assert(len >= consts.min_length and len <= consts.max_length);
44 assert(dist >= consts.min_distance and dist <= consts.max_distance);
43 assert(len >= match.min_length and len <= match.max_length);
44 assert(dist >= match.min_distance and dist <= match.max_distance);
4545 return .{
4646 .kind = .match,
47 .dist = @intCast(dist - consts.min_distance),
48 .len_lit = @intCast(len - consts.base_length),
47 .dist = @intCast(dist - match.min_distance),
48 .len_lit = @intCast(len - match.base_length),
4949 };
5050}
5151
lib/std/compress/flate/consts.zig deleted-49
......@@ -1,49 +0,0 @@
1pub const deflate = struct {
2 // Number of tokens to accumulate in deflate before starting block encoding.
3 //
4 // In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
5 // 8 and max 9 that gives 14 or 15 bits.
6 pub const tokens = 1 << 15;
7};
8
9pub const match = struct {
10 pub const base_length = 3; // smallest match length per the RFC section 3.2.5
11 pub const min_length = 4; // min length used in this algorithm
12 pub const max_length = 258;
13
14 pub const min_distance = 1;
15 pub const max_distance = 32768;
16};
17
18pub const history = struct {
19 pub const len = match.max_distance;
20};
21
22pub const lookup = struct {
23 pub const bits = 15;
24 pub const len = 1 << bits;
25 pub const shift = 32 - bits;
26};
27
28pub const huffman = struct {
29 // The odd order in which the codegen code sizes are written.
30 pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
31 // The number of codegen codes.
32 pub const codegen_code_count = 19;
33
34 // The largest distance code.
35 pub const distance_code_count = 30;
36
37 // Maximum number of literals.
38 pub const max_num_lit = 286;
39
40 // Max number of frequencies used for a Huffman Code
41 // Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
42 // The largest of these is max_num_lit.
43 pub const max_num_frequencies = max_num_lit;
44
45 // Biggest block size for uncompressed block.
46 pub const max_store_block_size = 65535;
47 // The special code used to mark the end of a block.
48 pub const end_block_marker = 256;
49};
lib/std/compress/flate/container.zig deleted-208
......@@ -1,208 +0,0 @@
1//! Container of the deflate bit stream body. Container adds header before
2//! deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
3//! no footer, raw bit stream).
4//!
5//! Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
6//! addler 32 checksum.
7//!
8//! Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
9//! crc32 checksum and 4 bytes of uncompressed data length.
10//!
11//!
12//! rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
13//! rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
14//!
15
16const std = @import("std");
17
18pub const Container = enum {
19 raw, // no header or footer
20 gzip, // gzip header and footer
21 zlib, // zlib header and footer
22
23 pub fn size(w: Container) usize {
24 return headerSize(w) + footerSize(w);
25 }
26
27 pub fn headerSize(w: Container) usize {
28 return switch (w) {
29 .gzip => 10,
30 .zlib => 2,
31 .raw => 0,
32 };
33 }
34
35 pub fn footerSize(w: Container) usize {
36 return switch (w) {
37 .gzip => 8,
38 .zlib => 4,
39 .raw => 0,
40 };
41 }
42
43 pub const list = [_]Container{ .raw, .gzip, .zlib };
44
45 pub const Error = error{
46 BadGzipHeader,
47 BadZlibHeader,
48 WrongGzipChecksum,
49 WrongGzipSize,
50 WrongZlibChecksum,
51 };
52
53 pub fn writeHeader(comptime wrap: Container, writer: anytype) !void {
54 switch (wrap) {
55 .gzip => {
56 // GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
57 // - ID1 (IDentification 1), always 0x1f
58 // - ID2 (IDentification 2), always 0x8b
59 // - CM (Compression Method), always 8 = deflate
60 // - FLG (Flags), all set to 0
61 // - 4 bytes, MTIME (Modification time), not used, all set to zero
62 // - XFL (eXtra FLags), all set to zero
63 // - OS (Operating System), 03 = Unix
64 const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 };
65 try writer.writeAll(&gzipHeader);
66 },
67 .zlib => {
68 // ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
69 // 1st byte:
70 // - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
71 // - The next four bits is the CM (compression method), which is 8 for deflate.
72 // 2nd byte:
73 // - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
74 // - The next bit, FDICT, is set if a dictionary is given.
75 // - The final five FCHECK bits form a mod-31 checksum.
76 //
77 // CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
78 const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 };
79 try writer.writeAll(&zlibHeader);
80 },
81 .raw => {},
82 }
83 }
84
85 pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void {
86 var bits: [4]u8 = undefined;
87 switch (wrap) {
88 .gzip => {
89 // GZIP 8 bytes footer
90 // - 4 bytes, CRC32 (CRC-32)
91 // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
92 std.mem.writeInt(u32, &bits, hasher.chksum(), .little);
93 try writer.writeAll(&bits);
94
95 std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little);
96 try writer.writeAll(&bits);
97 },
98 .zlib => {
99 // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
100 // 4 bytes of ADLER32 (Adler-32 checksum)
101 // Checksum value of the uncompressed data (excluding any
102 // dictionary data) computed according to Adler-32
103 // algorithm.
104 std.mem.writeInt(u32, &bits, hasher.chksum(), .big);
105 try writer.writeAll(&bits);
106 },
107 .raw => {},
108 }
109 }
110
111 pub fn parseHeader(comptime wrap: Container, reader: anytype) !void {
112 switch (wrap) {
113 .gzip => try parseGzipHeader(reader),
114 .zlib => try parseZlibHeader(reader),
115 .raw => {},
116 }
117 }
118
119 fn parseGzipHeader(reader: anytype) !void {
120 const magic1 = try reader.read(u8);
121 const magic2 = try reader.read(u8);
122 const method = try reader.read(u8);
123 const flags = try reader.read(u8);
124 try reader.skipBytes(6); // mtime(4), xflags, os
125 if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
126 return error.BadGzipHeader;
127 // Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
128 if (flags != 0) {
129 if (flags & 0b0000_0100 != 0) { // FEXTRA
130 const extra_len = try reader.read(u16);
131 try reader.skipBytes(extra_len);
132 }
133 if (flags & 0b0000_1000 != 0) { // FNAME
134 try reader.skipStringZ();
135 }
136 if (flags & 0b0001_0000 != 0) { // FCOMMENT
137 try reader.skipStringZ();
138 }
139 if (flags & 0b0000_0010 != 0) { // FHCRC
140 try reader.skipBytes(2);
141 }
142 }
143 }
144
145 fn parseZlibHeader(reader: anytype) !void {
146 const cm = try reader.read(u4);
147 const cinfo = try reader.read(u4);
148 _ = try reader.read(u8);
149 if (cm != 8 or cinfo > 7) {
150 return error.BadZlibHeader;
151 }
152 }
153
154 pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {
155 switch (wrap) {
156 .gzip => {
157 try reader.fill(0);
158 if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
159 if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
160 },
161 .zlib => {
162 const chksum: u32 = @byteSwap(hasher.chksum());
163 if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
164 },
165 .raw => {},
166 }
167 }
168
169 pub fn Hasher(comptime wrap: Container) type {
170 const HasherType = switch (wrap) {
171 .gzip => std.hash.Crc32,
172 .zlib => std.hash.Adler32,
173 .raw => struct {
174 pub fn init() @This() {
175 return .{};
176 }
177 },
178 };
179
180 return struct {
181 hasher: HasherType = HasherType.init(),
182 bytes: usize = 0,
183
184 const Self = @This();
185
186 pub fn update(self: *Self, buf: []const u8) void {
187 switch (wrap) {
188 .raw => {},
189 else => {
190 self.hasher.update(buf);
191 self.bytes += buf.len;
192 },
193 }
194 }
195
196 pub fn chksum(self: *Self) u32 {
197 switch (wrap) {
198 .raw => return 0,
199 else => return self.hasher.final(),
200 }
201 }
202
203 pub fn bytesRead(self: *Self) u32 {
204 return @truncate(self.bytes);
205 }
206 };
207 }
208};
lib/std/compress/flate/deflate.zig deleted-740
......@@ -1,740 +0,0 @@
1const std = @import("std");
2const io = std.io;
3const assert = std.debug.assert;
4const testing = std.testing;
5const expect = testing.expect;
6const print = std.debug.print;
7
8const Token = @import("Token.zig");
9const consts = @import("consts.zig");
10const BlockWriter = @import("BlockWriter.zig");
11const Container = @import("container.zig").Container;
12const SlidingWindow = @import("SlidingWindow.zig");
13const Lookup = @import("Lookup.zig");
14
15pub const Options = struct {
16 level: Level = .default,
17};
18
19/// Trades between speed and compression size.
20/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
21/// levels 1-3 are using different algorithm to perform faster but with less
22/// compression. That is not implemented here.
23pub const Level = enum(u4) {
24 // zig fmt: off
25 fast = 0xb, level_4 = 4,
26 level_5 = 5,
27 default = 0xc, level_6 = 6,
28 level_7 = 7,
29 level_8 = 8,
30 best = 0xd, level_9 = 9,
31 // zig fmt: on
32};
33
34/// Algorithm knobs for each level.
35const LevelArgs = struct {
36 good: u16, // Do less lookups if we already have match of this length.
37 nice: u16, // Stop looking for better match if we found match with at least this length.
38 lazy: u16, // Don't do lazy match find if got match with at least this length.
39 chain: u16, // How many lookups for previous match to perform.
40
41 pub fn get(level: Level) LevelArgs {
42 // zig fmt: off
43 return switch (level) {
44 .fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
45 .level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
46 .default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
47 .level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
48 .level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
49 .best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
50 };
51 // zig fmt: on
52 }
53};
54
55/// Compress plain data from reader into compressed stream written to writer.
56pub fn compress(
57 comptime container: Container,
58 reader: *std.io.BufferedReader,
59 writer: *std.io.BufferedWriter,
60 options: Options,
61) !void {
62 var c = try Compressor.init(container, writer, options);
63 try c.compress(reader);
64 try c.finish();
65}
66
67/// Compressor type.
68pub fn Compressor(comptime container: Container) type {
69 return Deflate(container, BlockWriter);
70}
71
72/// Default compression algorithm. Has two steps: tokenization and token
73/// encoding.
74///
75/// Tokenization takes uncompressed input stream and produces list of tokens.
76/// Each token can be literal (byte of data) or match (backrefernce to previous
77/// data with length and distance). Tokenization accumulators 32K tokens, when
78/// full or `flush` is called tokens are passed to the `block_writer`. Level
79/// defines how hard (how slow) it tries to find match.
80///
81/// Block writer will decide which type of deflate block to write (stored, fixed,
82/// dynamic) and encode tokens to the output byte stream. Client has to call
83/// `finish` to write block with the final bit set.
84///
85/// Container defines type of header and footer which can be gzip, zlib or raw.
86/// They all share same deflate body. Raw has no header or footer just deflate
87/// body.
88///
89/// Compression algorithm explained in rfc-1951 (slightly edited for this case):
90///
91/// The compressor uses a chained hash table `lookup` to find duplicated
92/// strings, using a hash function that operates on 4-byte sequences. At any
93/// given point during compression, let XYZW be the next 4 input bytes
94/// (lookahead) to be examined (not necessarily all different, of course).
95/// First, the compressor examines the hash chain for XYZW. If the chain is
96/// empty, the compressor simply writes out X as a literal byte and advances
97/// one byte in the input. If the hash chain is not empty, indicating that the
98/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
99/// hash function value) has occurred recently, the compressor compares all
100/// strings on the XYZW hash chain with the actual input data sequence
101/// starting at the current point, and selects the longest match.
102///
103/// To improve overall compression, the compressor defers the selection of
104/// matches ("lazy matching"): after a match of length N has been found, the
105/// compressor searches for a longer match starting at the next input byte. If
106/// it finds a longer match, it truncates the previous match to a length of
107/// one (thus producing a single literal byte) and then emits the longer
108/// match. Otherwise, it emits the original match, and, as described above,
109/// advances N bytes before continuing.
110///
111///
112/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
113///
114/// Deflate function accepts BlockWriterType so we can change that in test to test
115/// just tokenization part.
116///
117fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type {
118 return struct {
119 lookup: Lookup = .{},
120 win: SlidingWindow = .{},
121 tokens: Tokens = .{},
122 wrt: WriterType,
123 block_writer: BlockWriterType,
124 level: LevelArgs,
125 hasher: container.Hasher() = .{},
126
127 // Match and literal at the previous position.
128 // Used for lazy match finding in processWindow.
129 prev_match: ?Token = null,
130 prev_literal: ?u8 = null,
131
132 const Self = @This();
133
134 pub fn init(wrt: WriterType, options: Options) !Self {
135 const self = Self{
136 .wrt = wrt,
137 .block_writer = BlockWriterType.init(wrt),
138 .level = LevelArgs.get(options.level),
139 };
140 try container.writeHeader(self.wrt);
141 return self;
142 }
143
144 const FlushOption = enum { none, flush, final };
145
146 // Process data in window and create tokens. If token buffer is full
147 // flush tokens to the token writer. In the case of `flush` or `final`
148 // option it will process all data from the window. In the `none` case
149 // it will preserve some data for the next match.
150 fn tokenize(self: *Self, flush_opt: FlushOption) !void {
151 // flush - process all data from window
152 const should_flush = (flush_opt != .none);
153
154 // While there is data in active lookahead buffer.
155 while (self.win.activeLookahead(should_flush)) |lh| {
156 var step: u16 = 1; // 1 in the case of literal, match length otherwise
157 const pos: u16 = self.win.pos();
158 const literal = lh[0]; // literal at current position
159 const min_len: u16 = if (self.prev_match) |m| m.length() else 0;
160
161 // Try to find match at least min_len long.
162 if (self.findMatch(pos, lh, min_len)) |match| {
163 // Found better match than previous.
164 try self.addPrevLiteral();
165
166 // Is found match length good enough?
167 if (match.length() >= self.level.lazy) {
168 // Don't try to lazy find better match, use this.
169 step = try self.addMatch(match);
170 } else {
171 // Store this match.
172 self.prev_literal = literal;
173 self.prev_match = match;
174 }
175 } else {
176 // There is no better match at current pos then it was previous.
177 // Write previous match or literal.
178 if (self.prev_match) |m| {
179 // Write match from previous position.
180 step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
181 } else {
182 // No match at previous position.
183 // Write previous literal if any, and remember this literal.
184 try self.addPrevLiteral();
185 self.prev_literal = literal;
186 }
187 }
188 // Advance window and add hashes.
189 self.windowAdvance(step, lh, pos);
190 }
191
192 if (should_flush) {
193 // In the case of flushing, last few lookahead buffers were smaller then min match len.
194 // So only last literal can be unwritten.
195 assert(self.prev_match == null);
196 try self.addPrevLiteral();
197 self.prev_literal = null;
198
199 try self.flushTokens(flush_opt);
200 }
201 }
202
203 fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void {
204 // current position is already added in findMatch
205 self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
206 self.win.advance(step);
207 }
208
209 // Add previous literal (if any) to the tokens list.
210 fn addPrevLiteral(self: *Self) !void {
211 if (self.prev_literal) |l| try self.addToken(Token.initLiteral(l));
212 }
213
214 // Add match to the tokens list, reset prev pointers.
215 // Returns length of the added match.
216 fn addMatch(self: *Self, m: Token) !u16 {
217 try self.addToken(m);
218 self.prev_literal = null;
219 self.prev_match = null;
220 return m.length();
221 }
222
223 fn addToken(self: *Self, token: Token) !void {
224 self.tokens.add(token);
225 if (self.tokens.full()) try self.flushTokens(.none);
226 }
227
228 // Finds largest match in the history window with the data at current pos.
229 fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token {
230 var len: u16 = min_len;
231 // Previous location with the same hash (same 4 bytes).
232 var prev_pos = self.lookup.add(lh, pos);
233 // Last found match.
234 var match: ?Token = null;
235
236 // How much back-references to try, performance knob.
237 var chain: usize = self.level.chain;
238 if (len >= self.level.good) {
239 // If we've got a match that's good enough, only look in 1/4 the chain.
240 chain >>= 2;
241 }
242
243 // Hot path loop!
244 while (prev_pos > 0 and chain > 0) : (chain -= 1) {
245 const distance = pos - prev_pos;
246 if (distance > consts.match.max_distance)
247 break;
248
249 const new_len = self.win.match(prev_pos, pos, len);
250 if (new_len > len) {
251 match = Token.initMatch(@intCast(distance), new_len);
252 if (new_len >= self.level.nice) {
253 // The match is good enough that we don't try to find a better one.
254 return match;
255 }
256 len = new_len;
257 }
258 prev_pos = self.lookup.prev(prev_pos);
259 }
260
261 return match;
262 }
263
264 fn flushTokens(self: *Self, flush_opt: FlushOption) !void {
265 // Pass tokens to the token writer
266 try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
267 // Stored block ensures byte alignment.
268 // It has 3 bits (final, block_type) and then padding until byte boundary.
269 // After that everything is aligned to the boundary in the stored block.
270 // Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
271 // Last 4 bytes are byte aligned.
272 if (flush_opt == .flush) {
273 try self.block_writer.storedBlock("", false);
274 }
275 if (flush_opt != .none) {
276 // Safe to call only when byte aligned or it is OK to add
277 // padding bits (on last byte of the final block).
278 try self.block_writer.flush();
279 }
280 // Reset internal tokens store.
281 self.tokens.reset();
282 // Notify win that tokens are flushed.
283 self.win.flush();
284 }
285
286 // Slide win and if needed lookup tables.
287 fn slide(self: *Self) void {
288 const n = self.win.slide();
289 self.lookup.slide(n);
290 }
291
292 /// Compresses as much data as possible, stops when the reader becomes
293 /// empty. It will introduce some output latency (reading input without
294 /// producing all output) because some data are still in internal
295 /// buffers.
296 ///
297 /// It is up to the caller to call flush (if needed) or finish (required)
298 /// when is need to output any pending data or complete stream.
299 ///
300 pub fn compress(self: *Self, reader: anytype) !void {
301 while (true) {
302 // Fill window from reader
303 const buf = self.win.writable();
304 if (buf.len == 0) {
305 try self.tokenize(.none);
306 self.slide();
307 continue;
308 }
309 const n = try reader.readAll(buf);
310 self.hasher.update(buf[0..n]);
311 self.win.written(n);
312 // Process window
313 try self.tokenize(.none);
314 // Exit when no more data in reader
315 if (n < buf.len) break;
316 }
317 }
318
319 /// Flushes internal buffers to the output writer. Outputs empty stored
320 /// block to sync bit stream to the byte boundary, so that the
321 /// decompressor can get all input data available so far.
322 ///
323 /// It is useful mainly in compressed network protocols, to ensure that
324 /// deflate bit stream can be used as byte stream. May degrade
325 /// compression so it should be used only when necessary.
326 ///
327 /// Completes the current deflate block and follows it with an empty
328 /// stored block that is three zero bits plus filler bits to the next
329 /// byte, followed by four bytes (00 00 ff ff).
330 ///
331 pub fn flush(self: *Self) !void {
332 try self.tokenize(.flush);
333 }
334
335 /// Completes deflate bit stream by writing any pending data as deflate
336 /// final deflate block. HAS to be called once all data are written to
337 /// the compressor as a signal that next block has to have final bit
338 /// set.
339 ///
340 pub fn finish(self: *Self) !void {
341 try self.tokenize(.final);
342 try container.writeFooter(&self.hasher, self.wrt);
343 }
344
345 /// Use another writer while preserving history. Most probably flush
346 /// should be called on old writer before setting new.
347 pub fn setWriter(self: *Self, new_writer: WriterType) void {
348 self.block_writer.setWriter(new_writer);
349 self.wrt = new_writer;
350 }
351
352 // Writer interface
353
354 pub const Writer = io.Writer(*Self, Error, write);
355 pub const Error = BlockWriterType.Error;
356
357 /// Write `input` of uncompressed data.
358 /// See compress.
359 pub fn write(self: *Self, input: []const u8) !usize {
360 var fbs = io.fixedBufferStream(input);
361 try self.compress(fbs.reader());
362 return input.len;
363 }
364
365 pub fn writer(self: *Self) Writer {
366 return .{ .context = self };
367 }
368 };
369}
370
371// Tokens store
372const Tokens = struct {
373 list: [consts.deflate.tokens]Token = undefined,
374 pos: usize = 0,
375
376 fn add(self: *Tokens, t: Token) void {
377 self.list[self.pos] = t;
378 self.pos += 1;
379 }
380
381 fn full(self: *Tokens) bool {
382 return self.pos == self.list.len;
383 }
384
385 fn reset(self: *Tokens) void {
386 self.pos = 0;
387 }
388
389 fn tokens(self: *Tokens) []const Token {
390 return self.list[0..self.pos];
391 }
392};
393
394/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
395/// only performs Huffman entropy encoding. Results in faster compression, much
396/// less memory requirements during compression but bigger compressed sizes.
397pub const huffman = struct {
398 pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
399 var c = try huffman.compressor(container, writer);
400 try c.compress(reader);
401 try c.finish();
402 }
403
404 pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
405 return SimpleCompressor(.huffman, container, WriterType);
406 }
407
408 pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) {
409 return try huffman.Compressor(container, @TypeOf(writer)).init(writer);
410 }
411};
412
413/// Creates store blocks only. Data are not compressed only packed into deflate
414/// store blocks. That adds 9 bytes of header for each block. Max stored block
415/// size is 64K. Block is emitted when flush is called on on finish.
416pub const store = struct {
417 pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
418 var c = try store.compressor(container, writer);
419 try c.compress(reader);
420 try c.finish();
421 }
422
423 pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
424 return SimpleCompressor(.store, container, WriterType);
425 }
426
427 pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
428 return try store.Compressor(container, @TypeOf(writer)).init(writer);
429 }
430};
431
432const SimpleCompressorKind = enum {
433 huffman,
434 store,
435};
436
437fn simpleCompressor(
438 comptime kind: SimpleCompressorKind,
439 comptime container: Container,
440 writer: anytype,
441) !SimpleCompressor(kind, container, @TypeOf(writer)) {
442 return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
443}
444
445fn SimpleCompressor(
446 comptime kind: SimpleCompressorKind,
447 comptime container: Container,
448 comptime WriterType: type,
449) type {
450 const BlockWriterType = BlockWriter(WriterType);
451 return struct {
452 buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
453 wp: usize = 0,
454
455 wrt: WriterType,
456 block_writer: BlockWriterType,
457 hasher: container.Hasher() = .{},
458
459 const Self = @This();
460
461 pub fn init(wrt: WriterType) !Self {
462 const self = Self{
463 .wrt = wrt,
464 .block_writer = BlockWriterType.init(wrt),
465 };
466 try container.writeHeader(self.wrt);
467 return self;
468 }
469
470 pub fn flush(self: *Self) !void {
471 try self.flushBuffer(false);
472 try self.block_writer.storedBlock("", false);
473 try self.block_writer.flush();
474 }
475
476 pub fn finish(self: *Self) !void {
477 try self.flushBuffer(true);
478 try self.block_writer.flush();
479 try container.writeFooter(&self.hasher, self.wrt);
480 }
481
482 fn flushBuffer(self: *Self, final: bool) !void {
483 const buf = self.buffer[0..self.wp];
484 switch (kind) {
485 .huffman => try self.block_writer.huffmanBlock(buf, final),
486 .store => try self.block_writer.storedBlock(buf, final),
487 }
488 self.wp = 0;
489 }
490
491 // Writes all data from the input reader of uncompressed data.
492 // It is up to the caller to call flush or finish if there is need to
493 // output compressed blocks.
494 pub fn compress(self: *Self, reader: anytype) !void {
495 while (true) {
496 // read from rdr into buffer
497 const buf = self.buffer[self.wp..];
498 if (buf.len == 0) {
499 try self.flushBuffer(false);
500 continue;
501 }
502 const n = try reader.readAll(buf);
503 self.hasher.update(buf[0..n]);
504 self.wp += n;
505 if (n < buf.len) break; // no more data in reader
506 }
507 }
508
509 // Writer interface
510
511 pub const Writer = io.Writer(*Self, Error, write);
512 pub const Error = BlockWriterType.Error;
513
514 // Write `input` of uncompressed data.
515 pub fn write(self: *Self, input: []const u8) !usize {
516 var fbs = io.fixedBufferStream(input);
517 try self.compress(fbs.reader());
518 return input.len;
519 }
520
521 pub fn writer(self: *Self) Writer {
522 return .{ .context = self };
523 }
524 };
525}
526
527const builtin = @import("builtin");
528
529test "tokenization" {
530 const L = Token.initLiteral;
531 const M = Token.initMatch;
532
533 const cases = [_]struct {
534 data: []const u8,
535 tokens: []const Token,
536 }{
537 .{
538 .data = "Blah blah blah blah blah!",
539 .tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
540 },
541 .{
542 .data = "ABCDEABCD ABCDEABCD",
543 .tokens = &[_]Token{
544 L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
545 L('A'), M(10, 8),
546 },
547 },
548 };
549
550 for (cases) |c| {
551 inline for (Container.list) |container| { // for each wrapping
552
553 var cw = io.countingWriter(io.null_writer);
554 const cww = cw.writer();
555 var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
556
557 _ = try df.write(c.data);
558 try df.flush();
559
560 // df.token_writer.show();
561 try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
562 try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
563
564 try testing.expectEqual(container.headerSize(), cw.bytes_written);
565 try df.finish();
566 try testing.expectEqual(container.size(), cw.bytes_written);
567 }
568 }
569}
570
571// Tests that tokens written are equal to expected token list.
572const TestTokenWriter = struct {
573 const Self = @This();
574
575 pos: usize = 0,
576 actual: [128]Token = undefined,
577
578 pub fn init(_: anytype) Self {
579 return .{};
580 }
581 pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
582 for (tokens) |t| {
583 self.actual[self.pos] = t;
584 self.pos += 1;
585 }
586 }
587
588 pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
589
590 pub fn get(self: *Self) []Token {
591 return self.actual[0..self.pos];
592 }
593
594 pub fn show(self: *Self) void {
595 print("\n", .{});
596 for (self.get()) |t| {
597 t.show();
598 }
599 }
600
601 pub fn flush(_: *Self) !void {}
602};
603
604test "file tokenization" {
605 const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
606 const cases = [_]struct {
607 data: []const u8, // uncompressed content
608 // expected number of tokens producet in deflate tokenization
609 tokens_count: [levels.len]usize = .{0} ** levels.len,
610 }{
611 .{
612 .data = @embedFile("testdata/rfc1951.txt"),
613 .tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
614 },
615
616 .{
617 .data = @embedFile("testdata/block_writer/huffman-null-max.input"),
618 .tokens_count = .{ 257, 257, 257, 257, 257, 257 },
619 },
620 .{
621 .data = @embedFile("testdata/block_writer/huffman-pi.input"),
622 .tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
623 },
624 .{
625 .data = @embedFile("testdata/block_writer/huffman-text.input"),
626 .tokens_count = .{ 235, 234, 234, 234, 234, 234 },
627 },
628 .{
629 .data = @embedFile("testdata/fuzz/roundtrip1.input"),
630 .tokens_count = .{ 333, 331, 331, 331, 331, 331 },
631 },
632 .{
633 .data = @embedFile("testdata/fuzz/roundtrip2.input"),
634 .tokens_count = .{ 334, 334, 334, 334, 334, 334 },
635 },
636 };
637
638 for (cases) |case| { // for each case
639 const data = case.data;
640
641 for (levels, 0..) |level, i| { // for each compression level
642 var original = io.fixedBufferStream(data);
643
644 // buffer for decompressed data
645 var al = std.ArrayList(u8).init(testing.allocator);
646 defer al.deinit();
647 const writer = al.writer();
648
649 // create compressor
650 const WriterType = @TypeOf(writer);
651 const TokenWriter = TokenDecoder(@TypeOf(writer));
652 var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
653
654 // Stream uncompressed `original` data to the compressor. It will
655 // produce tokens list and pass that list to the TokenDecoder. This
656 // TokenDecoder uses CircularBuffer from inflate to convert list of
657 // tokens back to the uncompressed stream.
658 try cmp.compress(original.reader());
659 try cmp.flush();
660 const expected_count = case.tokens_count[i];
661 const actual = cmp.block_writer.tokens_count;
662 if (expected_count == 0) {
663 print("actual token count {d}\n", .{actual});
664 } else {
665 try testing.expectEqual(expected_count, actual);
666 }
667
668 try testing.expectEqual(data.len, al.items.len);
669 try testing.expectEqualSlices(u8, data, al.items);
670 }
671 }
672}
673
674fn TokenDecoder(comptime WriterType: type) type {
675 return struct {
676 const CircularBuffer = @import("CircularBuffer.zig");
677 hist: CircularBuffer = .{},
678 wrt: WriterType,
679 tokens_count: usize = 0,
680
681 const Self = @This();
682
683 pub fn init(wrt: WriterType) Self {
684 return .{ .wrt = wrt };
685 }
686
687 pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
688 self.tokens_count += tokens.len;
689 for (tokens) |t| {
690 switch (t.kind) {
691 .literal => self.hist.write(t.literal()),
692 .match => try self.hist.writeMatch(t.length(), t.distance()),
693 }
694 if (self.hist.free() < 285) try self.flushWin();
695 }
696 try self.flushWin();
697 }
698
699 pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
700
701 fn flushWin(self: *Self) !void {
702 while (true) {
703 const buf = self.hist.read();
704 if (buf.len == 0) break;
705 try self.wrt.writeAll(buf);
706 }
707 }
708
709 pub fn flush(_: *Self) !void {}
710 };
711}
712
713test "store simple compressor" {
714 const data = "Hello world!";
715 const expected = [_]u8{
716 0x1, // block type 0, final bit set
717 0xc, 0x0, // len = 12
718 0xf3, 0xff, // ~len
719 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
720 //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
721 };
722
723 var fbs = std.io.fixedBufferStream(data);
724 var al = std.ArrayList(u8).init(testing.allocator);
725 defer al.deinit();
726
727 var cmp = try store.compressor(.raw, al.writer());
728 try cmp.compress(fbs.reader());
729 try cmp.finish();
730 try testing.expectEqualSlices(u8, &expected, al.items);
731
732 fbs.reset();
733 try al.resize(0);
734
735 // huffman only compresoor will also emit store block for this small sample
736 var hc = try huffman.compressor(.raw, al.writer());
737 try hc.compress(fbs.reader());
738 try hc.finish();
739 try testing.expectEqualSlices(u8, &expected, al.items);
740}
lib/std/compress/flate/huffman_decoder.zig deleted-302
......@@ -1,302 +0,0 @@
1const std = @import("std");
2const testing = std.testing;
3
4pub const Symbol = packed struct {
5 pub const Kind = enum(u2) {
6 literal,
7 end_of_block,
8 match,
9 };
10
11 symbol: u8 = 0, // symbol from alphabet
12 code_bits: u4 = 0, // number of bits in code 0-15
13 kind: Kind = .literal,
14
15 code: u16 = 0, // huffman code of the symbol
16 next: u16 = 0, // pointer to the next symbol in linked list
17 // it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
18
19 // Sorting less than function.
20 pub fn asc(_: void, a: Symbol, b: Symbol) bool {
21 if (a.code_bits == b.code_bits) {
22 if (a.kind == b.kind) {
23 return a.symbol < b.symbol;
24 }
25 return @intFromEnum(a.kind) < @intFromEnum(b.kind);
26 }
27 return a.code_bits < b.code_bits;
28 }
29};
30
31pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
32pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
33pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
34
35pub const Error = error{
36 InvalidCode,
37 OversubscribedHuffmanTree,
38 IncompleteHuffmanTree,
39 MissingEndOfBlockCode,
40};
41
42/// Creates huffman tree codes from list of code lengths (in `build`).
43///
44/// `find` then finds symbol for code bits. Code can be any length between 1 and
45/// 15 bits. When calling `find` we don't know how many bits will be used to
46/// find symbol. When symbol is returned it has code_bits field which defines
47/// how much we should advance in bit stream.
48///
49/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
50/// many times in this table; 32K places for 286 (at most) symbols.
51/// Small lookup table is optimization for faster search.
52/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
53/// with difference that we here use statically allocated arrays.
54///
55fn HuffmanDecoder(
56 comptime alphabet_size: u16,
57 comptime max_code_bits: u4,
58 comptime lookup_bits: u4,
59) type {
60 const lookup_shift = max_code_bits - lookup_bits;
61
62 return struct {
63 // all symbols in alaphabet, sorted by code_len, symbol
64 symbols: [alphabet_size]Symbol = undefined,
65 // lookup table code -> symbol
66 lookup: [1 << lookup_bits]Symbol = undefined,
67
68 const Self = @This();
69
70 /// Generates symbols and lookup tables from list of code lens for each symbol.
71 pub fn generate(self: *Self, lens: []const u4) !void {
72 try checkCompleteness(lens);
73
74 // init alphabet with code_bits
75 for (self.symbols, 0..) |_, i| {
76 const cb: u4 = if (i < lens.len) lens[i] else 0;
77 self.symbols[i] = if (i < 256)
78 .{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
79 else if (i == 256)
80 .{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
81 else
82 .{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
83 }
84 std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
85
86 // reset lookup table
87 for (0..self.lookup.len) |i| {
88 self.lookup[i] = .{};
89 }
90
91 // assign code to symbols
92 // reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
93 var code: u16 = 0;
94 var idx: u16 = 0;
95 for (&self.symbols, 0..) |*sym, pos| {
96 if (sym.code_bits == 0) continue; // skip unused
97 sym.code = code;
98
99 const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
100 const next_idx = next_code >> lookup_shift;
101
102 if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
103 if (sym.code_bits <= lookup_bits) {
104 // fill small lookup table
105 for (idx..next_idx) |j|
106 self.lookup[j] = sym.*;
107 } else {
108 // insert into linked table starting at root
109 const root = &self.lookup[idx];
110 const root_next = root.next;
111 root.next = @intCast(pos);
112 sym.next = root_next;
113 }
114
115 idx = next_idx;
116 code = next_code;
117 }
118 }
119
120 /// Given the list of code lengths check that it represents a canonical
121 /// Huffman code for n symbols.
122 ///
123 /// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
124 fn checkCompleteness(lens: []const u4) !void {
125 if (alphabet_size == 286)
126 if (lens[256] == 0) return error.MissingEndOfBlockCode;
127
128 var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
129 var max: usize = 0;
130 for (lens) |n| {
131 if (n == 0) continue;
132 if (n > max) max = n;
133 count[n] += 1;
134 }
135 if (max == 0) // empty tree
136 return;
137
138 // check for an over-subscribed or incomplete set of lengths
139 var left: usize = 1; // one possible code of zero length
140 for (1..count.len) |len| {
141 left <<= 1; // one more bit, double codes left
142 if (count[len] > left)
143 return error.OversubscribedHuffmanTree;
144 left -= count[len]; // deduct count from possible codes
145 }
146 if (left > 0) { // left > 0 means incomplete
147 // incomplete code ok only for single length 1 code
148 if (max_code_bits > 7 and max == count[0] + count[1]) return;
149 return error.IncompleteHuffmanTree;
150 }
151 }
152
153 /// Finds symbol for lookup table code.
154 pub fn find(self: *Self, code: u16) !Symbol {
155 // try to find in lookup table
156 const idx = code >> lookup_shift;
157 const sym = self.lookup[idx];
158 if (sym.code_bits != 0) return sym;
159 // if not use linked list of symbols with same prefix
160 return self.findLinked(code, sym.next);
161 }
162
163 inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
164 var pos = start;
165 while (pos > 0) {
166 const sym = self.symbols[pos];
167 const shift = max_code_bits - sym.code_bits;
168 // compare code_bits number of upper bits
169 if ((code ^ sym.code) >> shift == 0) return sym;
170 pos = sym.next;
171 }
172 return error.InvalidCode;
173 }
174 };
175}
176
177test "init/find" {
178 // example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
179 const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
180 var h: CodegenDecoder = .{};
181 try h.generate(&code_lens);
182
183 const expected = [_]struct {
184 sym: Symbol,
185 code: u16,
186 }{
187 .{
188 .code = 0b00_00000,
189 .sym = .{ .symbol = 3, .code_bits = 2 },
190 },
191 .{
192 .code = 0b01_00000,
193 .sym = .{ .symbol = 18, .code_bits = 2 },
194 },
195 .{
196 .code = 0b100_0000,
197 .sym = .{ .symbol = 1, .code_bits = 3 },
198 },
199 .{
200 .code = 0b101_0000,
201 .sym = .{ .symbol = 4, .code_bits = 3 },
202 },
203 .{
204 .code = 0b110_0000,
205 .sym = .{ .symbol = 17, .code_bits = 3 },
206 },
207 .{
208 .code = 0b1110_000,
209 .sym = .{ .symbol = 0, .code_bits = 4 },
210 },
211 .{
212 .code = 0b1111_000,
213 .sym = .{ .symbol = 16, .code_bits = 4 },
214 },
215 };
216
217 // unused symbols
218 for (0..12) |i| {
219 try testing.expectEqual(0, h.symbols[i].code_bits);
220 }
221 // used, from index 12
222 for (expected, 12..) |e, i| {
223 try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
224 try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
225 const sym_from_code = try h.find(e.code);
226 try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
227 }
228
229 // All possible codes for each symbol.
230 // Lookup table has 126 elements, to cover all possible 7 bit codes.
231 for (0b0000_000..0b0100_000) |c| // 0..32 (32)
232 try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
233
234 for (0b0100_000..0b1000_000) |c| // 32..64 (32)
235 try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
236
237 for (0b1000_000..0b1010_000) |c| // 64..80 (16)
238 try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
239
240 for (0b1010_000..0b1100_000) |c| // 80..96 (16)
241 try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
242
243 for (0b1100_000..0b1110_000) |c| // 96..112 (16)
244 try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
245
246 for (0b1110_000..0b1111_000) |c| // 112..120 (8)
247 try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
248
249 for (0b1111_000..0b1_0000_000) |c| // 120...128 (8)
250 try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
251}
252
253test "encode/decode literals" {
254 const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder;
255
256 for (1..286) |j| { // for all different number of codes
257 var enc: LiteralEncoder = .{};
258 // create frequencies
259 var freq = [_]u16{0} ** 286;
260 freq[256] = 1; // ensure we have end of block code
261 for (&freq, 1..) |*f, i| {
262 if (i % j == 0)
263 f.* = @intCast(i);
264 }
265
266 // encoder from frequencies
267 enc.generate(&freq, 15);
268
269 // get code_lens from encoder
270 var code_lens = [_]u4{0} ** 286;
271 for (code_lens, 0..) |_, i| {
272 code_lens[i] = @intCast(enc.codes[i].len);
273 }
274 // generate decoder from code lens
275 var dec: LiteralDecoder = .{};
276 try dec.generate(&code_lens);
277
278 // expect decoder code to match original encoder code
279 for (dec.symbols) |s| {
280 if (s.code_bits == 0) continue;
281 const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
282 const symbol: u16 = switch (s.kind) {
283 .literal => s.symbol,
284 .end_of_block => 256,
285 .match => @as(u16, s.symbol) + 257,
286 };
287
288 const c = enc.codes[symbol];
289 try testing.expect(c.code == c_code);
290 }
291
292 // find each symbol by code
293 for (enc.codes) |c| {
294 if (c.len == 0) continue;
295
296 const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
297 const s = try dec.find(s_code);
298 try testing.expect(s.code == s_code);
299 try testing.expect(s.code_bits == c.len);
300 }
301 }
302}
lib/std/compress/flate/huffman_encoder.zig deleted-536
......@@ -1,536 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const math = std.math;
4const mem = std.mem;
5const sort = std.sort;
6const testing = std.testing;
7
8const consts = @import("consts.zig").huffman;
9
10const LiteralNode = struct {
11 literal: u16,
12 freq: u16,
13};
14
15// Describes the state of the constructed tree for a given depth.
16const LevelInfo = struct {
17 // Our level. for better printing
18 level: u32,
19
20 // The frequency of the last node at this level
21 last_freq: u32,
22
23 // The frequency of the next character to add to this level
24 next_char_freq: u32,
25
26 // The frequency of the next pair (from level below) to add to this level.
27 // Only valid if the "needed" value of the next lower level is 0.
28 next_pair_freq: u32,
29
30 // The number of chains remaining to generate for this level before moving
31 // up to the next level
32 needed: u32,
33};
34
35// hcode is a huffman code with a bit code and bit length.
36pub const HuffCode = struct {
37 code: u16 = 0,
38 len: u16 = 0,
39
40 // set sets the code and length of an hcode.
41 fn set(self: *HuffCode, code: u16, length: u16) void {
42 self.len = length;
43 self.code = code;
44 }
45};
46
47pub fn HuffmanEncoder(comptime size: usize) type {
48 return struct {
49 codes: [size]HuffCode = undefined,
50 // Reusable buffer with the longest possible frequency table.
51 freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined,
52 bit_count: [17]u32 = undefined,
53 lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
54 lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
55
56 const Self = @This();
57
58 // Update this Huffman Code object to be the minimum code for the specified frequency count.
59 //
60 // freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
61 // max_bits The maximum number of bits to use for any literal.
62 pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
63 var list = self.freq_cache[0 .. freq.len + 1];
64 // Number of non-zero literals
65 var count: u32 = 0;
66 // Set list to be the set of all non-zero literals and their frequencies
67 for (freq, 0..) |f, i| {
68 if (f != 0) {
69 list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
70 count += 1;
71 } else {
72 list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
73 self.codes[i].len = 0;
74 }
75 }
76 list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
77
78 list = list[0..count];
79 if (count <= 2) {
80 // Handle the small cases here, because they are awkward for the general case code. With
81 // two or fewer literals, everything has bit length 1.
82 for (list, 0..) |node, i| {
83 // "list" is in order of increasing literal value.
84 self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
85 }
86 return;
87 }
88 self.lfs = list;
89 mem.sort(LiteralNode, self.lfs, {}, byFreq);
90
91 // Get the number of literals for each bit count
92 const bit_count = self.bitCounts(list, max_bits);
93 // And do the assignment
94 self.assignEncodingAndSize(bit_count, list);
95 }
96
97 pub fn bitLength(self: *Self, freq: []u16) u32 {
98 var total: u32 = 0;
99 for (freq, 0..) |f, i| {
100 if (f != 0) {
101 total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
102 }
103 }
104 return total;
105 }
106
107 // Return the number of literals assigned to each bit size in the Huffman encoding
108 //
109 // This method is only called when list.len >= 3
110 // The cases of 0, 1, and 2 literals are handled by special case code.
111 //
112 // list: An array of the literals with non-zero frequencies
113 // and their associated frequencies. The array is in order of increasing
114 // frequency, and has as its last element a special element with frequency
115 // std.math.maxInt(i32)
116 //
117 // max_bits: The maximum number of bits that should be used to encode any literal.
118 // Must be less than 16.
119 //
120 // Returns an integer array in which array[i] indicates the number of literals
121 // that should be encoded in i bits.
122 fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
123 var max_bits = max_bits_to_use;
124 const n = list.len;
125 const max_bits_limit = 16;
126
127 assert(max_bits < max_bits_limit);
128
129 // The tree can't have greater depth than n - 1, no matter what. This
130 // saves a little bit of work in some small cases
131 max_bits = @min(max_bits, n - 1);
132
133 // Create information about each of the levels.
134 // A bogus "Level 0" whose sole purpose is so that
135 // level1.prev.needed == 0. This makes level1.next_pair_freq
136 // be a legitimate value that never gets chosen.
137 var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
138 // leaf_counts[i] counts the number of literals at the left
139 // of ancestors of the rightmost node at level i.
140 // leaf_counts[i][j] is the number of literals at the left
141 // of the level j ancestor.
142 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
143
144 {
145 var level = @as(u32, 1);
146 while (level <= max_bits) : (level += 1) {
147 // For every level, the first two items are the first two characters.
148 // We initialize the levels as if we had already figured this out.
149 levels[level] = LevelInfo{
150 .level = level,
151 .last_freq = list[1].freq,
152 .next_char_freq = list[2].freq,
153 .next_pair_freq = list[0].freq + list[1].freq,
154 .needed = 0,
155 };
156 leaf_counts[level][level] = 2;
157 if (level == 1) {
158 levels[level].next_pair_freq = math.maxInt(i32);
159 }
160 }
161 }
162
163 // We need a total of 2*n - 2 items at top level and have already generated 2.
164 levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
165
166 {
167 var level = max_bits;
168 while (true) {
169 var l = &levels[level];
170 if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
171 // We've run out of both leaves and pairs.
172 // End all calculations for this level.
173 // To make sure we never come back to this level or any lower level,
174 // set next_pair_freq impossibly large.
175 l.needed = 0;
176 levels[level + 1].next_pair_freq = math.maxInt(i32);
177 level += 1;
178 continue;
179 }
180
181 const prev_freq = l.last_freq;
182 if (l.next_char_freq < l.next_pair_freq) {
183 // The next item on this row is a leaf node.
184 const next = leaf_counts[level][level] + 1;
185 l.last_freq = l.next_char_freq;
186 // Lower leaf_counts are the same of the previous node.
187 leaf_counts[level][level] = next;
188 if (next >= list.len) {
189 l.next_char_freq = maxNode().freq;
190 } else {
191 l.next_char_freq = list[next].freq;
192 }
193 } else {
194 // The next item on this row is a pair from the previous row.
195 // next_pair_freq isn't valid until we generate two
196 // more values in the level below
197 l.last_freq = l.next_pair_freq;
198 // Take leaf counts from the lower level, except counts[level] remains the same.
199 @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
200 levels[l.level - 1].needed = 2;
201 }
202
203 l.needed -= 1;
204 if (l.needed == 0) {
205 // We've done everything we need to do for this level.
206 // Continue calculating one level up. Fill in next_pair_freq
207 // of that level with the sum of the two nodes we've just calculated on
208 // this level.
209 if (l.level == max_bits) {
210 // All done!
211 break;
212 }
213 levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
214 level += 1;
215 } else {
216 // If we stole from below, move down temporarily to replenish it.
217 while (levels[level - 1].needed > 0) {
218 level -= 1;
219 if (level == 0) {
220 break;
221 }
222 }
223 }
224 }
225 }
226
227 // Somethings is wrong if at the end, the top level is null or hasn't used
228 // all of the leaves.
229 assert(leaf_counts[max_bits][max_bits] == n);
230
231 var bit_count = self.bit_count[0 .. max_bits + 1];
232 var bits: u32 = 1;
233 const counts = &leaf_counts[max_bits];
234 {
235 var level = max_bits;
236 while (level > 0) : (level -= 1) {
237 // counts[level] gives the number of literals requiring at least "bits"
238 // bits to encode.
239 bit_count[bits] = counts[level] - counts[level - 1];
240 bits += 1;
241 if (level == 0) {
242 break;
243 }
244 }
245 }
246 return bit_count;
247 }
248
249 // Look at the leaves and assign them a bit count and an encoding as specified
250 // in RFC 1951 3.2.2
251 fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
252 var code = @as(u16, 0);
253 var list = list_arg;
254
255 for (bit_count, 0..) |bits, n| {
256 code <<= 1;
257 if (n == 0 or bits == 0) {
258 continue;
259 }
260 // The literals list[list.len-bits] .. list[list.len-bits]
261 // are encoded using "bits" bits, and get the values
262 // code, code + 1, .... The code values are
263 // assigned in literal order (not frequency order).
264 const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
265
266 self.lns = chunk;
267 mem.sort(LiteralNode, self.lns, {}, byLiteral);
268
269 for (chunk) |node| {
270 self.codes[node.literal] = HuffCode{
271 .code = bitReverse(u16, code, @as(u5, @intCast(n))),
272 .len = @as(u16, @intCast(n)),
273 };
274 code += 1;
275 }
276 list = list[0 .. list.len - @as(u32, @intCast(bits))];
277 }
278 }
279 };
280}
281
282fn maxNode() LiteralNode {
283 return LiteralNode{
284 .literal = math.maxInt(u16),
285 .freq = math.maxInt(u16),
286 };
287}
288
289pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
290 return .{};
291}
292
293pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies);
294pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count);
295pub const CodegenEncoder = HuffmanEncoder(19);
296
297// Generates a HuffmanCode corresponding to the fixed literal table
298pub fn fixedLiteralEncoder() LiteralEncoder {
299 var h: LiteralEncoder = undefined;
300 var ch: u16 = 0;
301
302 while (ch < consts.max_num_frequencies) : (ch += 1) {
303 var bits: u16 = undefined;
304 var size: u16 = undefined;
305 switch (ch) {
306 0...143 => {
307 // size 8, 000110000 .. 10111111
308 bits = ch + 48;
309 size = 8;
310 },
311 144...255 => {
312 // size 9, 110010000 .. 111111111
313 bits = ch + 400 - 144;
314 size = 9;
315 },
316 256...279 => {
317 // size 7, 0000000 .. 0010111
318 bits = ch - 256;
319 size = 7;
320 },
321 else => {
322 // size 8, 11000000 .. 11000111
323 bits = ch + 192 - 280;
324 size = 8;
325 },
326 }
327 h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
328 }
329 return h;
330}
331
332pub fn fixedDistanceEncoder() DistanceEncoder {
333 var h: DistanceEncoder = undefined;
334 for (h.codes, 0..) |_, ch| {
335 h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
336 }
337 return h;
338}
339
340pub fn huffmanDistanceEncoder() DistanceEncoder {
341 var distance_freq = [1]u16{0} ** consts.distance_code_count;
342 distance_freq[0] = 1;
343 // huff_distance is a static distance encoder used for huffman only encoding.
344 // It can be reused since we will not be encoding distance values.
345 var h: DistanceEncoder = .{};
346 h.generate(distance_freq[0..], 15);
347 return h;
348}
349
350fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
351 _ = context;
352 return a.literal < b.literal;
353}
354
355fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
356 _ = context;
357 if (a.freq == b.freq) {
358 return a.literal < b.literal;
359 }
360 return a.freq < b.freq;
361}
362
363test "generate a Huffman code from an array of frequencies" {
364 var freqs: [19]u16 = [_]u16{
365 8, // 0
366 1, // 1
367 1, // 2
368 2, // 3
369 5, // 4
370 10, // 5
371 9, // 6
372 1, // 7
373 0, // 8
374 0, // 9
375 0, // 10
376 0, // 11
377 0, // 12
378 0, // 13
379 0, // 14
380 0, // 15
381 1, // 16
382 3, // 17
383 5, // 18
384 };
385
386 var enc = huffmanEncoder(19);
387 enc.generate(freqs[0..], 7);
388
389 try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391 try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392 try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393 try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394 try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395 try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396 try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397 try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398 try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399 try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400 try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401 try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402 try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403 try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404 try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405 try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406 try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407 try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408 try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409 try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411 try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412 try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413 try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414 try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415 try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416 try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417 try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418 try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419 try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420 try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421 try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422}
423
424test "generate a Huffman code for the fixed literal table specific to Deflate" {
425 const enc = fixedLiteralEncoder();
426 for (enc.codes) |c| {
427 switch (c.len) {
428 7 => {
429 const v = @bitReverse(@as(u7, @intCast(c.code)));
430 try testing.expect(v <= 0b0010111);
431 },
432 8 => {
433 const v = @bitReverse(@as(u8, @intCast(c.code)));
434 try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
435 (v >= 0b11000000 and v <= 11000111));
436 },
437 9 => {
438 const v = @bitReverse(@as(u9, @intCast(c.code)));
439 try testing.expect(v >= 0b110010000 and v <= 0b111111111);
440 },
441 else => unreachable,
442 }
443 }
444}
445
446test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
447 const enc = fixedDistanceEncoder();
448 for (enc.codes) |c| {
449 const v = @bitReverse(@as(u5, @intCast(c.code)));
450 try testing.expect(v <= 29);
451 try testing.expect(c.len == 5);
452 }
453}
454
455// Reverse bit-by-bit a N-bit code.
456fn bitReverse(comptime T: type, value: T, n: usize) T {
457 const r = @bitReverse(value);
458 return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
459}
460
461test bitReverse {
462 const ReverseBitsTest = struct {
463 in: u16,
464 bit_count: u5,
465 out: u16,
466 };
467
468 const reverse_bits_tests = [_]ReverseBitsTest{
469 .{ .in = 1, .bit_count = 1, .out = 1 },
470 .{ .in = 1, .bit_count = 2, .out = 2 },
471 .{ .in = 1, .bit_count = 3, .out = 4 },
472 .{ .in = 1, .bit_count = 4, .out = 8 },
473 .{ .in = 1, .bit_count = 5, .out = 16 },
474 .{ .in = 17, .bit_count = 5, .out = 17 },
475 .{ .in = 257, .bit_count = 9, .out = 257 },
476 .{ .in = 29, .bit_count = 5, .out = 23 },
477 };
478
479 for (reverse_bits_tests) |h| {
480 const v = bitReverse(u16, h.in, h.bit_count);
481 try std.testing.expectEqual(h.out, v);
482 }
483}
484
485test "fixedLiteralEncoder codes" {
486 var al = std.ArrayList(u8).init(testing.allocator);
487 defer al.deinit();
488 var bw = std.io.bitWriter(.little, al.writer());
489
490 const f = fixedLiteralEncoder();
491 for (f.codes) |c| {
492 try bw.writeBits(c.code, c.len);
493 }
494 try testing.expectEqualSlices(u8, &fixed_codes, al.items);
495}
496
497pub const fixed_codes = [_]u8{
498 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
499 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
500 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
501 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
502 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
503 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
504 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
505 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
506 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
507 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
508 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
509 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
510 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
511 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
512 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
513 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
514 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
515 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
516 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
517 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
518 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
519 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
520 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
521 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
522 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
523 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
524 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
525 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
526 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
527 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
528 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
529 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
530 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
531 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
532 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
533 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
534 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
535 0b10100011,
536};
lib/std/compress/flate/inflate.zig deleted-1045
......@@ -1,1045 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const testing = std.testing;
4
5const hfd = @import("huffman_decoder.zig");
6const CircularBuffer = @import("CircularBuffer.zig");
7const Container = @import("container.zig").Container;
8const Token = @import("Token.zig");
9const codegen_order = @import("consts.zig").huffman.codegen_order;
10
11/// Decompresses deflate bit stream `reader` and writes uncompressed data to the
12/// `writer` stream.
13pub fn decompress(comptime container: Container, reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
14 var d = decompressor(container, reader);
15 try d.decompress(writer);
16}
17
18/// Inflate decompressor for the reader type.
19pub fn decompressor(comptime container: Container, reader: *std.io.BufferedReader) Decompressor(container) {
20 return Decompressor(container).init(reader);
21}
22
23pub fn Decompressor(comptime container: Container) type {
24 // zlib has 4 bytes footer, lookahead of 4 bytes ensures that we will not overshoot.
25 // gzip has 8 bytes footer so we will not overshoot even with 8 bytes of lookahead.
26 // For raw deflate there is always possibility of overshot so we use 8 bytes lookahead.
27 const lookahead: type = if (container == .zlib) u32 else u64;
28 return Inflate(container, lookahead);
29}
30
31/// Inflate decompresses deflate bit stream. Reads compressed data from reader
32/// provided in init. Decompressed data are stored in internal hist buffer and
33/// can be accesses iterable `next` or reader interface.
34///
35/// Container defines header/footer wrapper around deflate bit stream. Can be
36/// gzip or zlib.
37///
38/// Deflate bit stream consists of multiple blocks. Block can be one of three types:
39/// * stored, non compressed, max 64k in size
40/// * fixed, huffman codes are predefined
41/// * dynamic, huffman code tables are encoded at the block start
42///
43/// `step` function runs decoder until internal `hist` buffer is full. Client
44/// than needs to read that data in order to proceed with decoding.
45///
46/// Allocates 74.5K of internal buffers, most important are:
47/// * 64K for history (CircularBuffer)
48/// * ~10K huffman decoders (Literal and DistanceDecoder)
49///
50pub fn Inflate(comptime container: Container, comptime Lookahead: type) type {
51 assert(Lookahead == u32 or Lookahead == u64);
52 const LookaheadBitReader = BitReader(Lookahead);
53
54 return struct {
55 bits: LookaheadBitReader,
56 hist: CircularBuffer = .{},
57 // Hashes, produces checksum, of uncompressed data for gzip/zlib footer.
58 hasher: container.Hasher() = .{},
59
60 // dynamic block huffman code decoders
61 lit_dec: hfd.LiteralDecoder = .{}, // literals
62 dst_dec: hfd.DistanceDecoder = .{}, // distances
63
64 // current read state
65 bfinal: u1 = 0,
66 block_type: u2 = 0b11,
67 state: ReadState = .protocol_header,
68
69 read_err: Error!void = {},
70
71 const ReadState = enum {
72 protocol_header,
73 block_header,
74 block,
75 protocol_footer,
76 end,
77 };
78
79 const Self = @This();
80
81 pub const Error = Container.Error || hfd.Error || error{
82 InvalidCode,
83 InvalidMatch,
84 InvalidBlockType,
85 WrongStoredBlockNlen,
86 InvalidDynamicBlockHeader,
87 EndOfStream,
88 ReadFailed,
89 };
90
91 pub fn init(bw: *std.io.BufferedReader) Self {
92 return .{ .bits = LookaheadBitReader.init(bw) };
93 }
94
95 fn blockHeader(self: *Self) Error!void {
96 self.bfinal = try self.bits.read(u1);
97 self.block_type = try self.bits.read(u2);
98 }
99
100 fn storedBlock(self: *Self) !bool {
101 self.bits.alignToByte(); // skip padding until byte boundary
102 // everything after this is byte aligned in stored block
103 var len = try self.bits.read(u16);
104 const nlen = try self.bits.read(u16);
105 if (len != ~nlen) return error.WrongStoredBlockNlen;
106
107 while (len > 0) {
108 const buf = self.hist.getWritable(len);
109 try self.bits.readAll(buf);
110 len -= @intCast(buf.len);
111 }
112 return true;
113 }
114
115 fn fixedBlock(self: *Self) !bool {
116 while (!self.hist.full()) {
117 const code = try self.bits.readFixedCode();
118 switch (code) {
119 0...255 => self.hist.write(@intCast(code)),
120 256 => return true, // end of block
121 257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
122 else => return error.InvalidCode,
123 }
124 }
125 return false;
126 }
127
128 // Handles fixed block non literal (length) code.
129 // Length code is followed by 5 bits of distance code.
130 fn fixedDistanceCode(self: *Self, code: u8) !void {
131 try self.bits.fill(5 + 5 + 13);
132 const length = try self.decodeLength(code);
133 const distance = try self.decodeDistance(try self.bits.readF(u5, .{
134 .buffered = true,
135 .reverse = true,
136 }));
137 try self.hist.writeMatch(length, distance);
138 }
139
140 inline fn decodeLength(self: *Self, code: u8) !u16 {
141 if (code > 28) return error.InvalidCode;
142 const ml = Token.matchLength(code);
143 return if (ml.extra_bits == 0) // 0 - 5 extra bits
144 ml.base
145 else
146 ml.base + try self.bits.readN(ml.extra_bits, .{ .buffered = true });
147 }
148
149 fn decodeDistance(self: *Self, code: u8) !u16 {
150 if (code > 29) return error.InvalidCode;
151 const md = Token.matchDistance(code);
152 return if (md.extra_bits == 0) // 0 - 13 extra bits
153 md.base
154 else
155 md.base + try self.bits.readN(md.extra_bits, .{ .buffered = true });
156 }
157
158 fn dynamicBlockHeader(self: *Self) !void {
159 const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
160 const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
161 const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lengths are encoded
162
163 if (hlit > 286 or hdist > 30)
164 return error.InvalidDynamicBlockHeader;
165
166 // lengths for code lengths
167 var cl_lens = [_]u4{0} ** 19;
168 for (0..hclen) |i| {
169 cl_lens[codegen_order[i]] = try self.bits.read(u3);
170 }
171 var cl_dec: hfd.CodegenDecoder = .{};
172 try cl_dec.generate(&cl_lens);
173
174 // decoded code lengths
175 var dec_lens = [_]u4{0} ** (286 + 30);
176 var pos: usize = 0;
177 while (pos < hlit + hdist) {
178 const sym = try cl_dec.find(try self.bits.peekF(u7, .{ .reverse = true }));
179 try self.bits.shift(sym.code_bits);
180 pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
181 }
182 if (pos > hlit + hdist) {
183 return error.InvalidDynamicBlockHeader;
184 }
185
186 // literal code lengths to literal decoder
187 try self.lit_dec.generate(dec_lens[0..hlit]);
188
189 // distance code lengths to distance decoder
190 try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
191 }
192
193 // Decode code length symbol to code length. Writes decoded length into
194 // lens slice starting at position pos. Returns number of positions
195 // advanced.
196 fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize {
197 if (pos >= lens.len)
198 return error.InvalidDynamicBlockHeader;
199
200 switch (code) {
201 0...15 => {
202 // Represent code lengths of 0 - 15
203 lens[pos] = @intCast(code);
204 return 1;
205 },
206 16 => {
207 // Copy the previous code length 3 - 6 times.
208 // The next 2 bits indicate repeat length
209 const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
210 if (pos == 0 or pos + n > lens.len)
211 return error.InvalidDynamicBlockHeader;
212 for (0..n) |i| {
213 lens[pos + i] = lens[pos + i - 1];
214 }
215 return n;
216 },
217 // Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
218 17 => return @as(u8, try self.bits.read(u3)) + 3,
219 // Repeat a code length of 0 for 11 - 138 times (7 bits of length)
220 18 => return @as(u8, try self.bits.read(u7)) + 11,
221 else => return error.InvalidDynamicBlockHeader,
222 }
223 }
224
225 // In larger archives most blocks are usually dynamic, so decompression
226 // performance depends on this function.
227 fn dynamicBlock(self: *Self) !bool {
228 // Hot path loop!
229 while (!self.hist.full()) {
230 try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path)
231 const sym = try self.decodeSymbol(&self.lit_dec);
232
233 switch (sym.kind) {
234 .literal => self.hist.write(sym.symbol),
235 .match => { // Decode match backreference <length, distance>
236 // fill so we can use buffered reads
237 if (Lookahead == u32)
238 try self.bits.fill(5 + 15)
239 else
240 try self.bits.fill(5 + 15 + 13);
241 const length = try self.decodeLength(sym.symbol);
242 const dsm = try self.decodeSymbol(&self.dst_dec);
243 if (Lookahead == u32) try self.bits.fill(13);
244 const distance = try self.decodeDistance(dsm.symbol);
245 try self.hist.writeMatch(length, distance);
246 },
247 .end_of_block => return true,
248 }
249 }
250 return false;
251 }
252
253 // Peek 15 bits from bits reader (maximum code len is 15 bits). Use
254 // decoder to find symbol for that code. We then know how many bits is
255 // used. Shift bit reader for that much bits, those bits are used. And
256 // return symbol.
257 fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
258 const sym = try decoder.find(try self.bits.peekF(u15, .{ .buffered = true, .reverse = true }));
259 try self.bits.shift(sym.code_bits);
260 return sym;
261 }
262
263 fn step(self: *Self) !void {
264 switch (self.state) {
265 .protocol_header => {
266 try container.parseHeader(&self.bits);
267 self.state = .block_header;
268 },
269 .block_header => {
270 try self.blockHeader();
271 self.state = .block;
272 if (self.block_type == 2) try self.dynamicBlockHeader();
273 },
274 .block => {
275 const done = switch (self.block_type) {
276 0 => try self.storedBlock(),
277 1 => try self.fixedBlock(),
278 2 => try self.dynamicBlock(),
279 else => return error.InvalidBlockType,
280 };
281 if (done) {
282 self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
283 }
284 },
285 .protocol_footer => {
286 self.bits.alignToByte();
287 try container.parseFooter(&self.hasher, &self.bits);
288 self.state = .end;
289 },
290 .end => {},
291 }
292 }
293
294 /// Replaces the inner reader with new reader.
295 pub fn setReader(self: *Self, new_reader: *std.io.BufferedReader) void {
296 self.bits.forward_reader = new_reader;
297 if (self.state == .end or self.state == .protocol_footer) {
298 self.state = .protocol_header;
299 }
300 }
301
302 // Reads all compressed data from the internal reader and outputs plain
303 // (uncompressed) data to the provided writer.
304 pub fn decompress(self: *Self, writer: *std.io.BufferedWriter) !void {
305 while (try self.next()) |buf| {
306 try writer.writeAll(buf);
307 }
308 }
309
310 /// Returns the number of bytes that have been read from the internal
311 /// reader but not yet consumed by the decompressor.
312 pub fn unreadBytes(self: Self) usize {
313 // There can be no error here: the denominator is not zero, and
314 // overflow is not possible since the type is unsigned.
315 return std.math.divCeil(usize, self.bits.nbits, 8) catch unreachable;
316 }
317
318 // Iterator interface
319
320 /// Can be used in iterator like loop without memcpy to another buffer:
321 /// while (try inflate.next()) |buf| { ... }
322 pub fn next(self: *Self) Error!?[]const u8 {
323 const out = try self.get(0);
324 if (out.len == 0) return null;
325 return out;
326 }
327
328 /// Returns decompressed data from internal sliding window buffer.
329 /// Returned buffer can be any length between 0 and `limit` bytes. 0
330 /// returned bytes means end of stream reached. With limit=0 returns as
331 /// much data it can. It newer will be more than 65536 bytes, which is
332 /// size of internal buffer.
333 /// TODO merge this logic into readerRead and readerReadVec
334 pub fn get(self: *Self, limit: usize) Error![]const u8 {
335 while (true) {
336 const out = self.hist.readAtMost(limit);
337 if (out.len > 0) {
338 self.hasher.update(out);
339 return out;
340 }
341 if (self.state == .end) return out;
342 try self.step();
343 }
344 }
345
346 fn readerRead(
347 context: ?*anyopaque,
348 bw: *std.io.BufferedWriter,
349 limit: std.io.Reader.Limit,
350 ) std.io.Reader.RwError!usize {
351 const self: *Self = @alignCast(@ptrCast(context));
352 const out = try bw.writableSliceGreedy(1);
353 const in = self.get(limit.minInt(out.len)) catch |err| switch (err) {
354 error.EndOfStream => return error.EndOfStream,
355 error.ReadFailed => return error.ReadFailed,
356 else => |e| {
357 self.read_err = e;
358 return error.ReadFailed;
359 },
360 };
361 if (in.len == 0) return error.EndOfStream;
362 @memcpy(out[0..in.len], in);
363 bw.advance(in.len);
364 return in.len;
365 }
366
367 fn readerReadVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
368 const self: *Self = @alignCast(@ptrCast(context));
369 return readVec(self, data) catch |err| switch (err) {
370 error.EndOfStream => return error.EndOfStream,
371 error.ReadFailed => return error.ReadFailed,
372 else => |e| {
373 self.read_err = e;
374 return error.ReadFailed;
375 },
376 };
377 }
378
379 fn readerDiscard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
380 _ = context;
381 _ = limit;
382 @panic("TODO");
383 }
384
385 pub fn readVec(self: *Self, data: []const []u8) Error!usize {
386 for (data) |out| {
387 if (out.len == 0) continue;
388 const in = try self.get(out.len);
389 @memcpy(out[0..in.len], in);
390 if (in.len == 0) return error.EndOfStream;
391 return in.len;
392 }
393 return 0;
394 }
395
396 pub fn reader(self: *Self) std.io.Reader {
397 return .{
398 .context = self,
399 .vtable = &.{
400 .read = readerRead,
401 .readVec = readerReadVec,
402 .discard = readerDiscard,
403 },
404 };
405 }
406
407 pub fn readable(self: *Self, buffer: []u8) std.io.BufferedReader {
408 return reader(self).buffered(buffer);
409 }
410 };
411}
412
413test "decompress" {
414 const cases = [_]struct {
415 in: []const u8,
416 out: []const u8,
417 }{
418 // non compressed block (type 0)
419 .{
420 .in = &[_]u8{
421 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
422 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
423 },
424 .out = "Hello world\n",
425 },
426 // fixed code block (type 1)
427 .{
428 .in = &[_]u8{
429 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
430 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
431 },
432 .out = "Hello world\n",
433 },
434 // dynamic block (type 2)
435 .{
436 .in = &[_]u8{
437 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
438 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
439 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
440 },
441 .out = "ABCDEABCD ABCDEABCD",
442 },
443 };
444 for (cases) |c| {
445 var fb = std.io.fixedBufferStream(c.in);
446 var al = std.ArrayList(u8).init(testing.allocator);
447 defer al.deinit();
448
449 try decompress(.raw, fb.reader(), al.writer());
450 try testing.expectEqualStrings(c.out, al.items);
451 }
452}
453
454test "gzip decompress" {
455 const cases = [_]struct {
456 in: []const u8,
457 out: []const u8,
458 }{
459 // non compressed block (type 0)
460 .{
461 .in = &[_]u8{
462 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
463 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
464 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
465 0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
466 0x0c, 0x00, 0x00, 0x00, // gzip footer: size
467 },
468 .out = "Hello world\n",
469 },
470 // fixed code block (type 1)
471 .{
472 .in = &[_]u8{
473 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
474 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
475 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
476 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
477 },
478 .out = "Hello world\n",
479 },
480 // dynamic block (type 2)
481 .{
482 .in = &[_]u8{
483 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
484 0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
485 0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
486 0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
487 0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
488 },
489 .out = "ABCDEABCD ABCDEABCD",
490 },
491 // gzip header with name
492 .{
493 .in = &[_]u8{
494 0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
495 0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
496 0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
497 },
498 .out = "Hello world\n",
499 },
500 };
501 for (cases) |c| {
502 var fb = std.io.fixedBufferStream(c.in);
503 var al = std.ArrayList(u8).init(testing.allocator);
504 defer al.deinit();
505
506 try decompress(.gzip, fb.reader(), al.writer());
507 try testing.expectEqualStrings(c.out, al.items);
508 }
509}
510
511test "zlib decompress" {
512 const cases = [_]struct {
513 in: []const u8,
514 out: []const u8,
515 }{
516 // non compressed block (type 0)
517 .{
518 .in = &[_]u8{
519 0x78, 0b10_0_11100, // zlib header (2 bytes)
520 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
521 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
522 0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
523 },
524 .out = "Hello world\n",
525 },
526 };
527 for (cases) |c| {
528 var fb = std.io.fixedBufferStream(c.in);
529 var al = std.ArrayList(u8).init(testing.allocator);
530 defer al.deinit();
531
532 try decompress(.zlib, fb.reader(), al.writer());
533 try testing.expectEqualStrings(c.out, al.items);
534 }
535}
536
537test "fuzzing tests" {
538 const cases = [_]struct {
539 input: []const u8,
540 out: []const u8 = "",
541 err: ?anyerror = null,
542 }{
543 .{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
544 .{ .input = "empty-distance-alphabet01" },
545 .{ .input = "empty-distance-alphabet02" },
546 .{ .input = "end-of-stream", .err = error.EndOfStream },
547 .{ .input = "invalid-distance", .err = error.InvalidMatch },
548 .{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
549 .{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
550 .{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
551 .{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
552 .{ .input = "out-of-codes", .err = error.InvalidCode },
553 .{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
554 .{ .input = "puff02", .err = error.EndOfStream },
555 .{ .input = "puff03", .out = &[_]u8{0xa} },
556 .{ .input = "puff04", .err = error.InvalidCode },
557 .{ .input = "puff05", .err = error.EndOfStream },
558 .{ .input = "puff06", .err = error.EndOfStream },
559 .{ .input = "puff08", .err = error.InvalidCode },
560 .{ .input = "puff09", .out = "P" },
561 .{ .input = "puff10", .err = error.InvalidCode },
562 .{ .input = "puff11", .err = error.InvalidMatch },
563 .{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
564 .{ .input = "puff13", .err = error.IncompleteHuffmanTree },
565 .{ .input = "puff14", .err = error.EndOfStream },
566 .{ .input = "puff15", .err = error.IncompleteHuffmanTree },
567 .{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
568 .{ .input = "puff17", .err = error.MissingEndOfBlockCode }, // 25
569 .{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
570 .{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
571 .{ .input = "fuzz3", .err = error.InvalidMatch },
572 .{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
573 .{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
574 .{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
575 .{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
576 .{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
577 .{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
578 .{ .input = "puff23", .err = error.OversubscribedHuffmanTree }, // 35
579 .{ .input = "puff24", .err = error.IncompleteHuffmanTree },
580 .{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
581 .{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
582 .{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
583 };
584
585 inline for (cases, 0..) |c, case_no| {
586 var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
587 var out = std.ArrayList(u8).init(testing.allocator);
588 defer out.deinit();
589 errdefer std.debug.print("test case failed {}\n", .{case_no});
590
591 if (c.err) |expected_err| {
592 try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
593 } else {
594 try decompress(.raw, in.reader(), out.writer());
595 try testing.expectEqualStrings(c.out, out.items);
596 }
597 }
598}
599
600test "bug 18966" {
601 const input = @embedFile("testdata/fuzz/bug_18966.input");
602 const expect = @embedFile("testdata/fuzz/bug_18966.expect");
603
604 var in = std.io.fixedBufferStream(input);
605 var out = std.ArrayList(u8).init(testing.allocator);
606 defer out.deinit();
607
608 try decompress(.gzip, in.reader(), out.writer());
609 try testing.expectEqualStrings(expect, out.items);
610}
611
612test "bug 19895" {
613 const input = &[_]u8{
614 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
615 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
616 };
617 var in = std.io.fixedBufferStream(input);
618 var decomp = decompressor(.raw, in.reader());
619 var buf: [0]u8 = undefined;
620 try testing.expectEqual(0, try decomp.read(&buf));
621}
622
623/// Bit reader used during inflate (decompression). Has internal buffer of 64
624/// bits which shifts right after bits are consumed. Uses forward_reader to fill
625/// that internal buffer when needed.
626///
627/// readF is the core function. Supports few different ways of getting bits
628/// controlled by flags. In hot path we try to avoid checking whether we need to
629/// fill buffer from forward_reader by calling fill in advance and readF with
630/// buffered flag set.
631///
632pub fn BitReader(comptime T: type) type {
633 assert(T == u32 or T == u64);
634 const t_bytes: usize = @sizeOf(T);
635 const Tshift = if (T == u64) u6 else u5;
636
637 return struct {
638 // Underlying reader used for filling internal bits buffer
639 forward_reader: *std.io.BufferedReader,
640 // Internal buffer of 64 bits
641 bits: T = 0,
642 // Number of bits in the buffer
643 nbits: u32 = 0,
644
645 const Self = @This();
646
647 pub const Flags = packed struct(u3) {
648 /// dont advance internal buffer, just get bits, leave them in buffer
649 peek: bool = false,
650 /// assume that there is no need to fill, fill should be called before
651 buffered: bool = false,
652 /// bit reverse read bits
653 reverse: bool = false,
654
655 /// work around https://github.com/ziglang/zig/issues/18882
656 pub inline fn toInt(f: Flags) u3 {
657 return @bitCast(f);
658 }
659 };
660
661 pub fn init(forward_reader: *std.io.BufferedReader) Self {
662 var self = Self{ .forward_reader = forward_reader };
663 self.fill(1) catch {};
664 return self;
665 }
666
667 /// Try to have `nice` bits are available in buffer. Reads from
668 /// forward reader if there is no `nice` bits in buffer. Returns error
669 /// if end of forward stream is reached and internal buffer is empty.
670 /// It will not error if less than `nice` bits are in buffer, only when
671 /// all bits are exhausted. During inflate we usually know what is the
672 /// maximum bits for the next step but usually that step will need less
673 /// bits to decode. So `nice` is not hard limit, it will just try to have
674 /// that number of bits available. If end of forward stream is reached
675 /// it may be some extra zero bits in buffer.
676 pub fn fill(self: *Self, nice: u6) !void {
677 if (self.nbits >= nice and nice != 0) {
678 return; // We have enough bits
679 }
680 // Read more bits from forward reader
681
682 // Number of empty bytes in bits, round nbits to whole bytes.
683 const empty_bytes =
684 @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
685 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
686
687 var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
688 const bytes_read = self.forward_reader.readSliceShort(buf[0..empty_bytes]) catch 0;
689 if (bytes_read > 0) {
690 const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
691 self.bits |= u << @as(Tshift, @intCast(self.nbits));
692 self.nbits += 8 * @as(u8, @intCast(bytes_read));
693 return;
694 }
695
696 if (self.nbits == 0)
697 return error.EndOfStream;
698 }
699
700 /// Read exactly buf.len bytes into buf.
701 pub fn readAll(self: *Self, buf: []u8) std.io.Reader.Error!void {
702 assert(self.alignBits() == 0); // internal bits must be at byte boundary
703
704 // First read from internal bits buffer.
705 var n: usize = 0;
706 while (self.nbits > 0 and n < buf.len) {
707 buf[n] = try self.readF(u8, .{ .buffered = true });
708 n += 1;
709 }
710 // Then use forward reader for all other bytes.
711 try self.forward_reader.readSlice(buf[n..]);
712 }
713
714 /// Alias for readF(U, 0).
715 pub fn read(self: *Self, comptime U: type) !U {
716 return self.readF(U, .{});
717 }
718
719 /// Alias for readF with flag.peak set.
720 pub inline fn peekF(self: *Self, comptime U: type, comptime how: Flags) !U {
721 return self.readF(U, .{
722 .peek = true,
723 .buffered = how.buffered,
724 .reverse = how.reverse,
725 });
726 }
727
728 /// Read with flags provided.
729 pub fn readF(self: *Self, comptime U: type, comptime how: Flags) !U {
730 if (U == T) {
731 assert(how.toInt() == 0);
732 assert(self.alignBits() == 0);
733 try self.fill(@bitSizeOf(T));
734 if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
735 const v = self.bits;
736 self.nbits = 0;
737 self.bits = 0;
738 return v;
739 }
740 const n: Tshift = @bitSizeOf(U);
741 // work around https://github.com/ziglang/zig/issues/18882
742 switch (how.toInt()) {
743 @as(Flags, .{}).toInt() => { // `normal` read
744 try self.fill(n); // ensure that there are n bits in the buffer
745 const u: U = @truncate(self.bits); // get n bits
746 try self.shift(n); // advance buffer for n
747 return u;
748 },
749 @as(Flags, .{ .peek = true }).toInt() => { // no shift, leave bits in the buffer
750 try self.fill(n);
751 return @truncate(self.bits);
752 },
753 @as(Flags, .{ .buffered = true }).toInt() => { // no fill, assume that buffer has enough bits
754 const u: U = @truncate(self.bits);
755 try self.shift(n);
756 return u;
757 },
758 @as(Flags, .{ .reverse = true }).toInt() => { // same as 0 with bit reverse
759 try self.fill(n);
760 const u: U = @truncate(self.bits);
761 try self.shift(n);
762 return @bitReverse(u);
763 },
764 @as(Flags, .{ .peek = true, .reverse = true }).toInt() => {
765 try self.fill(n);
766 return @bitReverse(@as(U, @truncate(self.bits)));
767 },
768 @as(Flags, .{ .buffered = true, .reverse = true }).toInt() => {
769 const u: U = @truncate(self.bits);
770 try self.shift(n);
771 return @bitReverse(u);
772 },
773 @as(Flags, .{ .peek = true, .buffered = true }).toInt() => {
774 return @truncate(self.bits);
775 },
776 @as(Flags, .{ .peek = true, .buffered = true, .reverse = true }).toInt() => {
777 return @bitReverse(@as(U, @truncate(self.bits)));
778 },
779 }
780 }
781
782 /// Read n number of bits.
783 /// Only buffered flag can be used in how.
784 pub fn readN(self: *Self, n: u4, comptime how: Flags) !u16 {
785 // work around https://github.com/ziglang/zig/issues/18882
786 switch (how.toInt()) {
787 @as(Flags, .{}).toInt() => {
788 try self.fill(n);
789 },
790 @as(Flags, .{ .buffered = true }).toInt() => {},
791 else => unreachable,
792 }
793 const mask: u16 = (@as(u16, 1) << n) - 1;
794 const u: u16 = @as(u16, @truncate(self.bits)) & mask;
795 try self.shift(n);
796 return u;
797 }
798
799 /// Advance buffer for n bits.
800 pub fn shift(self: *Self, n: Tshift) !void {
801 if (n > self.nbits) return error.EndOfStream;
802 self.bits >>= n;
803 self.nbits -= n;
804 }
805
806 /// Skip n bytes.
807 pub fn skipBytes(self: *Self, n: u16) !void {
808 for (0..n) |_| {
809 try self.fill(8);
810 try self.shift(8);
811 }
812 }
813
814 // Number of bits to align stream to the byte boundary.
815 fn alignBits(self: *Self) u3 {
816 return @intCast(self.nbits & 0x7);
817 }
818
819 /// Align stream to the byte boundary.
820 pub fn alignToByte(self: *Self) void {
821 const ab = self.alignBits();
822 if (ab > 0) self.shift(ab) catch unreachable;
823 }
824
825 /// Skip zero terminated string.
826 pub fn skipStringZ(self: *Self) !void {
827 while (true) {
828 if (try self.readF(u8, .{}) == 0) break;
829 }
830 }
831
832 /// Read deflate fixed fixed code.
833 /// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
834 /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
835 /// Lit Value Bits Codes
836 /// --------- ---- -----
837 /// 0 - 143 8 00110000 through
838 /// 10111111
839 /// 144 - 255 9 110010000 through
840 /// 111111111
841 /// 256 - 279 7 0000000 through
842 /// 0010111
843 /// 280 - 287 8 11000000 through
844 /// 11000111
845 pub fn readFixedCode(self: *Self) !u16 {
846 try self.fill(7 + 2);
847 const code7 = try self.readF(u7, .{ .buffered = true, .reverse = true });
848 if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
849 return @as(u16, code7) + 256;
850 } else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
851 return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, .{ .buffered = true })) - 0b0011_0000;
852 } else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
853 return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, .{ .buffered = true }) + 280;
854 } else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
855 return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, .{ .buffered = true, .reverse = true })) + 144;
856 }
857 }
858 };
859}
860
861test "readF" {
862 var input: std.io.BufferedReader = undefined;
863 input.initFixed(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
864 var br: BitReader(u64) = .init(&input);
865
866 try testing.expectEqual(@as(u8, 48), br.nbits);
867 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
868
869 try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
870 try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
871 try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
872 try testing.expectEqual(@as(u3, 5), br.alignBits());
873
874 try testing.expect(try br.readF(u8, .{ .peek = true }) == 0b0001_1110);
875 try testing.expect(try br.readF(u9, .{ .peek = true }) == 0b1_0001_1110);
876 try br.shift(9);
877 try testing.expectEqual(@as(u8, 36), br.nbits);
878 try testing.expectEqual(@as(u3, 4), br.alignBits());
879
880 try testing.expect(try br.readF(u4, 0) == 0b0100);
881 try testing.expectEqual(@as(u8, 32), br.nbits);
882 try testing.expectEqual(@as(u3, 0), br.alignBits());
883
884 try br.shift(1);
885 try testing.expectEqual(@as(u3, 7), br.alignBits());
886 try br.shift(1);
887 try testing.expectEqual(@as(u3, 6), br.alignBits());
888 br.alignToByte();
889 try testing.expectEqual(@as(u3, 0), br.alignBits());
890
891 try testing.expectEqual(@as(u64, 0xc9), br.bits);
892 try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
893 try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
894}
895
896test "read block type 1 data" {
897 inline for ([_]type{ u64, u32 }) |T| {
898 const data = [_]u8{
899 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
900 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
901 0x0c, 0x01, 0x02, 0x03, //
902 0xaa, 0xbb, 0xcc, 0xdd,
903 };
904 var fbs: std.io.BufferedReader = undefined;
905 fbs.initFixed(&data);
906 var br: BitReader(T) = .init(&fbs);
907
908 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
909 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
910
911 for ("Hello world\n") |c| {
912 try testing.expectEqual(@as(u8, c), try br.readF(u8, .{ .reverse = true }) - 0x30);
913 }
914 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
915 br.alignToByte();
916 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
917 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
918 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
919 }
920}
921
922test "shift/fill" {
923 const data = [_]u8{
924 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
925 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
926 };
927 var fbs: std.io.BufferedReader = undefined;
928 fbs.initFixed(&data);
929 var br: BitReader(u64) = .init(&fbs);
930
931 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
932 try br.shift(8);
933 try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
934 try br.fill(60); // fill with 1 byte
935 try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
936 try br.shift(8 * 4 + 4);
937 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
938
939 try br.fill(60); // fill with 4 bytes (shift by 4)
940 try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
941 try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
942
943 try br.shift(@intCast(br.nbits)); // clear buffer
944 try br.fill(8); // refill with the rest of the bytes
945 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
946}
947
948test "readAll" {
949 inline for ([_]type{ u64, u32 }) |T| {
950 const data = [_]u8{
951 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
952 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
953 };
954 var fbs: std.io.BufferedReader = undefined;
955 fbs.initFixed(&data);
956 var br: BitReader(T) = .init(&fbs);
957
958 switch (T) {
959 u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
960 u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
961 else => unreachable,
962 }
963
964 var out: [16]u8 = undefined;
965 try br.readAll(out[0..]);
966 try testing.expect(br.nbits == 0);
967 try testing.expect(br.bits == 0);
968
969 try testing.expectEqualSlices(u8, data[0..16], &out);
970 }
971}
972
973test "readFixedCode" {
974 inline for ([_]type{ u64, u32 }) |T| {
975 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
976
977 var fbs: std.io.BufferedReader = undefined;
978 fbs.initFixed(&fixed_codes);
979 var rdr: BitReader(T) = .init(&fbs);
980
981 for (0..286) |c| {
982 try testing.expectEqual(c, try rdr.readFixedCode());
983 }
984 try testing.expect(rdr.nbits == 0);
985 }
986}
987
988test "u32 leaves no bits on u32 reads" {
989 const data = [_]u8{
990 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
991 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
992 };
993 var fbs: std.io.BufferedReader = undefined;
994 fbs.initFixed(&data);
995 var br: BitReader(u32) = .init(&fbs);
996
997 _ = try br.read(u3);
998 try testing.expectEqual(29, br.nbits);
999 br.alignToByte();
1000 try testing.expectEqual(24, br.nbits);
1001 try testing.expectEqual(0x04_03_02_01, try br.read(u32));
1002 try testing.expectEqual(0, br.nbits);
1003 try testing.expectEqual(0x08_07_06_05, try br.read(u32));
1004 try testing.expectEqual(0, br.nbits);
1005
1006 _ = try br.read(u9);
1007 try testing.expectEqual(23, br.nbits);
1008 br.alignToByte();
1009 try testing.expectEqual(16, br.nbits);
1010 try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
1011 try testing.expectEqual(0, br.nbits);
1012}
1013
1014test "u64 need fill after alignToByte" {
1015 const data = [_]u8{
1016 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
1017 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
1018 };
1019
1020 // without fill
1021 var fbs: std.io.BufferedReader = undefined;
1022 fbs.initFixed(&data);
1023 var br: BitReader(u64) = .init(&fbs);
1024 _ = try br.read(u23);
1025 try testing.expectEqual(41, br.nbits);
1026 br.alignToByte();
1027 try testing.expectEqual(40, br.nbits);
1028 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
1029 try testing.expectEqual(8, br.nbits);
1030 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
1031 try testing.expectEqual(32, br.nbits);
1032
1033 // fill after align ensures all bits filled
1034 fbs.reset();
1035 br = .init(&fbs);
1036 _ = try br.read(u23);
1037 try testing.expectEqual(41, br.nbits);
1038 br.alignToByte();
1039 try br.fill(0);
1040 try testing.expectEqual(64, br.nbits);
1041 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
1042 try testing.expectEqual(32, br.nbits);
1043 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
1044 try testing.expectEqual(0, br.nbits);
1045}
lib/std/compress/gzip.zig deleted-39
......@@ -1,39 +0,0 @@
1const std = @import("../std.zig");
2const deflate = @import("flate/deflate.zig");
3const inflate = @import("flate/inflate.zig");
4
5/// Decompress compressed data from reader and write plain data to the writer.
6pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
7 try inflate.decompress(.gzip, reader, writer);
8}
9
10pub const Decompressor = inflate.Decompressor(.gzip);
11
12/// Compression level, trades between speed and compression size.
13pub const Options = deflate.Options;
14
15/// Compress plain data from reader and write compressed data to the writer.
16pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter, options: Options) !void {
17 try deflate.compress(.gzip, reader, writer, options);
18}
19
20pub const Compressor = deflate.Compressor(.gzip);
21
22/// Huffman only compression. Without Lempel-Ziv match searching. Faster
23/// compression, less memory requirements but bigger compressed sizes.
24pub const huffman = struct {
25 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
26 try deflate.huffman.compress(.gzip, reader, writer);
27 }
28
29 pub const Compressor = deflate.huffman.Compressor(.gzip);
30};
31
32// No compression store only. Compressed size is slightly bigger than plain.
33pub const store = struct {
34 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
35 try deflate.store.compress(.gzip, reader, writer);
36 }
37
38 pub const Compressor = deflate.store.Compressor(.gzip);
39};
lib/std/compress/zlib.zig deleted-77
......@@ -1,77 +0,0 @@
1const std = @import("../std.zig");
2const deflate = @import("flate/deflate.zig");
3const inflate = @import("flate/inflate.zig");
4
5/// When decompressing, the output buffer is used as the history window, so
6/// less than this may result in failure to decompress streams that were
7/// compressed with a larger window.
8pub const max_window_len = std.compress.flate.max_window_len;
9
10/// Decompress compressed data from reader and write plain data to the writer.
11pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
12 try inflate.decompress(.zlib, reader, writer);
13}
14
15pub const Decompressor = inflate.Decompressor(.zlib);
16
17/// Compression level, trades between speed and compression size.
18pub const Options = deflate.Options;
19
20/// Compress plain data from reader and write compressed data to the writer.
21pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter, options: Options) !void {
22 try deflate.compress(.zlib, reader, writer, options);
23}
24
25pub const Compressor = deflate.Compressor(.zlib);
26
27/// Huffman only compression. Without Lempel-Ziv match searching. Faster
28/// compression, less memory requirements but bigger compressed sizes.
29pub const huffman = struct {
30 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
31 try deflate.huffman.compress(.zlib, reader, writer);
32 }
33
34 pub const Compressor = deflate.huffman.Compressor(.zlib);
35};
36
37// No compression store only. Compressed size is slightly bigger than plain.
38pub const store = struct {
39 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
40 try deflate.store.compress(.zlib, reader, writer);
41 }
42
43 pub const Compressor = deflate.store.Compressor(.zlib);
44};
45
46test "should not overshoot" {
47 // Compressed zlib data with extra 4 bytes at the end.
48 const data = [_]u8{
49 0x78, 0x9c, 0x73, 0xce, 0x2f, 0xa8, 0x2c, 0xca, 0x4c, 0xcf, 0x28, 0x51, 0x08, 0xcf, 0xcc, 0xc9,
50 0x49, 0xcd, 0x55, 0x28, 0x4b, 0xcc, 0x53, 0x08, 0x4e, 0xce, 0x48, 0xcc, 0xcc, 0xd6, 0x51, 0x08,
51 0xce, 0xcc, 0x4b, 0x4f, 0x2c, 0xc8, 0x2f, 0x4a, 0x55, 0x30, 0xb4, 0xb4, 0x34, 0xd5, 0xb5, 0x34,
52 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
53 };
54
55 var stream = std.io.fixedBufferStream(data[0..]);
56 const reader = stream.reader();
57
58 var dcp = Decompressor.init(reader);
59 var out: [128]u8 = undefined;
60
61 // Decompress
62 var n = try dcp.reader().readAll(out[0..]);
63
64 // Expected decompressed data
65 try std.testing.expectEqual(46, n);
66 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
67
68 // Decompressor don't overshoot underlying reader.
69 // It is leaving it at the end of compressed data chunk.
70 try std.testing.expectEqual(data.len - 4, stream.getPos());
71 try std.testing.expectEqual(0, dcp.unreadBytes());
72
73 // 4 bytes after compressed chunk are available in reader.
74 n = try reader.readAll(out[0..]);
75 try std.testing.expectEqual(n, 4);
76 try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
77}