Commit d645114f7e3cbbfa19bb47ace2671fcac75e32ca | zig

build.zig+1

...	@@ -151,6 +151,7 @@ pub fn build(b: *std.Build) !void {	...	@@ -151,6 +151,7 @@ pub fn build(b: *std.Build) !void {
151	"rfc1952.txt",	151	"rfc1952.txt",
152	"rfc8478.txt",	152	"rfc8478.txt",
153	// exclude files from lib/std/compress/deflate/testdata	153	// exclude files from lib/std/compress/deflate/testdata
		154	// and lib/std/compress/flate/testdata
154	".expect",	155	".expect",
155	".expect-noinput",	156	".expect-noinput",
156	".golden",	157	".golden",

lib/std/compress.zig+17-6

 @@ -1,13 +1,21 @@
 const std = @import("std.zig");
-pub const deflate = @import("compress/deflate.zig");
-pub const gzip = @import("compress/gzip.zig");
 pub const lzma = @import("compress/lzma.zig");
 pub const lzma2 = @import("compress/lzma2.zig");
 pub const xz = @import("compress/xz.zig");
-pub const zlib = @import("compress/zlib.zig");
 pub const zstd = @import("compress/zstandard.zig");
+pub const flate = @import("compress/flate/root.zig").flate;
+pub const gzip = @import("compress/flate/root.zig").gzip;
+pub const zlib = @import("compress/flate/root.zig").zlib;
+// Version 1 interface
+pub const v1 = struct {
+    pub const deflate = @import("compress/deflate.zig");
+    pub const gzip = @import("compress/gzip.zig");
+    pub const zlib = @import("compress/zlib.zig");
+};
 pub fn HashedReader(
     comptime ReaderType: anytype,
     comptime HasherType: anytype,
 @@ -69,11 +77,14 @@ pub fn hashedWriter(
 }
 test {
-    _ = deflate;
+    _ = v1.deflate;
-    _ = gzip;
+    _ = v1.gzip;
     _ = lzma;
     _ = lzma2;
     _ = xz;
-    _ = zlib;
+    _ = v1.zlib;
     _ = zstd;
+    _ = flate;
+    _ = gzip;
+    _ = zlib;
 }

lib/std/compress/deflate/compressor.zig+2-2

...	@@ -327,7 +327,7 @@ pub fn Compressor(comptime WriterType: anytype) type {	...	@@ -327,7 +327,7 @@ pub fn Compressor(comptime WriterType: anytype) type {
327	}	327	}
328	}	328	}
329	}	329	}
330	const n = std.compress.deflate.copy(self.window[self.window_end..], b);	330	const n = std.compress.v1.deflate.copy(self.window[self.window_end..], b);
331	self.window_end += n;	331	self.window_end += n;
332	return @as(u32, @intCast(n));	332	return @as(u32, @intCast(n));
333	}	333	}
...	@@ -705,7 +705,7 @@ pub fn Compressor(comptime WriterType: anytype) type {	...	@@ -705,7 +705,7 @@ pub fn Compressor(comptime WriterType: anytype) type {
705	}	705	}
706		706
707	fn fillStore(self: *Self, b: []const u8) u32 {	707	fn fillStore(self: *Self, b: []const u8) u32 {
708	const n = std.compress.deflate.copy(self.window[self.window_end..], b);	708	const n = std.compress.v1.deflate.copy(self.window[self.window_end..], b);
709	self.window_end += n;	709	self.window_end += n;
710	return @as(u32, @intCast(n));	710	return @as(u32, @intCast(n));
711	}	711	}

lib/std/compress/deflate/decompressor.zig+1-1

...	@@ -450,7 +450,7 @@ pub fn Decompressor(comptime ReaderType: type) type {	...	@@ -450,7 +450,7 @@ pub fn Decompressor(comptime ReaderType: type) type {
450	pub fn read(self: *Self, output: []u8) Error!usize {	450	pub fn read(self: *Self, output: []u8) Error!usize {
451	while (true) {	451	while (true) {
452	if (self.to_read.len > 0) {	452	if (self.to_read.len > 0) {
453	const n = std.compress.deflate.copy(output, self.to_read);	453	const n = std.compress.v1.deflate.copy(output, self.to_read);
454	self.to_read = self.to_read[n..];	454	self.to_read = self.to_read[n..];
455	if (self.to_read.len == 0 and	455	if (self.to_read.len == 0 and
456	self.err != null)	456	self.err != null)

lib/std/compress/flate/CircularBuffer.zig created+234

...	@@ -0,0 +1,234 @@
1	/// 64K buffer of uncompressed data created in inflate (decompression). Has enough
2	/// history to support writing match<length, distance>; copying length of bytes
3	/// from the position distance backward from current.
4	///
5	/// Reads can return less than available bytes if they are spread across
6	/// different circles. So reads should repeat until get required number of bytes
7	/// or until returned slice is zero length.
8	///
9	/// Note on deflate limits:
10	/// * non-compressible block is limited to 65,535 bytes.
11	/// * backward pointer is limited in distance to 32K bytes and in length to 258 bytes.
12	///
13	/// Whole non-compressed block can be written without overlap. We always have
14	/// history of up to 64K, more then 32K needed.
15	///
16	const std = @import("std");
17	const assert = std.debug.assert;
18	const testing = std.testing;
19
20	const consts = @import("consts.zig").match;
21
22	const mask = 0xffff; // 64K - 1
23	const buffer_len = mask + 1; // 64K buffer
24
25	const Self = @This();
26
27	buffer: [buffer_len]u8 = undefined,
28	wp: usize = 0, // write position
29	rp: usize = 0, // read position
30
31	fn writeAll(self: *Self, buf: []const u8) void {
32	for (buf) \|c\| self.write(c);
33	}
34
35	// Write literal.
36	pub fn write(self: *Self, b: u8) void {
37	assert(self.wp - self.rp < mask);
38	self.buffer[self.wp & mask] = b;
39	self.wp += 1;
40	}
41
42	// Write match (back-reference to the same data slice) starting at `distance`
43	// back from current write position, and `length` of bytes.
44	pub fn writeMatch(self: *Self, length: u16, distance: u16) !void {
45	if (self.wp < distance or
46	length < consts.base_length or length > consts.max_length or
47	distance < consts.min_distance or distance > consts.max_distance)
48	{
49	return error.InvalidMatch;
50	}
51	assert(self.wp - self.rp < mask);
52
53	var from: usize = self.wp - distance;
54	const from_end: usize = from + length;
55	var to: usize = self.wp;
56	const to_end: usize = to + length;
57
58	self.wp += length;
59
60	// Fast path using memcpy
61	if (length <= distance and // no overlapping buffers
62	(from >> 16 == from_end >> 16) and // start and and at the same circle
63	(to >> 16 == to_end >> 16))
64	{
65	@memcpy(self.buffer[to & mask .. to_end & mask], self.buffer[from & mask .. from_end & mask]);
66	return;
67	}
68
69	// Slow byte by byte
70	while (to < to_end) {
71	self.buffer[to & mask] = self.buffer[from & mask];
72	to += 1;
73	from += 1;
74	}
75	}
76
77	// Returns writable part of the internal buffer of size `n` at most. Advances
78	// write pointer, assumes that returned buffer will be filled with data.
79	pub fn getWritable(self: *Self, n: usize) []u8 {
80	const wp = self.wp & mask;
81	const len = @min(n, buffer_len - wp);
82	self.wp += len;
83	return self.buffer[wp .. wp + len];
84	}
85
86	// Read available data. Can return part of the available data if it is
87	// spread across two circles. So read until this returns zero length.
88	pub fn read(self: *Self) []const u8 {
89	return self.readAtMost(buffer_len);
90	}
91
92	// Read part of available data. Can return less than max even if there are
93	// more than max decoded data.
94	pub fn readAtMost(self: *Self, limit: usize) []const u8 {
95	const rb = self.readBlock(if (limit == 0) buffer_len else limit);
96	defer self.rp += rb.len;
97	return self.buffer[rb.head..rb.tail];
98	}
99
100	const ReadBlock = struct {
101	head: usize,
102	tail: usize,
103	len: usize,
104	};
105
106	// Returns position of continous read block data.
107	fn readBlock(self: *Self, max: usize) ReadBlock {
108	const r = self.rp & mask;
109	const w = self.wp & mask;
110	const n = @min(
111	max,
112	if (w >= r) w - r else buffer_len - r,
113	);
114	return .{
115	.head = r,
116	.tail = r + n,
117	.len = n,
118	};
119	}
120
121	// Number of free bytes for write.
122	pub fn free(self: *Self) usize {
123	return buffer_len - (self.wp - self.rp);
124	}
125
126	// Full if largest match can't fit. 258 is largest match length. That much bytes
127	// can be produced in single decode step.
128	pub fn full(self: *Self) bool {
129	return self.free() < 258 + 1;
130	}
131
132	// example from: https://youtu.be/SJPvNi4HrWQ?t=3558
133	test "flate.CircularBuffer writeMatch" {
134	var cb: Self = .{};
135
136	cb.writeAll("a salad; ");
137	try cb.writeMatch(5, 9);
138	try cb.writeMatch(3, 3);
139
140	try testing.expectEqualStrings("a salad; a salsal", cb.read());
141	}
142
143	test "flate.CircularBuffer writeMatch overlap" {
144	var cb: Self = .{};
145
146	cb.writeAll("a b c ");
147	try cb.writeMatch(8, 4);
148	cb.write('d');
149
150	try testing.expectEqualStrings("a b c b c b c d", cb.read());
151	}
152
153	test "flate.CircularBuffer readAtMost" {
154	var cb: Self = .{};
155
156	cb.writeAll("0123456789");
157	try cb.writeMatch(50, 10);
158
159	try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]);
160	for (0..6) \|i\| {
161	try testing.expectEqual(i * 10, cb.rp);
162	try testing.expectEqualStrings("0123456789", cb.readAtMost(10));
163	}
164	try testing.expectEqualStrings("", cb.readAtMost(10));
165	try testing.expectEqualStrings("", cb.read());
166	}
167
168	test "flate.CircularBuffer" {
169	var cb: Self = .{};
170
171	const data = "0123456789abcdef" ** (1024 / 16);
172	cb.writeAll(data);
173	try testing.expectEqual(@as(usize, 0), cb.rp);
174	try testing.expectEqual(@as(usize, 1024), cb.wp);
175	try testing.expectEqual(@as(usize, 1024 * 63), cb.free());
176
177	for (0..62 * 4) \|_\|
178	try cb.writeMatch(256, 1024); // write 62K
179
180	try testing.expectEqual(@as(usize, 0), cb.rp);
181	try testing.expectEqual(@as(usize, 63 * 1024), cb.wp);
182	try testing.expectEqual(@as(usize, 1024), cb.free());
183
184	cb.writeAll(data[0..200]);
185	_ = cb.readAtMost(1024); // make some space
186	cb.writeAll(data); // overflows write position
187	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
188	try testing.expectEqual(@as(usize, 1024), cb.rp);
189	try testing.expectEqual(@as(usize, 1024 - 200), cb.free());
190
191	const rb = cb.readBlock(Self.buffer_len);
192	try testing.expectEqual(@as(usize, 65536 - 1024), rb.len);
193	try testing.expectEqual(@as(usize, 1024), rb.head);
194	try testing.expectEqual(@as(usize, 65536), rb.tail);
195
196	try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer
197	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
198	try testing.expectEqual(@as(usize, 65536), cb.rp);
199	try testing.expectEqual(@as(usize, 65536 - 200), cb.free());
200
201	try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest
202	}
203
204	test "flate.CircularBuffer write overlap" {
205	var cb: Self = .{};
206	cb.wp = cb.buffer.len - 15;
207	cb.rp = cb.wp;
208
209	cb.writeAll("0123456789");
210	cb.writeAll("abcdefghij");
211
212	try testing.expectEqual(cb.buffer.len + 5, cb.wp);
213	try testing.expectEqual(cb.buffer.len - 15, cb.rp);
214
215	try testing.expectEqualStrings("0123456789abcde", cb.read());
216	try testing.expectEqualStrings("fghij", cb.read());
217
218	try testing.expect(cb.wp == cb.rp);
219	}
220
221	test "flate.CircularBuffer writeMatch/read overlap" {
222	var cb: Self = .{};
223	cb.wp = cb.buffer.len - 15;
224	cb.rp = cb.wp;
225
226	cb.writeAll("0123456789");
227	try cb.writeMatch(15, 5);
228
229	try testing.expectEqualStrings("012345678956789", cb.read());
230	try testing.expectEqualStrings("5678956789", cb.read());
231
232	try cb.writeMatch(20, 25);
233	try testing.expectEqualStrings("01234567895678956789", cb.read());
234	}

lib/std/compress/flate/Lookup.zig created+125

...	@@ -0,0 +1,125 @@
1	/// Lookup of the previous locations for the same 4 byte data. Works on hash of
2	/// 4 bytes data. Head contains position of the first match for each hash. Chain
3	/// points to the previous position of the same hash given the current location.
4	///
5	const std = @import("std");
6	const testing = std.testing;
7	const expect = testing.expect;
8	const consts = @import("consts.zig");
9
10	const Self = @This();
11
12	const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
13	const chain_len = 2 * consts.history.len;
14
15	// Maps hash => first position
16	head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len,
17	// Maps position => previous positions for the same hash value
18	chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
19
20	// Calculates hash of the 4 bytes from data.
21	// Inserts `pos` position of that hash in the lookup tables.
22	// Returns previous location with the same hash value.
23	pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
24	if (data.len < 4) return 0;
25	const h = hash(data[0..4]);
26	return self.set(h, pos);
27	}
28
29	// Retruns previous location with the same hash value given the current
30	// position.
31	pub fn prev(self: *Self, pos: u16) u16 {
32	return self.chain[pos];
33	}
34
35	fn set(self: *Self, h: u32, pos: u16) u16 {
36	const p = self.head[h];
37	self.head[h] = pos;
38	self.chain[pos] = p;
39	return p;
40	}
41
42	// Slide all positions in head and chain for `n`
43	pub fn slide(self: *Self, n: u16) void {
44	for (&self.head) \|*v\| {
45	v.* -\|= n;
46	}
47	var i: usize = 0;
48	while (i < n) : (i += 1) {
49	self.chain[i] = self.chain[i + n] -\| n;
50	}
51	}
52
53	// Add `len` 4 bytes hashes from `data` into lookup.
54	// Position of the first byte is `pos`.
55	pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {
56	if (len == 0 or data.len < consts.match.min_length) {
57	return;
58	}
59	var hb =
60	@as(u32, data[3]) \|
61	@as(u32, data[2]) << 8 \|
62	@as(u32, data[1]) << 16 \|
63	@as(u32, data[0]) << 24;
64	_ = self.set(hashu(hb), pos);
65
66	var i = pos;
67	for (4..@min(len + 3, data.len)) \|j\| {
68	hb = (hb << 8) \| @as(u32, data[j]);
69	i += 1;
70	_ = self.set(hashu(hb), i);
71	}
72	}
73
74	// Calculates hash of the first 4 bytes of `b`.
75	fn hash(b: *const [4]u8) u32 {
76	return hashu(@as(u32, b[3]) \|
77	@as(u32, b[2]) << 8 \|
78	@as(u32, b[1]) << 16 \|
79	@as(u32, b[0]) << 24);
80	}
81
82	fn hashu(v: u32) u32 {
83	return @intCast((v *% prime4) >> consts.lookup.shift);
84	}
85
86	test "flate.Lookup add/prev" {
87	const data = [_]u8{
88	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
89	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
90	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
91	0x01, 0x02, 0x03,
92	};
93
94	var h: Self = .{};
95	for (data, 0..) \|_, i\| {
96	const p = h.add(data[i..], @intCast(i));
97	if (i >= 8 and i < 24) {
98	try expect(p == i - 8);
99	} else {
100	try expect(p == 0);
101	}
102	}
103
104	const v = Self.hash(data[2 .. 2 + 4]);
105	try expect(h.head[v] == 2 + 16);
106	try expect(h.chain[2 + 16] == 2 + 8);
107	try expect(h.chain[2 + 8] == 2);
108	}
109
110	test "flate.Lookup bulkAdd" {
111	const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
112
113	// one by one
114	var h: Self = .{};
115	for (data, 0..) \|_, i\| {
116	_ = h.add(data[i..], @intCast(i));
117	}
118
119	// in bulk
120	var bh: Self = .{};
121	bh.bulkAdd(data, data.len, 0);
122
123	try testing.expectEqualSlices(u16, &h.head, &bh.head);
124	try testing.expectEqualSlices(u16, &h.chain, &bh.chain);
125	}

lib/std/compress/flate/SlidingWindow.zig created+160

...	@@ -0,0 +1,160 @@
1	/// Used in deflate (compression), holds uncompressed data form which Tokens are
2	/// produces. In combination with Lookup it is used to find matches in history data.
3	///
4	const std = @import("std");
5	const consts = @import("consts.zig");
6
7	const expect = testing.expect;
8	const assert = std.debug.assert;
9	const testing = std.testing;
10
11	const hist_len = consts.history.len;
12	const buffer_len = 2 * hist_len;
13	const min_lookahead = consts.match.min_length + consts.match.max_length;
14	const max_rp = buffer_len - min_lookahead;
15
16	const Self = @This();
17
18	buffer: [buffer_len]u8 = undefined,
19	wp: usize = 0, // write position
20	rp: usize = 0, // read position
21	fp: isize = 0, // last flush position, tokens are build from fp..rp
22
23	// Returns number of bytes written, or 0 if buffer is full and need to slide.
24	pub fn write(self: *Self, buf: []const u8) usize {
25	if (self.rp >= max_rp) return 0; // need to slide
26
27	const n = @min(buf.len, buffer_len - self.wp);
28	@memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]);
29	self.wp += n;
30	return n;
31	}
32
33	// Slide buffer for hist_len.
34	// Drops old history, preserves between hist_len and hist_len - min_lookahead.
35	// Returns number of bytes removed.
36	pub fn slide(self: *Self) u16 {
37	assert(self.rp >= max_rp and self.wp >= self.rp);
38	const n = self.wp - hist_len;
39	@memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]);
40	self.rp -= hist_len;
41	self.wp -= hist_len;
42	self.fp -= hist_len;
43	return @intCast(n);
44	}
45
46	// Data from the current position (read position). Those part of the buffer is
47	// not converted to tokens yet.
48	fn lookahead(self: *Self) []const u8 {
49	assert(self.wp >= self.rp);
50	return self.buffer[self.rp..self.wp];
51	}
52
53	// Returns part of the lookahead buffer. If should_flush is set no lookahead is
54	// preserved otherwise preserves enough data for the longest match. Returns
55	// null if there is not enough data.
56	pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 {
57	const min: usize = if (should_flush) 0 else min_lookahead;
58	const lh = self.lookahead();
59	return if (lh.len > min) lh else null;
60	}
61
62	// Advances read position, shrinks lookahead.
63	pub fn advance(self: *Self, n: u16) void {
64	assert(self.wp >= self.rp + n);
65	self.rp += n;
66	}
67
68	// Returns writable part of the buffer, where new uncompressed data can be
69	// written.
70	pub fn writable(self: *Self) []u8 {
71	return self.buffer[self.wp..];
72	}
73
74	// Notification of what part of writable buffer is filled with data.
75	pub fn written(self: *Self, n: usize) void {
76	self.wp += n;
77	}
78
79	// Finds match length between previous and current position.
80	// Used in hot path!
81	pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 {
82	const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length);
83	// lookahead buffers from previous and current positions
84	const prev_lh = self.buffer[prev_pos..][0..max_len];
85	const curr_lh = self.buffer[curr_pos..][0..max_len];
86
87	// If we alread have match (min_len > 0),
88	// test the first byte above previous len a[min_len] != b[min_len]
89	// and then all the bytes from that position to zero.
90	// That is likely positions to find difference than looping from first bytes.
91	var i: usize = min_len;
92	if (i > 0) {
93	if (max_len <= i) return 0;
94	while (true) {
95	if (prev_lh[i] != curr_lh[i]) return 0;
96	if (i == 0) break;
97	i -= 1;
98	}
99	i = min_len;
100	}
101	while (i < max_len) : (i += 1)
102	if (prev_lh[i] != curr_lh[i]) break;
103	return if (i >= consts.match.min_length) @intCast(i) else 0;
104	}
105
106	// Current position of non-compressed data. Data before rp are already converted
107	// to tokens.
108	pub fn pos(self: *Self) u16 {
109	return @intCast(self.rp);
110	}
111
112	// Notification that token list is cleared.
113	pub fn flush(self: *Self) void {
114	self.fp = @intCast(self.rp);
115	}
116
117	// Part of the buffer since last flush or null if there was slide in between (so
118	// fp becomes negative).
119	pub fn tokensBuffer(self: *Self) ?[]const u8 {
120	assert(self.fp <= self.rp);
121	if (self.fp < 0) return null;
122	return self.buffer[@intCast(self.fp)..self.rp];
123	}
124
125	test "flate.SlidingWindow match" {
126	const data = "Blah blah blah blah blah!";
127	var win: Self = .{};
128	try expect(win.write(data) == data.len);
129	try expect(win.wp == data.len);
130	try expect(win.rp == 0);
131
132	// length between l symbols
133	try expect(win.match(1, 6, 0) == 18);
134	try expect(win.match(1, 11, 0) == 13);
135	try expect(win.match(1, 16, 0) == 8);
136	try expect(win.match(1, 21, 0) == 0);
137
138	// position 15 = "blah blah!"
139	// position 20 = "blah!"
140	try expect(win.match(15, 20, 0) == 4);
141	try expect(win.match(15, 20, 3) == 4);
142	try expect(win.match(15, 20, 4) == 0);
143	}
144
145	test "flate.SlidingWindow slide" {
146	var win: Self = .{};
147	win.wp = Self.buffer_len - 11;
148	win.rp = Self.buffer_len - 111;
149	win.buffer[win.rp] = 0xab;
150	try expect(win.lookahead().len == 100);
151	try expect(win.tokensBuffer().?.len == win.rp);
152
153	const n = win.slide();
154	try expect(n == 32757);
155	try expect(win.buffer[win.rp] == 0xab);
156	try expect(win.rp == Self.hist_len - 111);
157	try expect(win.wp == Self.hist_len - 11);
158	try expect(win.lookahead().len == 100);
159	try expect(win.tokensBuffer() == null);
160	}

lib/std/compress/flate/Token.zig created+327

...	@@ -0,0 +1,327 @@
1	/// Token cat be literal: single byte of data or match; reference to the slice of
2	/// data in the same stream represented with <length, distance>. Where length
3	/// can be 3 - 258 bytes, and distance 1 - 32768 bytes.
4	///
5	const std = @import("std");
6	const assert = std.debug.assert;
7	const print = std.debug.print;
8	const expect = std.testing.expect;
9	const consts = @import("consts.zig").match;
10
11	const Token = @This();
12
13	pub const Kind = enum(u1) {
14	literal,
15	match,
16	};
17
18	// Distance range 1 - 32768, stored in dist as 0 - 32767 (fits u15)
19	dist: u15 = 0,
20	// Length range 3 - 258, stored in len_lit as 0 - 255 (fits u8)
21	len_lit: u8 = 0,
22	kind: Kind = .literal,
23
24	pub fn literal(t: Token) u8 {
25	return t.len_lit;
26	}
27
28	pub fn distance(t: Token) u16 {
29	return @as(u16, t.dist) + consts.min_distance;
30	}
31
32	pub fn length(t: Token) u16 {
33	return @as(u16, t.len_lit) + consts.base_length;
34	}
35
36	pub fn initLiteral(lit: u8) Token {
37	return .{ .kind = .literal, .len_lit = lit };
38	}
39
40	// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
41	// length range 3 - 258, stored in len_lit as 0 - 255 (u8)
42	pub fn initMatch(dist: u16, len: u16) Token {
43	assert(len >= consts.min_length and len <= consts.max_length);
44	assert(dist >= consts.min_distance and dist <= consts.max_distance);
45	return .{
46	.kind = .match,
47	.dist = @intCast(dist - consts.min_distance),
48	.len_lit = @intCast(len - consts.base_length),
49	};
50	}
51
52	pub fn eql(t: Token, o: Token) bool {
53	return t.kind == o.kind and
54	t.dist == o.dist and
55	t.len_lit == o.len_lit;
56	}
57
58	pub fn lengthCode(t: Token) u16 {
59	return match_lengths[match_lengths_index[t.len_lit]].code;
60	}
61
62	pub fn lengthEncoding(t: Token) MatchLength {
63	var c = match_lengths[match_lengths_index[t.len_lit]];
64	c.extra_length = t.len_lit - c.base_scaled;
65	return c;
66	}
67
68	// Returns the distance code corresponding to a specific distance.
69	// Distance code is in range: 0 - 29.
70	pub fn distanceCode(t: Token) u8 {
71	var dist: u16 = t.dist;
72	if (dist < match_distances_index.len) {
73	return match_distances_index[dist];
74	}
75	dist >>= 7;
76	if (dist < match_distances_index.len) {
77	return match_distances_index[dist] + 14;
78	}
79	dist >>= 7;
80	return match_distances_index[dist] + 28;
81	}
82
83	pub fn distanceEncoding(t: Token) MatchDistance {
84	var c = match_distances[t.distanceCode()];
85	c.extra_distance = t.dist - c.base_scaled;
86	return c;
87	}
88
89	pub fn lengthExtraBits(code: u32) u8 {
90	return match_lengths[code - length_codes_start].extra_bits;
91	}
92
93	pub fn matchLength(code: u8) MatchLength {
94	return match_lengths[code];
95	}
96
97	pub fn matchDistance(code: u8) MatchDistance {
98	return match_distances[code];
99	}
100
101	pub fn distanceExtraBits(code: u32) u8 {
102	return match_distances[code].extra_bits;
103	}
104
105	pub fn show(t: Token) void {
106	if (t.kind == .literal) {
107	print("L('{c}'), ", .{t.literal()});
108	} else {
109	print("M({d}, {d}), ", .{ t.distance(), t.length() });
110	}
111	}
112
113	// Retruns index in match_lengths table for each length in range 0-255.
114	const match_lengths_index = [_]u8{
115	0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
116	9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
117	13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
118	15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
119	17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
120	18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
121	19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
122	20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
123	21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
124	21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
125	22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
126	22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
127	23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
128	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
129	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
130	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
131	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
132	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
133	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
134	25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
135	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
136	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
137	26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
138	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
139	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
140	27, 27, 27, 27, 27, 28,
141	};
142
143	const MatchLength = struct {
144	code: u16,
145	base_scaled: u8, // base - 3, scaled to fit into u8 (0-255), same as lit_len field in Token.
146	base: u16, // 3-258
147	extra_length: u8 = 0,
148	extra_bits: u4,
149	};
150
151	// match_lengths represents table from rfc (https://datatracker.ietf.org/doc/html/rfc1951#page-12)
152	//
153	// Extra Extra Extra
154	// Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
155	// ---- ---- ------ ---- ---- ------- ---- ---- -------
156	// 257 0 3 267 1 15,16 277 4 67-82
157	// 258 0 4 268 1 17,18 278 4 83-98
158	// 259 0 5 269 2 19-22 279 4 99-114
159	// 260 0 6 270 2 23-26 280 4 115-130
160	// 261 0 7 271 2 27-30 281 5 131-162
161	// 262 0 8 272 2 31-34 282 5 163-194
162	// 263 0 9 273 3 35-42 283 5 195-226
163	// 264 0 10 274 3 43-50 284 5 227-257
164	// 265 1 11,12 275 3 51-58 285 0 258
165	// 266 1 13,14 276 3 59-66
166	//
167	pub const length_codes_start = 257;
168
169	const match_lengths = [_]MatchLength{
170	.{ .extra_bits = 0, .base_scaled = 0, .base = 3, .code = 257 },
171	.{ .extra_bits = 0, .base_scaled = 1, .base = 4, .code = 258 },
172	.{ .extra_bits = 0, .base_scaled = 2, .base = 5, .code = 259 },
173	.{ .extra_bits = 0, .base_scaled = 3, .base = 6, .code = 260 },
174	.{ .extra_bits = 0, .base_scaled = 4, .base = 7, .code = 261 },
175	.{ .extra_bits = 0, .base_scaled = 5, .base = 8, .code = 262 },
176	.{ .extra_bits = 0, .base_scaled = 6, .base = 9, .code = 263 },
177	.{ .extra_bits = 0, .base_scaled = 7, .base = 10, .code = 264 },
178	.{ .extra_bits = 1, .base_scaled = 8, .base = 11, .code = 265 },
179	.{ .extra_bits = 1, .base_scaled = 10, .base = 13, .code = 266 },
180	.{ .extra_bits = 1, .base_scaled = 12, .base = 15, .code = 267 },
181	.{ .extra_bits = 1, .base_scaled = 14, .base = 17, .code = 268 },
182	.{ .extra_bits = 2, .base_scaled = 16, .base = 19, .code = 269 },
183	.{ .extra_bits = 2, .base_scaled = 20, .base = 23, .code = 270 },
184	.{ .extra_bits = 2, .base_scaled = 24, .base = 27, .code = 271 },
185	.{ .extra_bits = 2, .base_scaled = 28, .base = 31, .code = 272 },
186	.{ .extra_bits = 3, .base_scaled = 32, .base = 35, .code = 273 },
187	.{ .extra_bits = 3, .base_scaled = 40, .base = 43, .code = 274 },
188	.{ .extra_bits = 3, .base_scaled = 48, .base = 51, .code = 275 },
189	.{ .extra_bits = 3, .base_scaled = 56, .base = 59, .code = 276 },
190	.{ .extra_bits = 4, .base_scaled = 64, .base = 67, .code = 277 },
191	.{ .extra_bits = 4, .base_scaled = 80, .base = 83, .code = 278 },
192	.{ .extra_bits = 4, .base_scaled = 96, .base = 99, .code = 279 },
193	.{ .extra_bits = 4, .base_scaled = 112, .base = 115, .code = 280 },
194	.{ .extra_bits = 5, .base_scaled = 128, .base = 131, .code = 281 },
195	.{ .extra_bits = 5, .base_scaled = 160, .base = 163, .code = 282 },
196	.{ .extra_bits = 5, .base_scaled = 192, .base = 195, .code = 283 },
197	.{ .extra_bits = 5, .base_scaled = 224, .base = 227, .code = 284 },
198	.{ .extra_bits = 0, .base_scaled = 255, .base = 258, .code = 285 },
199	};
200
201	// Used in distanceCode fn to get index in match_distance table for each distance in range 0-32767.
202	const match_distances_index = [_]u8{
203	0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
204	8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
205	10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
206	11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
207	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
208	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
209	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
210	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
211	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
212	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
213	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
214	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
215	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
216	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
217	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
218	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
219	};
220
221	const MatchDistance = struct {
222	base_scaled: u16, // base - 1, same as Token dist field
223	base: u16,
224	extra_distance: u16 = 0,
225	code: u8,
226	extra_bits: u4,
227	};
228
229	// match_distances represents table from rfc (https://datatracker.ietf.org/doc/html/rfc1951#page-12)
230	//
231	// Extra Extra Extra
232	// Code Bits Dist Code Bits Dist Code Bits Distance
233	// ---- ---- ---- ---- ---- ------ ---- ---- --------
234	// 0 0 1 10 4 33-48 20 9 1025-1536
235	// 1 0 2 11 4 49-64 21 9 1537-2048
236	// 2 0 3 12 5 65-96 22 10 2049-3072
237	// 3 0 4 13 5 97-128 23 10 3073-4096
238	// 4 1 5,6 14 6 129-192 24 11 4097-6144
239	// 5 1 7,8 15 6 193-256 25 11 6145-8192
240	// 6 2 9-12 16 7 257-384 26 12 8193-12288
241	// 7 2 13-16 17 7 385-512 27 12 12289-16384
242	// 8 3 17-24 18 8 513-768 28 13 16385-24576
243	// 9 3 25-32 19 8 769-1024 29 13 24577-32768
244	//
245	const match_distances = [_]MatchDistance{
246	.{ .extra_bits = 0, .base_scaled = 0x0000, .code = 0, .base = 1 },
247	.{ .extra_bits = 0, .base_scaled = 0x0001, .code = 1, .base = 2 },
248	.{ .extra_bits = 0, .base_scaled = 0x0002, .code = 2, .base = 3 },
249	.{ .extra_bits = 0, .base_scaled = 0x0003, .code = 3, .base = 4 },
250	.{ .extra_bits = 1, .base_scaled = 0x0004, .code = 4, .base = 5 },
251	.{ .extra_bits = 1, .base_scaled = 0x0006, .code = 5, .base = 7 },
252	.{ .extra_bits = 2, .base_scaled = 0x0008, .code = 6, .base = 9 },
253	.{ .extra_bits = 2, .base_scaled = 0x000c, .code = 7, .base = 13 },
254	.{ .extra_bits = 3, .base_scaled = 0x0010, .code = 8, .base = 17 },
255	.{ .extra_bits = 3, .base_scaled = 0x0018, .code = 9, .base = 25 },
256	.{ .extra_bits = 4, .base_scaled = 0x0020, .code = 10, .base = 33 },
257	.{ .extra_bits = 4, .base_scaled = 0x0030, .code = 11, .base = 49 },
258	.{ .extra_bits = 5, .base_scaled = 0x0040, .code = 12, .base = 65 },
259	.{ .extra_bits = 5, .base_scaled = 0x0060, .code = 13, .base = 97 },
260	.{ .extra_bits = 6, .base_scaled = 0x0080, .code = 14, .base = 129 },
261	.{ .extra_bits = 6, .base_scaled = 0x00c0, .code = 15, .base = 193 },
262	.{ .extra_bits = 7, .base_scaled = 0x0100, .code = 16, .base = 257 },
263	.{ .extra_bits = 7, .base_scaled = 0x0180, .code = 17, .base = 385 },
264	.{ .extra_bits = 8, .base_scaled = 0x0200, .code = 18, .base = 513 },
265	.{ .extra_bits = 8, .base_scaled = 0x0300, .code = 19, .base = 769 },
266	.{ .extra_bits = 9, .base_scaled = 0x0400, .code = 20, .base = 1025 },
267	.{ .extra_bits = 9, .base_scaled = 0x0600, .code = 21, .base = 1537 },
268	.{ .extra_bits = 10, .base_scaled = 0x0800, .code = 22, .base = 2049 },
269	.{ .extra_bits = 10, .base_scaled = 0x0c00, .code = 23, .base = 3073 },
270	.{ .extra_bits = 11, .base_scaled = 0x1000, .code = 24, .base = 4097 },
271	.{ .extra_bits = 11, .base_scaled = 0x1800, .code = 25, .base = 6145 },
272	.{ .extra_bits = 12, .base_scaled = 0x2000, .code = 26, .base = 8193 },
273	.{ .extra_bits = 12, .base_scaled = 0x3000, .code = 27, .base = 12289 },
274	.{ .extra_bits = 13, .base_scaled = 0x4000, .code = 28, .base = 16385 },
275	.{ .extra_bits = 13, .base_scaled = 0x6000, .code = 29, .base = 24577 },
276	};
277
278	test "flate.Token size" {
279	try expect(@sizeOf(Token) == 4);
280	}
281
282	// testing table https://datatracker.ietf.org/doc/html/rfc1951#page-12
283	test "flate.Token MatchLength" {
284	var c = Token.initMatch(1, 4).lengthEncoding();
285	try expect(c.code == 258);
286	try expect(c.extra_bits == 0);
287	try expect(c.extra_length == 0);
288
289	c = Token.initMatch(1, 11).lengthEncoding();
290	try expect(c.code == 265);
291	try expect(c.extra_bits == 1);
292	try expect(c.extra_length == 0);
293
294	c = Token.initMatch(1, 12).lengthEncoding();
295	try expect(c.code == 265);
296	try expect(c.extra_bits == 1);
297	try expect(c.extra_length == 1);
298
299	c = Token.initMatch(1, 130).lengthEncoding();
300	try expect(c.code == 280);
301	try expect(c.extra_bits == 4);
302	try expect(c.extra_length == 130 - 115);
303	}
304
305	test "flate.Token MatchDistance" {
306	var c = Token.initMatch(1, 4).distanceEncoding();
307	try expect(c.code == 0);
308	try expect(c.extra_bits == 0);
309	try expect(c.extra_distance == 0);
310
311	c = Token.initMatch(192, 4).distanceEncoding();
312	try expect(c.code == 14);
313	try expect(c.extra_bits == 6);
314	try expect(c.extra_distance == 192 - 129);
315	}
316
317	test "flate.Token match_lengths" {
318	for (match_lengths, 0..) \|ml, i\| {
319	try expect(@as(u16, ml.base_scaled) + 3 == ml.base);
320	try expect(i + 257 == ml.code);
321	}
322
323	for (match_distances, 0..) \|mo, i\| {
324	try expect(mo.base_scaled + 1 == mo.base);
325	try expect(i == mo.code);
326	}
327	}

lib/std/compress/flate/bit_reader.zig created+333

...	@@ -0,0 +1,333 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
6	return BitReader(@TypeOf(reader)).init(reader);
7	}
8
9	/// Bit reader used during inflate (decompression). Has internal buffer of 64
10	/// bits which shifts right after bits are consumed. Uses forward_reader to fill
11	/// that internal buffer when needed.
12	///
13	/// readF is the core function. Supports few different ways of getting bits
14	/// controlled by flags. In hot path we try to avoid checking whether we need to
15	/// fill buffer from forward_reader by calling fill in advance and readF with
16	/// buffered flag set.
17	///
18	pub fn BitReader(comptime ReaderType: type) type {
19	return struct {
20	// Underlying reader used for filling internal bits buffer
21	forward_reader: ReaderType = undefined,
22	// Internal buffer of 64 bits
23	bits: u64 = 0,
24	// Number of bits in the buffer
25	nbits: u32 = 0,
26
27	const Self = @This();
28
29	pub const Error = ReaderType.Error \|\| error{EndOfStream};
30
31	pub fn init(rdr: ReaderType) Self {
32	var self = Self{ .forward_reader = rdr };
33	self.fill(1) catch {};
34	return self;
35	}
36
37	// Try to have `nice` bits are available in buffer. Reads from
38	// forward reader if there is no `nice` bits in buffer. Returns error
39	// if end of forward stream is reached and internal buffer is empty.
40	// It will not error if less than `nice` bits are in buffer, only when
41	// all bits are exhausted. During inflate we usually know what is the
42	// maximum bits for the next step but usually that step will need less
43	// bits to decode. So `nice` is not hard limit, it will just try to have
44	// that number of bits available. If end of forward stream is reached
45	// it may be some extra zero bits in buffer.
46	pub inline fn fill(self: *Self, nice: u6) !void {
47	if (self.nbits >= nice) {
48	return; // We have enought bits
49	}
50	// Read more bits from forward reader
51
52	// Number of empty bytes in bits, round nbits to whole bytes.
53	const empty_bytes =
54	@as(u8, if (self.nbits & 0x7 == 0) 8 else 7) - // 8 for 8, 16, 24..., 7 otherwise
55	(self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
56
57	var buf: [8]u8 = [_]u8{0} ** 8;
58	const bytes_read = self.forward_reader.read(buf[0..empty_bytes]) catch 0;
59	if (bytes_read > 0) {
60	const u: u64 = std.mem.readInt(u64, buf[0..8], .little);
61	self.bits \|= u << @as(u6, @intCast(self.nbits));
62	self.nbits += 8 * @as(u8, @intCast(bytes_read));
63	return;
64	}
65
66	if (self.nbits == 0)
67	return error.EndOfStream;
68	}
69
70	// Read exactly buf.len bytes into buf.
71	pub fn readAll(self: *Self, buf: []u8) !void {
72	assert(self.alignBits() == 0); // internal bits must be at byte boundary
73
74	// First read from internal bits buffer.
75	var n: usize = 0;
76	while (self.nbits > 0 and n < buf.len) {
77	buf[n] = try self.readF(u8, flag.buffered);
78	n += 1;
79	}
80	// Then use forward reader for all other bytes.
81	try self.forward_reader.readNoEof(buf[n..]);
82	}
83
84	pub const flag = struct {
85	pub const peek: u3 = 0b001; // dont advance internal buffer, just get bits, leave them in buffer
86	pub const buffered: u3 = 0b010; // assume that there is no need to fill, fill should be called before
87	pub const reverse: u3 = 0b100; // bit reverse readed bits
88	};
89
90	// Alias for readF(U, 0).
91	pub fn read(self: *Self, comptime U: type) !U {
92	return self.readF(U, 0);
93	}
94
95	// Alias for readF with flag.peak set.
96	pub inline fn peekF(self: *Self, comptime U: type, comptime how: u3) !U {
97	return self.readF(U, how \| flag.peek);
98	}
99
100	// Read with flags provided.
101	pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {
102	const n: u6 = @bitSizeOf(U);
103	switch (how) {
104	0 => { // `normal` read
105	try self.fill(n); // ensure that there are n bits in the buffer
106	const u: U = @truncate(self.bits); // get n bits
107	try self.shift(n); // advance buffer for n
108	return u;
109	},
110	(flag.peek) => { // no shift, leave bits in the buffer
111	try self.fill(n);
112	return @truncate(self.bits);
113	},
114	flag.buffered => { // no fill, assume that buffer has enought bits
115	const u: U = @truncate(self.bits);
116	try self.shift(n);
117	return u;
118	},
119	(flag.reverse) => { // same as 0 with bit reverse
120	try self.fill(n);
121	const u: U = @truncate(self.bits);
122	try self.shift(n);
123	return @bitReverse(u);
124	},
125	(flag.peek \| flag.reverse) => {
126	try self.fill(n);
127	return @bitReverse(@as(U, @truncate(self.bits)));
128	},
129	(flag.buffered \| flag.reverse) => {
130	const u: U = @truncate(self.bits);
131	try self.shift(n);
132	return @bitReverse(u);
133	},
134	(flag.peek \| flag.buffered) => {
135	return @truncate(self.bits);
136	},
137	(flag.peek \| flag.buffered \| flag.reverse) => {
138	return @bitReverse(@as(U, @truncate(self.bits)));
139	},
140	}
141	}
142
143	// Read n number of bits.
144	// Only buffered flag can be used in how.
145	pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 {
146	switch (how) {
147	0 => {
148	try self.fill(n);
149	},
150	flag.buffered => {},
151	else => unreachable,
152	}
153	const mask: u16 = (@as(u16, 1) << n) - 1;
154	const u: u16 = @as(u16, @truncate(self.bits)) & mask;
155	try self.shift(n);
156	return u;
157	}
158
159	// Advance buffer for n bits.
160	pub fn shift(self: *Self, n: u6) !void {
161	if (n > self.nbits) return error.EndOfStream;
162	self.bits >>= n;
163	self.nbits -= n;
164	}
165
166	// Skip n bytes.
167	pub fn skipBytes(self: *Self, n: u16) !void {
168	for (0..n) \|_\| {
169	try self.fill(8);
170	try self.shift(8);
171	}
172	}
173
174	// Number of bits to align stream to the byte boundary.
175	fn alignBits(self: *Self) u3 {
176	return @intCast(self.nbits & 0x7);
177	}
178
179	// Align stream to the byte boundary.
180	pub fn alignToByte(self: *Self) void {
181	const ab = self.alignBits();
182	if (ab > 0) self.shift(ab) catch unreachable;
183	}
184
185	// Skip zero terminated string.
186	pub fn skipStringZ(self: *Self) !void {
187	while (true) {
188	if (try self.readF(u8, 0) == 0) break;
189	}
190	}
191
192	// Read deflate fixed fixed code.
193	// Reads first 7 bits, and then mybe 1 or 2 more to get full 7,8 or 9 bit code.
194	// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
195	// Lit Value Bits Codes
196	// --------- ---- -----
197	// 0 - 143 8 00110000 through
198	// 10111111
199	// 144 - 255 9 110010000 through
200	// 111111111
201	// 256 - 279 7 0000000 through
202	// 0010111
203	// 280 - 287 8 11000000 through
204	// 11000111
205	pub fn readFixedCode(self: *Self) !u16 {
206	try self.fill(7 + 2);
207	const code7 = try self.readF(u7, flag.buffered \| flag.reverse);
208	if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
209	return @as(u16, code7) + 256;
210	} else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
211	return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, flag.buffered)) - 0b0011_0000;
212	} else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
213	return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, flag.buffered) + 280;
214	} else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
215	return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, flag.buffered \| flag.reverse)) + 144;
216	}
217	}
218	};
219	}
220
221	test "flate.BitReader" {
222	var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
223	var br = bitReader(fbs.reader());
224	const F = BitReader(@TypeOf(fbs.reader())).flag;
225
226	try testing.expectEqual(@as(u8, 48), br.nbits);
227	try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
228
229	try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
230	try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
231	try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
232	try testing.expectEqual(@as(u3, 5), br.alignBits());
233
234	try testing.expect(try br.readF(u8, F.peek) == 0b0001_1110);
235	try testing.expect(try br.readF(u9, F.peek) == 0b1_0001_1110);
236	try br.shift(9);
237	try testing.expectEqual(@as(u8, 36), br.nbits);
238	try testing.expectEqual(@as(u3, 4), br.alignBits());
239
240	try testing.expect(try br.readF(u4, 0) == 0b0100);
241	try testing.expectEqual(@as(u8, 32), br.nbits);
242	try testing.expectEqual(@as(u3, 0), br.alignBits());
243
244	try br.shift(1);
245	try testing.expectEqual(@as(u3, 7), br.alignBits());
246	try br.shift(1);
247	try testing.expectEqual(@as(u3, 6), br.alignBits());
248	br.alignToByte();
249	try testing.expectEqual(@as(u3, 0), br.alignBits());
250
251	try testing.expectEqual(@as(u64, 0xc9), br.bits);
252	try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
253	try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
254	}
255
256	test "flate.BitReader read block type 1 data" {
257	const data = [_]u8{
258	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
259	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
260	0x0c, 0x01, 0x02, 0x03, //
261	0xaa, 0xbb, 0xcc, 0xdd,
262	};
263	var fbs = std.io.fixedBufferStream(&data);
264	var br = bitReader(fbs.reader());
265	const F = BitReader(@TypeOf(fbs.reader())).flag;
266
267	try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
268	try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
269
270	for ("Hello world\n") \|c\| {
271	try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
272	}
273	try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
274	br.alignToByte();
275	try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
276	try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
277	try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
278	}
279
280	test "flate.BitReader init" {
281	const data = [_]u8{
282	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
283	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
284	};
285	var fbs = std.io.fixedBufferStream(&data);
286	var br = bitReader(fbs.reader());
287
288	try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
289	try br.shift(8);
290	try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
291	try br.fill(60); // fill with 1 byte
292	try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
293	try br.shift(8 * 4 + 4);
294	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
295
296	try br.fill(60); // fill with 4 bytes (shift by 4)
297	try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
298	try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
299
300	try br.shift(@intCast(br.nbits)); // clear buffer
301	try br.fill(8); // refill with the rest of the bytes
302	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
303	}
304
305	test "flate.BitReader readAll" {
306	const data = [_]u8{
307	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
308	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
309	};
310	var fbs = std.io.fixedBufferStream(&data);
311	var br = bitReader(fbs.reader());
312
313	try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
314
315	var out: [16]u8 = undefined;
316	try br.readAll(out[0..]);
317	try testing.expect(br.nbits == 0);
318	try testing.expect(br.bits == 0);
319
320	try testing.expectEqualSlices(u8, data[0..16], &out);
321	}
322
323	test "flate.BitReader readFixedCode" {
324	const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
325
326	var fbs = std.io.fixedBufferStream(&fixed_codes);
327	var rdr = bitReader(fbs.reader());
328
329	for (0..286) \|c\| {
330	try testing.expectEqual(c, try rdr.readFixedCode());
331	}
332	try testing.expect(rdr.nbits == 0);
333	}

lib/std/compress/flate/bit_writer.zig created+99

...	@@ -0,0 +1,99 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3
4	/// Bit writer for use in deflate (compression).
5	///
6	/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
7	/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
8	/// accumulate 240 bytes they are flushed to the underlying inner_writer.
9	///
10	pub fn BitWriter(comptime WriterType: type) type {
11	// buffer_flush_size indicates the buffer size
12	// after which bytes are flushed to the writer.
13	// Should preferably be a multiple of 6, since
14	// we accumulate 6 bytes between writes to the buffer.
15	const buffer_flush_size = 240;
16
17	// buffer_size is the actual output byte buffer size.
18	// It must have additional headroom for a flush
19	// which can contain up to 8 bytes.
20	const buffer_size = buffer_flush_size + 8;
21
22	return struct {
23	inner_writer: WriterType,
24
25	// Data waiting to be written is bytes[0 .. nbytes]
26	// and then the low nbits of bits. Data is always written
27	// sequentially into the bytes array.
28	bits: u64 = 0,
29	nbits: u32 = 0, // number of bits
30	bytes: [buffer_size]u8 = undefined,
31	nbytes: u32 = 0, // number of bytes
32
33	const Self = @This();
34
35	pub const Error = WriterType.Error \|\| error{UnfinishedBits};
36
37	pub fn init(writer: WriterType) Self {
38	return .{ .inner_writer = writer };
39	}
40
41	pub fn setWriter(self: *Self, new_writer: WriterType) void {
42	//assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0);
43	self.inner_writer = new_writer;
44	}
45
46	pub fn flush(self: *Self) Error!void {
47	var n = self.nbytes;
48	while (self.nbits != 0) {
49	self.bytes[n] = @as(u8, @truncate(self.bits));
50	self.bits >>= 8;
51	if (self.nbits > 8) { // Avoid underflow
52	self.nbits -= 8;
53	} else {
54	self.nbits = 0;
55	}
56	n += 1;
57	}
58	self.bits = 0;
59	_ = try self.inner_writer.write(self.bytes[0..n]);
60	self.nbytes = 0;
61	}
62
63	pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
64	self.bits \|= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
65	self.nbits += nb;
66	if (self.nbits < 48)
67	return;
68
69	var n = self.nbytes;
70	std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
71	n += 6;
72	if (n >= buffer_flush_size) {
73	_ = try self.inner_writer.write(self.bytes[0..n]);
74	n = 0;
75	}
76	self.nbytes = n;
77	self.bits >>= 48;
78	self.nbits -= 48;
79	}
80
81	pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
82	var n = self.nbytes;
83	if (self.nbits & 7 != 0) {
84	return error.UnfinishedBits;
85	}
86	while (self.nbits != 0) {
87	self.bytes[n] = @as(u8, @truncate(self.bits));
88	self.bits >>= 8;
89	self.nbits -= 8;
90	n += 1;
91	}
92	if (n != 0) {
93	_ = try self.inner_writer.write(self.bytes[0..n]);
94	}
95	self.nbytes = 0;
96	_ = try self.inner_writer.write(bytes);
97	}
98	};
99	}

lib/std/compress/flate/block_writer.zig created+706

...	@@ -0,0 +1,706 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4
5	const hc = @import("huffman_encoder.zig");
6	const consts = @import("consts.zig").huffman;
7	const Token = @import("Token.zig");
8	const BitWriter = @import("bit_writer.zig").BitWriter;
9
10	pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) {
11	return BlockWriter(@TypeOf(writer)).init(writer);
12	}
13
14	/// Accepts list of tokens, decides what is best block type to write. What block
15	/// type will provide best compression. Writes header and body of the block.
16	///
17	pub fn BlockWriter(comptime WriterType: type) type {
18	const BitWriterType = BitWriter(WriterType);
19	return struct {
20	const codegen_order = consts.codegen_order;
21	const end_code_mark = 255;
22	const Self = @This();
23
24	pub const Error = BitWriterType.Error;
25	bit_writer: BitWriterType,
26
27	codegen_freq: [consts.codegen_code_count]u16 = undefined,
28	literal_freq: [consts.max_num_lit]u16 = undefined,
29	distance_freq: [consts.distance_code_count]u16 = undefined,
30	codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
31	literal_encoding: hc.LiteralEncoder = .{},
32	distance_encoding: hc.DistanceEncoder = .{},
33	codegen_encoding: hc.CodegenEncoder = .{},
34	fixed_literal_encoding: hc.LiteralEncoder,
35	fixed_distance_encoding: hc.DistanceEncoder,
36	huff_distance: hc.DistanceEncoder,
37
38	pub fn init(writer: WriterType) Self {
39	return .{
40	.bit_writer = BitWriterType.init(writer),
41	.fixed_literal_encoding = hc.fixedLiteralEncoder(),
42	.fixed_distance_encoding = hc.fixedDistanceEncoder(),
43	.huff_distance = hc.huffmanDistanceEncoder(),
44	};
45	}
46
47	/// Flush intrenal bit buffer to the writer.
48	/// Should be called only when bit stream is at byte boundary.
49	///
50	/// That is after final block; when last byte could be incomplete or
51	/// after stored block; which is aligned to the byte bounday (it has x
52	/// padding bits after first 3 bits).
53	pub fn flush(self: *Self) Error!void {
54	try self.bit_writer.flush();
55	}
56
57	pub fn setWriter(self: *Self, new_writer: WriterType) void {
58	self.bit_writer.setWriter(new_writer);
59	}
60
61	fn writeCode(self: *Self, c: hc.HuffCode) Error!void {
62	try self.bit_writer.writeBits(c.code, c.len);
63	}
64
65	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
66	// the literal and distance lengths arrays (which are concatenated into a single
67	// array). This method generates that run-length encoding.
68	//
69	// The result is written into the codegen array, and the frequencies
70	// of each code is written into the codegen_freq array.
71	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
72	// information. Code bad_code is an end marker
73	//
74	// num_literals: The number of literals in literal_encoding
75	// num_distances: The number of distances in distance_encoding
76	// lit_enc: The literal encoder to use
77	// dist_enc: The distance encoder to use
78	fn generateCodegen(
79	self: *Self,
80	num_literals: u32,
81	num_distances: u32,
82	lit_enc: *hc.LiteralEncoder,
83	dist_enc: *hc.DistanceEncoder,
84	) void {
85	for (self.codegen_freq, 0..) \|_, i\| {
86	self.codegen_freq[i] = 0;
87	}
88
89	// Note that we are using codegen both as a temporary variable for holding
90	// a copy of the frequencies, and as the place where we put the result.
91	// This is fine because the output is always shorter than the input used
92	// so far.
93	var codegen = &self.codegen; // cache
94	// Copy the concatenated code sizes to codegen. Put a marker at the end.
95	var cgnl = codegen[0..num_literals];
96	for (cgnl, 0..) \|_, i\| {
97	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
98	}
99
100	cgnl = codegen[num_literals .. num_literals + num_distances];
101	for (cgnl, 0..) \|_, i\| {
102	cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
103	}
104	codegen[num_literals + num_distances] = end_code_mark;
105
106	var size = codegen[0];
107	var count: i32 = 1;
108	var out_index: u32 = 0;
109	var in_index: u32 = 1;
110	while (size != end_code_mark) : (in_index += 1) {
111	// INVARIANT: We have seen "count" copies of size that have not yet
112	// had output generated for them.
113	const next_size = codegen[in_index];
114	if (next_size == size) {
115	count += 1;
116	continue;
117	}
118	// We need to generate codegen indicating "count" of size.
119	if (size != 0) {
120	codegen[out_index] = size;
121	out_index += 1;
122	self.codegen_freq[size] += 1;
123	count -= 1;
124	while (count >= 3) {
125	var n: i32 = 6;
126	if (n > count) {
127	n = count;
128	}
129	codegen[out_index] = 16;
130	out_index += 1;
131	codegen[out_index] = @as(u8, @intCast(n - 3));
132	out_index += 1;
133	self.codegen_freq[16] += 1;
134	count -= n;
135	}
136	} else {
137	while (count >= 11) {
138	var n: i32 = 138;
139	if (n > count) {
140	n = count;
141	}
142	codegen[out_index] = 18;
143	out_index += 1;
144	codegen[out_index] = @as(u8, @intCast(n - 11));
145	out_index += 1;
146	self.codegen_freq[18] += 1;
147	count -= n;
148	}
149	if (count >= 3) {
150	// 3 <= count <= 10
151	codegen[out_index] = 17;
152	out_index += 1;
153	codegen[out_index] = @as(u8, @intCast(count - 3));
154	out_index += 1;
155	self.codegen_freq[17] += 1;
156	count = 0;
157	}
158	}
159	count -= 1;
160	while (count >= 0) : (count -= 1) {
161	codegen[out_index] = size;
162	out_index += 1;
163	self.codegen_freq[size] += 1;
164	}
165	// Set up invariant for next time through the loop.
166	size = next_size;
167	count = 1;
168	}
169	// Marker indicating the end of the codegen.
170	codegen[out_index] = end_code_mark;
171	}
172
173	const DynamicSize = struct {
174	size: u32,
175	num_codegens: u32,
176	};
177
178	// dynamicSize returns the size of dynamically encoded data in bits.
179	fn dynamicSize(
180	self: *Self,
181	lit_enc: *hc.LiteralEncoder, // literal encoder
182	dist_enc: *hc.DistanceEncoder, // distance encoder
183	extra_bits: u32,
184	) DynamicSize {
185	var num_codegens = self.codegen_freq.len;
186	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
187	num_codegens -= 1;
188	}
189	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
190	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
191	self.codegen_freq[16] * 2 +
192	self.codegen_freq[17] * 3 +
193	self.codegen_freq[18] * 7;
194	const size = header +
195	lit_enc.bitLength(&self.literal_freq) +
196	dist_enc.bitLength(&self.distance_freq) +
197	extra_bits;
198
199	return DynamicSize{
200	.size = @as(u32, @intCast(size)),
201	.num_codegens = @as(u32, @intCast(num_codegens)),
202	};
203	}
204
205	// fixedSize returns the size of dynamically encoded data in bits.
206	fn fixedSize(self: *Self, extra_bits: u32) u32 {
207	return 3 +
208	self.fixed_literal_encoding.bitLength(&self.literal_freq) +
209	self.fixed_distance_encoding.bitLength(&self.distance_freq) +
210	extra_bits;
211	}
212
213	const StoredSize = struct {
214	size: u32,
215	storable: bool,
216	};
217
218	// storedSizeFits calculates the stored size, including header.
219	// The function returns the size in bits and whether the block
220	// fits inside a single block.
221	fn storedSizeFits(in: ?[]const u8) StoredSize {
222	if (in == null) {
223	return .{ .size = 0, .storable = false };
224	}
225	if (in.?.len <= consts.max_store_block_size) {
226	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
227	}
228	return .{ .size = 0, .storable = false };
229	}
230
231	// Write the header of a dynamic Huffman block to the output stream.
232	//
233	// num_literals: The number of literals specified in codegen
234	// num_distances: The number of distances specified in codegen
235	// num_codegens: The number of codegens used in codegen
236	// eof: Is it the end-of-file? (end of stream)
237	fn dynamicHeader(
238	self: *Self,
239	num_literals: u32,
240	num_distances: u32,
241	num_codegens: u32,
242	eof: bool,
243	) Error!void {
244	const first_bits: u32 = if (eof) 5 else 4;
245	try self.bit_writer.writeBits(first_bits, 3);
246	try self.bit_writer.writeBits(num_literals - 257, 5);
247	try self.bit_writer.writeBits(num_distances - 1, 5);
248	try self.bit_writer.writeBits(num_codegens - 4, 4);
249
250	var i: u32 = 0;
251	while (i < num_codegens) : (i += 1) {
252	const value = self.codegen_encoding.codes[codegen_order[i]].len;
253	try self.bit_writer.writeBits(value, 3);
254	}
255
256	i = 0;
257	while (true) {
258	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
259	i += 1;
260	if (code_word == end_code_mark) {
261	break;
262	}
263	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
264
265	switch (code_word) {
266	16 => {
267	try self.bit_writer.writeBits(self.codegen[i], 2);
268	i += 1;
269	},
270	17 => {
271	try self.bit_writer.writeBits(self.codegen[i], 3);
272	i += 1;
273	},
274	18 => {
275	try self.bit_writer.writeBits(self.codegen[i], 7);
276	i += 1;
277	},
278	else => {},
279	}
280	}
281	}
282
283	fn storedHeader(self: *Self, length: usize, eof: bool) Error!void {
284	assert(length <= 65535);
285	const flag: u32 = if (eof) 1 else 0;
286	try self.bit_writer.writeBits(flag, 3);
287	try self.flush();
288	const l: u16 = @intCast(length);
289	try self.bit_writer.writeBits(l, 16);
290	try self.bit_writer.writeBits(~l, 16);
291	}
292
293	fn fixedHeader(self: *Self, eof: bool) Error!void {
294	// Indicate that we are a fixed Huffman block
295	var value: u32 = 2;
296	if (eof) {
297	value = 3;
298	}
299	try self.bit_writer.writeBits(value, 3);
300	}
301
302	// Write a block of tokens with the smallest encoding. Will choose block type.
303	// The original input can be supplied, and if the huffman encoded data
304	// is larger than the original bytes, the data will be written as a
305	// stored block.
306	// If the input is null, the tokens will always be Huffman encoded.
307	pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void {
308	const lit_and_dist = self.indexTokens(tokens);
309	const num_literals = lit_and_dist.num_literals;
310	const num_distances = lit_and_dist.num_distances;
311
312	var extra_bits: u32 = 0;
313	const ret = storedSizeFits(input);
314	const stored_size = ret.size;
315	const storable = ret.storable;
316
317	if (storable) {
318	// We only bother calculating the costs of the extra bits required by
319	// the length of distance fields (which will be the same for both fixed
320	// and dynamic encoding), if we need to compare those two encodings
321	// against stored encoding.
322	var length_code: u16 = Token.length_codes_start + 8;
323	while (length_code < num_literals) : (length_code += 1) {
324	// First eight length codes have extra size = 0.
325	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
326	@as(u32, @intCast(Token.lengthExtraBits(length_code)));
327	}
328	var distance_code: u16 = 4;
329	while (distance_code < num_distances) : (distance_code += 1) {
330	// First four distance codes have extra size = 0.
331	extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
332	@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
333	}
334	}
335
336	// Figure out smallest code.
337	// Fixed Huffman baseline.
338	var literal_encoding = &self.fixed_literal_encoding;
339	var distance_encoding = &self.fixed_distance_encoding;
340	var size = self.fixedSize(extra_bits);
341
342	// Dynamic Huffman?
343	var num_codegens: u32 = 0;
344
345	// Generate codegen and codegenFrequencies, which indicates how to encode
346	// the literal_encoding and the distance_encoding.
347	self.generateCodegen(
348	num_literals,
349	num_distances,
350	&self.literal_encoding,
351	&self.distance_encoding,
352	);
353	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
354	const dynamic_size = self.dynamicSize(
355	&self.literal_encoding,
356	&self.distance_encoding,
357	extra_bits,
358	);
359	const dyn_size = dynamic_size.size;
360	num_codegens = dynamic_size.num_codegens;
361
362	if (dyn_size < size) {
363	size = dyn_size;
364	literal_encoding = &self.literal_encoding;
365	distance_encoding = &self.distance_encoding;
366	}
367
368	// Stored bytes?
369	if (storable and stored_size < size) {
370	try self.storedBlock(input.?, eof);
371	return;
372	}
373
374	// Huffman.
375	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
376	try self.fixedHeader(eof);
377	} else {
378	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
379	}
380
381	// Write the tokens.
382	try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
383	}
384
385	pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void {
386	try self.storedHeader(input.len, eof);
387	try self.bit_writer.writeBytes(input);
388	}
389
390	// writeBlockDynamic encodes a block using a dynamic Huffman table.
391	// This should be used if the symbols used have a disproportionate
392	// histogram distribution.
393	// If input is supplied and the compression savings are below 1/16th of the
394	// input size the block is stored.
395	fn dynamicBlock(
396	self: *Self,
397	tokens: []const Token,
398	eof: bool,
399	input: ?[]const u8,
400	) Error!void {
401	const total_tokens = self.indexTokens(tokens);
402	const num_literals = total_tokens.num_literals;
403	const num_distances = total_tokens.num_distances;
404
405	// Generate codegen and codegenFrequencies, which indicates how to encode
406	// the literal_encoding and the distance_encoding.
407	self.generateCodegen(
408	num_literals,
409	num_distances,
410	&self.literal_encoding,
411	&self.distance_encoding,
412	);
413	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
414	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
415	const size = dynamic_size.size;
416	const num_codegens = dynamic_size.num_codegens;
417
418	// Store bytes, if we don't get a reasonable improvement.
419
420	const stored_size = storedSizeFits(input);
421	const ssize = stored_size.size;
422	const storable = stored_size.storable;
423	if (storable and ssize < (size + (size >> 4))) {
424	try self.storedBlock(input.?, eof);
425	return;
426	}
427
428	// Write Huffman table.
429	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
430
431	// Write the tokens.
432	try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
433	}
434
435	const TotalIndexedTokens = struct {
436	num_literals: u32,
437	num_distances: u32,
438	};
439
440	// Indexes a slice of tokens followed by an end_block_marker, and updates
441	// literal_freq and distance_freq, and generates literal_encoding
442	// and distance_encoding.
443	// The number of literal and distance tokens is returned.
444	fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
445	var num_literals: u32 = 0;
446	var num_distances: u32 = 0;
447
448	for (self.literal_freq, 0..) \|_, i\| {
449	self.literal_freq[i] = 0;
450	}
451	for (self.distance_freq, 0..) \|_, i\| {
452	self.distance_freq[i] = 0;
453	}
454
455	for (tokens) \|t\| {
456	if (t.kind == Token.Kind.literal) {
457	self.literal_freq[t.literal()] += 1;
458	continue;
459	}
460	self.literal_freq[t.lengthCode()] += 1;
461	self.distance_freq[t.distanceCode()] += 1;
462	}
463	// add end_block_marker token at the end
464	self.literal_freq[consts.end_block_marker] += 1;
465
466	// get the number of literals
467	num_literals = @as(u32, @intCast(self.literal_freq.len));
468	while (self.literal_freq[num_literals - 1] == 0) {
469	num_literals -= 1;
470	}
471	// get the number of distances
472	num_distances = @as(u32, @intCast(self.distance_freq.len));
473	while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
474	num_distances -= 1;
475	}
476	if (num_distances == 0) {
477	// We haven't found a single match. If we want to go with the dynamic encoding,
478	// we should count at least one distance to be sure that the distance huffman tree could be encoded.
479	self.distance_freq[0] = 1;
480	num_distances = 1;
481	}
482	self.literal_encoding.generate(&self.literal_freq, 15);
483	self.distance_encoding.generate(&self.distance_freq, 15);
484	return TotalIndexedTokens{
485	.num_literals = num_literals,
486	.num_distances = num_distances,
487	};
488	}
489
490	// Writes a slice of tokens to the output followed by and end_block_marker.
491	// codes for literal and distance encoding must be supplied.
492	fn writeTokens(
493	self: *Self,
494	tokens: []const Token,
495	le_codes: []hc.HuffCode,
496	oe_codes: []hc.HuffCode,
497	) Error!void {
498	for (tokens) \|t\| {
499	if (t.kind == Token.Kind.literal) {
500	try self.writeCode(le_codes[t.literal()]);
501	continue;
502	}
503
504	// Write the length
505	const le = t.lengthEncoding();
506	try self.writeCode(le_codes[le.code]);
507	if (le.extra_bits > 0) {
508	try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
509	}
510
511	// Write the distance
512	const oe = t.distanceEncoding();
513	try self.writeCode(oe_codes[oe.code]);
514	if (oe.extra_bits > 0) {
515	try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
516	}
517	}
518	// add end_block_marker at the end
519	try self.writeCode(le_codes[consts.end_block_marker]);
520	}
521
522	// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
523	// if the results only gains very little from compression.
524	pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void {
525	// Add everything as literals
526	histogram(input, &self.literal_freq);
527
528	self.literal_freq[consts.end_block_marker] = 1;
529
530	const num_literals = consts.end_block_marker + 1;
531	self.distance_freq[0] = 1;
532	const num_distances = 1;
533
534	self.literal_encoding.generate(&self.literal_freq, 15);
535
536	// Figure out smallest code.
537	// Always use dynamic Huffman or Store
538	var num_codegens: u32 = 0;
539
540	// Generate codegen and codegenFrequencies, which indicates how to encode
541	// the literal_encoding and the distance_encoding.
542	self.generateCodegen(
543	num_literals,
544	num_distances,
545	&self.literal_encoding,
546	&self.huff_distance,
547	);
548	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
549	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
550	const size = dynamic_size.size;
551	num_codegens = dynamic_size.num_codegens;
552
553	// Store bytes, if we don't get a reasonable improvement.
554	const stored_size_ret = storedSizeFits(input);
555	const ssize = stored_size_ret.size;
556	const storable = stored_size_ret.storable;
557
558	if (storable and ssize < (size + (size >> 4))) {
559	try self.storedBlock(input, eof);
560	return;
561	}
562
563	// Huffman.
564	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
565	const encoding = self.literal_encoding.codes[0..257];
566
567	for (input) \|t\| {
568	const c = encoding[t];
569	try self.bit_writer.writeBits(c.code, c.len);
570	}
571	try self.writeCode(encoding[consts.end_block_marker]);
572	}
573
574	// histogram accumulates a histogram of b in h.
575	fn histogram(b: []const u8, h: *[286]u16) void {
576	// Clear histogram
577	for (h, 0..) \|_, i\| {
578	h[i] = 0;
579	}
580
581	var lh = h.*[0..256];
582	for (b) \|t\| {
583	lh[t] += 1;
584	}
585	}
586	};
587	}
588
589	// tests
590	const expect = std.testing.expect;
591	const fmt = std.fmt;
592	const testing = std.testing;
593	const ArrayList = std.ArrayList;
594
595	const TestCase = @import("testdata/block_writer.zig").TestCase;
596	const testCases = @import("testdata/block_writer.zig").testCases;
597
598	// tests if the writeBlock encoding has changed.
599	test "flate.BlockWriter write" {
600	inline for (0..testCases.len) \|i\| {
601	try testBlock(testCases[i], .write_block);
602	}
603	}
604
605	// tests if the writeBlockDynamic encoding has changed.
606	test "flate.BlockWriter dynamicBlock" {
607	inline for (0..testCases.len) \|i\| {
608	try testBlock(testCases[i], .write_dyn_block);
609	}
610	}
611
612	test "flate.BlockWriter huffmanBlock" {
613	inline for (0..testCases.len) \|i\| {
614	try testBlock(testCases[i], .write_huffman_block);
615	}
616	try testBlock(.{
617	.tokens = &[_]Token{},
618	.input = "huffman-rand-max.input",
619	.want = "huffman-rand-max.{s}.expect",
620	}, .write_huffman_block);
621	}
622
623	const TestFn = enum {
624	write_block,
625	write_dyn_block, // write dynamic block
626	write_huffman_block,
627
628	fn to_s(self: TestFn) []const u8 {
629	return switch (self) {
630	.write_block => "wb",
631	.write_dyn_block => "dyn",
632	.write_huffman_block => "huff",
633	};
634	}
635
636	fn write(
637	comptime self: TestFn,
638	bw: anytype,
639	tok: []const Token,
640	input: ?[]const u8,
641	final: bool,
642	) !void {
643	switch (self) {
644	.write_block => try bw.write(tok, final, input),
645	.write_dyn_block => try bw.dynamicBlock(tok, final, input),
646	.write_huffman_block => try bw.huffmanBlock(input.?, final),
647	}
648	try bw.flush();
649	}
650	};
651
652	// testBlock tests a block against its references
653	//
654	// size
655	// 64K [file-name].input - input non compressed file
656	// 8.1K [file-name].golden -
657	// 78 [file-name].dyn.expect - output with writeBlockDynamic
658	// 78 [file-name].wb.expect - output with writeBlock
659	// 8.1K [file-name].huff.expect - output with writeBlockHuff
660	// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
661	// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
662	//
663	// wb - writeBlock
664	// dyn - writeBlockDynamic
665	// huff - writeBlockHuff
666	//
667	fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
668	if (tc.input.len != 0 and tc.want.len != 0) {
669	const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
670	const input = @embedFile("testdata/block_writer/" ++ tc.input);
671	const want = @embedFile("testdata/block_writer/" ++ want_name);
672	try testWriteBlock(tfn, input, want, tc.tokens);
673	}
674
675	if (tfn == .write_huffman_block) {
676	return;
677	}
678
679	const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
680	const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
681	try testWriteBlock(tfn, null, want, tc.tokens);
682	}
683
684	// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
685	fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
686	var buf = ArrayList(u8).init(testing.allocator);
687	var bw = blockWriter(buf.writer());
688	try tfn.write(&bw, tokens, input, false);
689	var got = buf.items;
690	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
691	try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
692	//
693	// Test if the writer produces the same output after reset.
694	buf.deinit();
695	buf = ArrayList(u8).init(testing.allocator);
696	defer buf.deinit();
697	bw.setWriter(buf.writer());
698
699	try tfn.write(&bw, tokens, input, true);
700	try bw.flush();
701	got = buf.items;
702
703	try expect(got[0] & 1 == 1); // bfinal is set
704	buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
705	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
706	}

lib/std/compress/flate/consts.zig created+49

...	@@ -0,0 +1,49 @@
1	pub const deflate = struct {
2	// Number of tokens to accumlate in deflate before starting block encoding.
3	//
4	// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
5	// 8 and max 9 that gives 14 or 15 bits.
6	pub const tokens = 1 << 15;
7	};
8
9	pub const match = struct {
10	pub const base_length = 3; // smallest match length per the RFC section 3.2.5
11	pub const min_length = 4; // min length used in this algorithm
12	pub const max_length = 258;
13
14	pub const min_distance = 1;
15	pub const max_distance = 32768;
16	};
17
18	pub const history = struct {
19	pub const len = match.max_distance;
20	};
21
22	pub const lookup = struct {
23	pub const bits = 15;
24	pub const len = 1 << bits;
25	pub const shift = 32 - bits;
26	};
27
28	pub const huffman = struct {
29	// The odd order in which the codegen code sizes are written.
30	pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
31	// The number of codegen codes.
32	pub const codegen_code_count = 19;
33
34	// The largest distance code.
35	pub const distance_code_count = 30;
36
37	// Maximum number of literals.
38	pub const max_num_lit = 286;
39
40	// Max number of frequencies used for a Huffman Code
41	// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
42	// The largest of these is max_num_lit.
43	pub const max_num_frequencies = max_num_lit;
44
45	// Biggest block size for uncompressed block.
46	pub const max_store_block_size = 65535;
47	// The special code used to mark the end of a block.
48	pub const end_block_marker = 256;
49	};

lib/std/compress/flate/container.zig created+205

...	@@ -0,0 +1,205 @@
1	const std = @import("std");
2
3	/// Container of the deflate bit stream body. Container adds header before
4	/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
5	/// no footer, raw bit stream).
6	///
7	/// Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
8	/// addler 32 checksum.
9	///
10	/// Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
11	/// crc32 checksum and 4 bytes of uncompressed data length.
12	///
13	///
14	/// rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
15	/// rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
16	///
17	pub const Container = enum {
18	raw, // no header or footer
19	gzip, // gzip header and footer
20	zlib, // zlib header and footer
21
22	pub fn size(w: Container) usize {
23	return headerSize(w) + footerSize(w);
24	}
25
26	pub fn headerSize(w: Container) usize {
27	return switch (w) {
28	.gzip => 10,
29	.zlib => 2,
30	.raw => 0,
31	};
32	}
33
34	pub fn footerSize(w: Container) usize {
35	return switch (w) {
36	.gzip => 8,
37	.zlib => 4,
38	.raw => 0,
39	};
40	}
41
42	pub const list = [_]Container{ .raw, .gzip, .zlib };
43
44	pub const Error = error{
45	BadGzipHeader,
46	BadZlibHeader,
47	WrongGzipChecksum,
48	WrongGzipSize,
49	WrongZlibChecksum,
50	};
51
52	pub fn writeHeader(comptime wrap: Container, writer: anytype) !void {
53	switch (wrap) {
54	.gzip => {
55	// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
56	// - ID1 (IDentification 1), always 0x1f
57	// - ID2 (IDentification 2), always 0x8b
58	// - CM (Compression Method), always 8 = deflate
59	// - FLG (Flags), all set to 0
60	// - 4 bytes, MTIME (Modification time), not used, all set to zero
61	// - XFL (eXtra FLags), all set to zero
62	// - OS (Operating System), 03 = Unix
63	const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 };
64	try writer.writeAll(&gzipHeader);
65	},
66	.zlib => {
67	// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
68	// 1st byte:
69	// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
70	// - The next four bits is the CM (compression method), which is 8 for deflate.
71	// 2nd byte:
72	// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
73	// - The next bit, FDICT, is set if a dictionary is given.
74	// - The final five FCHECK bits form a mod-31 checksum.
75	//
76	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
77	const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 };
78	try writer.writeAll(&zlibHeader);
79	},
80	.raw => {},
81	}
82	}
83
84	pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void {
85	var bits: [4]u8 = undefined;
86	switch (wrap) {
87	.gzip => {
88	// GZIP 8 bytes footer
89	// - 4 bytes, CRC32 (CRC-32)
90	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
91	std.mem.writeInt(u32, &bits, hasher.chksum(), .little);
92	try writer.writeAll(&bits);
93
94	std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little);
95	try writer.writeAll(&bits);
96	},
97	.zlib => {
98	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
99	// 4 bytes of ADLER32 (Adler-32 checksum)
100	// Checksum value of the uncompressed data (excluding any
101	// dictionary data) computed according to Adler-32
102	// algorithm.
103	std.mem.writeInt(u32, &bits, hasher.chksum(), .big);
104	try writer.writeAll(&bits);
105	},
106	.raw => {},
107	}
108	}
109
110	pub fn parseHeader(comptime wrap: Container, reader: anytype) !void {
111	switch (wrap) {
112	.gzip => try parseGzipHeader(reader),
113	.zlib => try parseZlibHeader(reader),
114	.raw => {},
115	}
116	}
117
118	fn parseGzipHeader(reader: anytype) !void {
119	const magic1 = try reader.read(u8);
120	const magic2 = try reader.read(u8);
121	const method = try reader.read(u8);
122	const flags = try reader.read(u8);
123	try reader.skipBytes(6); // mtime(4), xflags, os
124	if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
125	return error.BadGzipHeader;
126	// Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
127	if (flags != 0) {
128	if (flags & 0b0000_0100 != 0) { // FEXTRA
129	const extra_len = try reader.read(u16);
130	try reader.skipBytes(extra_len);
131	}
132	if (flags & 0b0000_1000 != 0) { // FNAME
133	try reader.skipStringZ();
134	}
135	if (flags & 0b0001_0000 != 0) { // FCOMMENT
136	try reader.skipStringZ();
137	}
138	if (flags & 0b0000_0010 != 0) { // FHCRC
139	try reader.skipBytes(2);
140	}
141	}
142	}
143
144	fn parseZlibHeader(reader: anytype) !void {
145	const cinfo_cm = try reader.read(u8);
146	_ = try reader.read(u8);
147	if (cinfo_cm != 0x78) {
148	return error.BadZlibHeader;
149	}
150	}
151
152	pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {
153	switch (wrap) {
154	.gzip => {
155	if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
156	if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
157	},
158	.zlib => {
159	const chksum: u32 = @byteSwap(hasher.chksum());
160	if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
161	},
162	.raw => {},
163	}
164	}
165
166	pub fn Hasher(comptime wrap: Container) type {
167	const HasherType = switch (wrap) {
168	.gzip => std.hash.Crc32,
169	.zlib => std.hash.Adler32,
170	.raw => struct {
171	pub fn init() @This() {
172	return .{};
173	}
174	},
175	};
176
177	return struct {
178	hasher: HasherType = HasherType.init(),
179	bytes: usize = 0,
180
181	const Self = @This();
182
183	pub fn update(self: *Self, buf: []const u8) void {
184	switch (wrap) {
185	.raw => {},
186	else => {
187	self.hasher.update(buf);
188	self.bytes += buf.len;
189	},
190	}
191	}
192
193	pub fn chksum(self: *Self) u32 {
194	switch (wrap) {
195	.raw => return 0,
196	else => return self.hasher.final(),
197	}
198	}
199
200	pub fn bytesRead(self: *Self) u32 {
201	return @truncate(self.bytes);
202	}
203	};
204	}
205	};

lib/std/compress/flate/deflate.zig created+783

...	@@ -0,0 +1,783 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4	const testing = std.testing;
5	const expect = testing.expect;
6	const print = std.debug.print;
7
8	const Token = @import("Token.zig");
9	const consts = @import("consts.zig");
10	const BlockWriter = @import("block_writer.zig").BlockWriter;
11	const Container = @import("container.zig").Container;
12	const SlidingWindow = @import("SlidingWindow.zig");
13	const Lookup = @import("Lookup.zig");
14
15	pub const Options = struct {
16	level: Level = .default,
17	};
18
19	/// Trades between speed and compression size.
20	/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
21	/// levels 1-3 are using different algorithm to perform faster but with less
22	/// compression. That is not implemented here.
23	pub const Level = enum(u4) {
24	// zig fmt: off
25	fast = 0xb, level_4 = 4,
26	level_5 = 5,
27	default = 0xc, level_6 = 6,
28	level_7 = 7,
29	level_8 = 8,
30	best = 0xd, level_9 = 9,
31	// zig fmt: on
32	};
33
34	/// Algorithm knobs for each level.
35	const LevelArgs = struct {
36	good: u16, // Do less lookups if we already have match of this length.
37	nice: u16, // Stop looking for better match if we found match with at least this length.
38	lazy: u16, // Don't do lazy match find if got match with at least this length.
39	chain: u16, // How many lookups for previous match to perform.
40
41	pub fn get(level: Level) LevelArgs {
42	// zig fmt: off
43	return switch (level) {
44	.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
45	.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
46	.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
47	.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
48	.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
49	.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
50	};
51	// zig fmt: on
52	}
53	};
54
55	/// Compress plain data from reader into compressed stream written to writer.
56	pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void {
57	var c = try compressor(container, writer, options);
58	try c.compress(reader);
59	try c.finish();
60	}
61
62	/// Create compressor for writer type.
63	pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor(
64	container,
65	@TypeOf(writer),
66	) {
67	return try Compressor(container, @TypeOf(writer)).init(writer, options);
68	}
69
70	/// Compressor type.
71	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
72	const TokenWriterType = BlockWriter(WriterType);
73	return Deflate(container, WriterType, TokenWriterType);
74	}
75
76	/// Default compression algorithm. Has two steps: tokenization and token
77	/// encoding.
78	///
79	/// Tokenization takes uncompressed input stream and produces list of tokens.
80	/// Each token can be literal (byte of data) or match (backrefernce to previous
81	/// data with length and distance). Tokenization accumulators 32K tokens, when
82	/// full or `flush` is called tokens are passed to the `block_writer`. Level
83	/// defines how hard (how slow) it tries to find match.
84	///
85	/// Block writer will decide which type of deflate block to write (stored, fixed,
86	/// dynamic) and encode tokens to the output byte stream. Client has to call
87	/// `finish` to write block with the final bit set.
88	///
89	/// Container defines type of header and footer which can be gzip, zlib or raw.
90	/// They all share same deflate body. Raw has no header or footer just deflate
91	/// body.
92	///
93	/// Compression algorithm explained in rfc-1951 (slightly edited for this case):
94	///
95	/// The compressor uses a chained hash table `lookup` to find duplicated
96	/// strings, using a hash function that operates on 4-byte sequences. At any
97	/// given point during compression, let XYZW be the next 4 input bytes
98	/// (lookahead) to be examined (not necessarily all different, of course).
99	/// First, the compressor examines the hash chain for XYZW. If the chain is
100	/// empty, the compressor simply writes out X as a literal byte and advances
101	/// one byte in the input. If the hash chain is not empty, indicating that the
102	/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
103	/// hash function value) has occurred recently, the compressor compares all
104	/// strings on the XYZW hash chain with the actual input data sequence
105	/// starting at the current point, and selects the longest match.
106	///
107	/// To improve overall compression, the compressor defers the selection of
108	/// matches ("lazy matching"): after a match of length N has been found, the
109	/// compressor searches for a longer match starting at the next input byte. If
110	/// it finds a longer match, it truncates the previous match to a length of
111	/// one (thus producing a single literal byte) and then emits the longer
112	/// match. Otherwise, it emits the original match, and, as described above,
113	/// advances N bytes before continuing.
114	///
115	///
116	/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
117	///
118	/// Deflate function accepts BlockWriterType so we can change that in test to test
119	/// just tokenization part.
120	///
121	fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type {
122	return struct {
123	lookup: Lookup = .{},
124	win: SlidingWindow = .{},
125	tokens: Tokens = .{},
126	wrt: WriterType,
127	block_writer: BlockWriterType,
128	level: LevelArgs,
129	hasher: container.Hasher() = .{},
130
131	// Match and literal at the previous position.
132	// Used for lazy match finding in processWindow.
133	prev_match: ?Token = null,
134	prev_literal: ?u8 = null,
135
136	const Self = @This();
137
138	pub fn init(wrt: WriterType, options: Options) !Self {
139	const self = Self{
140	.wrt = wrt,
141	.block_writer = BlockWriterType.init(wrt),
142	.level = LevelArgs.get(options.level),
143	};
144	try container.writeHeader(self.wrt);
145	return self;
146	}
147
148	const FlushOption = enum { none, flush, final };
149
150	// Process data in window and create tokens. If token buffer is full
151	// flush tokens to the token writer. In the case of `flush` or `final`
152	// option it will process all data from the window. In the `none` case
153	// it will preserve some data for the next match.
154	fn tokenize(self: *Self, flush_opt: FlushOption) !void {
155	// flush - process all data from window
156	const should_flush = (flush_opt != .none);
157
158	// While there is data in active lookahead buffer.
159	while (self.win.activeLookahead(should_flush)) \|lh\| {
160	var step: u16 = 1; // 1 in the case of literal, match length otherwise
161	const pos: u16 = self.win.pos();
162	const literal = lh[0]; // literal at current position
163	const min_len: u16 = if (self.prev_match) \|m\| m.length() else 0;
164
165	// Try to find match at least min_len long.
166	if (self.findMatch(pos, lh, min_len)) \|match\| {
167	// Found better match than previous.
168	try self.addPrevLiteral();
169
170	// Is found match length good enough?
171	if (match.length() >= self.level.lazy) {
172	// Don't try to lazy find better match, use this.
173	step = try self.addMatch(match);
174	} else {
175	// Store this match.
176	self.prev_literal = literal;
177	self.prev_match = match;
178	}
179	} else {
180	// There is no better match at current pos then it was previous.
181	// Write previous match or literal.
182	if (self.prev_match) \|m\| {
183	// Write match from previous position.
184	step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
185	} else {
186	// No match at previous postition.
187	// Write previous literal if any, and remember this literal.
188	try self.addPrevLiteral();
189	self.prev_literal = literal;
190	}
191	}
192	// Advance window and add hashes.
193	self.windowAdvance(step, lh, pos);
194	}
195
196	if (should_flush) {
197	// In the case of flushing, last few lookahead buffers were smaller then min match len.
198	// So only last literal can be unwritten.
199	assert(self.prev_match == null);
200	try self.addPrevLiteral();
201	self.prev_literal = null;
202
203	try self.flushTokens(flush_opt);
204	}
205	}
206
207	fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void {
208	// current position is already added in findMatch
209	self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
210	self.win.advance(step);
211	}
212
213	// Add previous literal (if any) to the tokens list.
214	fn addPrevLiteral(self: *Self) !void {
215	if (self.prev_literal) \|l\| try self.addToken(Token.initLiteral(l));
216	}
217
218	// Add match to the tokens list, reset prev pointers.
219	// Returns length of the added match.
220	fn addMatch(self: *Self, m: Token) !u16 {
221	try self.addToken(m);
222	self.prev_literal = null;
223	self.prev_match = null;
224	return m.length();
225	}
226
227	fn addToken(self: *Self, token: Token) !void {
228	self.tokens.add(token);
229	if (self.tokens.full()) try self.flushTokens(.none);
230	}
231
232	// Finds largest match in the history window with the data at current pos.
233	fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token {
234	var len: u16 = min_len;
235	// Previous location with the same hash (same 4 bytes).
236	var prev_pos = self.lookup.add(lh, pos);
237	// Last found match.
238	var match: ?Token = null;
239
240	// How much back-references to try, performance knob.
241	var chain: usize = self.level.chain;
242	if (len >= self.level.good) {
243	// If we've got a match that's good enough, only look in 1/4 the chain.
244	chain >>= 2;
245	}
246
247	// Hot path loop!
248	while (prev_pos > 0 and chain > 0) : (chain -= 1) {
249	const distance = pos - prev_pos;
250	if (distance > consts.match.max_distance)
251	break;
252
253	const new_len = self.win.match(prev_pos, pos, len);
254	if (new_len > len) {
255	match = Token.initMatch(@intCast(distance), new_len);
256	if (new_len >= self.level.nice) {
257	// The match is good enough that we don't try to find a better one.
258	return match;
259	}
260	len = new_len;
261	}
262	prev_pos = self.lookup.prev(prev_pos);
263	}
264
265	return match;
266	}
267
268	fn flushTokens(self: *Self, flush_opt: FlushOption) !void {
269	// Pass tokens to the token writer
270	try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
271	// Stored block ensures byte aligment.
272	// It has 3 bits (final, block_type) and then padding until byte boundary.
273	// After that everyting is aligned to the boundary in the stored block.
274	// Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
275	// Last 4 bytes are byte aligned.
276	if (flush_opt == .flush) {
277	try self.block_writer.storedBlock("", false);
278	}
279	if (flush_opt != .none) {
280	// Safe to call only when byte aligned or it is OK to add
281	// padding bits (on last byte of the final block).
282	try self.block_writer.flush();
283	}
284	// Reset internal tokens store.
285	self.tokens.reset();
286	// Notify win that tokens are flushed.
287	self.win.flush();
288	}
289
290	// Slide win and if needed lookup tables.
291	fn slide(self: *Self) void {
292	const n = self.win.slide();
293	self.lookup.slide(n);
294	}
295
296	/// Compresses as much data as possible, stops when the reader becomes
297	/// empty. It will introduce some output latency (reading input without
298	/// producing all output) because some data are still in internal
299	/// buffers.
300	///
301	/// It is up to the caller to call flush (if needed) or finish (required)
302	/// when is need to output any pending data or complete stream.
303	///
304	pub fn compress(self: *Self, reader: anytype) !void {
305	while (true) {
306	// Fill window from reader
307	const buf = self.win.writable();
308	if (buf.len == 0) {
309	try self.tokenize(.none);
310	self.slide();
311	continue;
312	}
313	const n = try reader.readAll(buf);
314	self.hasher.update(buf[0..n]);
315	self.win.written(n);
316	// Process window
317	try self.tokenize(.none);
318	// Exit when no more data in reader
319	if (n < buf.len) break;
320	}
321	}
322
323	/// Flushes internal buffers to the output writer. Outputs empty stored
324	/// block to sync bit stream to the byte boundary, so that the
325	/// decompressor can get all input data available so far.
326	///
327	/// It is useful mainly in compressed network protocols, to ensure that
328	/// deflate bit stream can be used as byte stream. May degrade
329	/// compression so it should be used only when necessary.
330	///
331	/// Completes the current deflate block and follows it with an empty
332	/// stored block that is three zero bits plus filler bits to the next
333	/// byte, followed by four bytes (00 00 ff ff).
334	///
335	pub fn flush(self: *Self) !void {
336	try self.tokenize(.flush);
337	}
338
339	/// Completes deflate bit stream by writing any pending data as deflate
340	/// final deflate block. HAS to be called once all data are written to
341	/// the compressor as a signal that next block has to have final bit
342	/// set.
343	///
344	pub fn finish(self: *Self) !void {
345	try self.tokenize(.final);
346	try container.writeFooter(&self.hasher, self.wrt);
347	}
348
349	/// Use another writer while preserving history. Most probably flush
350	/// should be called on old writer before setting new.
351	pub fn setWriter(self: *Self, new_writer: WriterType) void {
352	self.block_writer.setWriter(new_writer);
353	self.wrt = new_writer;
354	}
355
356	// Writer interface
357
358	pub const Writer = io.Writer(*Self, Error, write);
359	pub const Error = BlockWriterType.Error;
360
361	/// Write `input` of uncompressed data.
362	/// See compress.
363	pub fn write(self: *Self, input: []const u8) !usize {
364	var fbs = io.fixedBufferStream(input);
365	try self.compress(fbs.reader());
366	return input.len;
367	}
368
369	pub fn writer(self: *Self) Writer {
370	return .{ .context = self };
371	}
372	};
373	}
374
375	// Tokens store
376	const Tokens = struct {
377	list: [consts.deflate.tokens]Token = undefined,
378	pos: usize = 0,
379
380	fn add(self: *Tokens, t: Token) void {
381	self.list[self.pos] = t;
382	self.pos += 1;
383	}
384
385	fn full(self: *Tokens) bool {
386	return self.pos == self.list.len;
387	}
388
389	fn reset(self: *Tokens) void {
390	self.pos = 0;
391	}
392
393	fn tokens(self: *Tokens) []const Token {
394	return self.list[0..self.pos];
395	}
396	};
397
398	/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
399	/// only performs Huffman entropy encoding. Results in faster compression, much
400	/// less memory requirements during compression but bigger compressed sizes.
401	pub const huffman = struct {
402	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
403	var c = try huffman.compressor(container, writer);
404	try c.compress(reader);
405	try c.finish();
406	}
407
408	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
409	return SimpleCompressor(.huffman, container, WriterType);
410	}
411
412	pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) {
413	return try huffman.Compressor(container, @TypeOf(writer)).init(writer);
414	}
415	};
416
417	/// Creates store blocks only. Data are not compressed only packed into deflate
418	/// store blocks. That adds 9 bytes of header for each block. Max stored block
419	/// size is 64K. Block is emitted when flush is called on on finish.
420	pub const store = struct {
421	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
422	var c = try store.compressor(container, writer);
423	try c.compress(reader);
424	try c.finish();
425	}
426
427	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
428	return SimpleCompressor(.store, container, WriterType);
429	}
430
431	pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
432	return try store.Compressor(container, @TypeOf(writer)).init(writer);
433	}
434	};
435
436	const SimpleCompressorKind = enum {
437	huffman,
438	store,
439	};
440
441	fn simpleCompressor(
442	comptime kind: SimpleCompressorKind,
443	comptime container: Container,
444	writer: anytype,
445	) !SimpleCompressor(kind, container, @TypeOf(writer)) {
446	return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
447	}
448
449	fn SimpleCompressor(
450	comptime kind: SimpleCompressorKind,
451	comptime container: Container,
452	comptime WriterType: type,
453	) type {
454	const BlockWriterType = BlockWriter(WriterType);
455	return struct {
456	buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
457	wp: usize = 0,
458
459	wrt: WriterType,
460	block_writer: BlockWriterType,
461	hasher: container.Hasher() = .{},
462
463	const Self = @This();
464
465	pub fn init(wrt: WriterType) !Self {
466	const self = Self{
467	.wrt = wrt,
468	.block_writer = BlockWriterType.init(wrt),
469	};
470	try container.writeHeader(self.wrt);
471	return self;
472	}
473
474	pub fn flush(self: *Self) !void {
475	try self.flushBuffer(false);
476	try self.block_writer.storedBlock("", false);
477	try self.block_writer.flush();
478	}
479
480	pub fn finish(self: *Self) !void {
481	try self.flushBuffer(true);
482	try self.block_writer.flush();
483	try container.writeFooter(&self.hasher, self.wrt);
484	}
485
486	fn flushBuffer(self: *Self, final: bool) !void {
487	const buf = self.buffer[0..self.wp];
488	switch (kind) {
489	.huffman => try self.block_writer.huffmanBlock(buf, final),
490	.store => try self.block_writer.storedBlock(buf, final),
491	}
492	self.wp = 0;
493	}
494
495	// Writes all data from the input reader of uncompressed data.
496	// It is up to the caller to call flush or finish if there is need to
497	// output compressed blocks.
498	pub fn compress(self: *Self, reader: anytype) !void {
499	while (true) {
500	// read from rdr into buffer
501	const buf = self.buffer[self.wp..];
502	if (buf.len == 0) {
503	try self.flushBuffer(false);
504	continue;
505	}
506	const n = try reader.readAll(buf);
507	self.hasher.update(buf[0..n]);
508	self.wp += n;
509	if (n < buf.len) break; // no more data in reader
510	}
511	}
512
513	// Writer interface
514
515	pub const Writer = io.Writer(*Self, Error, write);
516	pub const Error = BlockWriterType.Error;
517
518	// Write `input` of uncompressed data.
519	pub fn write(self: *Self, input: []const u8) !usize {
520	var fbs = io.fixedBufferStream(input);
521	try self.compress(fbs.reader());
522	return input.len;
523	}
524
525	pub fn writer(self: *Self) Writer {
526	return .{ .context = self };
527	}
528	};
529	}
530
531	test "flate.Deflate tokenization" {
532	const L = Token.initLiteral;
533	const M = Token.initMatch;
534
535	const cases = [_]struct {
536	data: []const u8,
537	tokens: []const Token,
538	}{
539	.{
540	.data = "Blah blah blah blah blah!",
541	.tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
542	},
543	.{
544	.data = "ABCDEABCD ABCDEABCD",
545	.tokens = &[_]Token{
546	L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
547	L('A'), M(10, 8),
548	},
549	},
550	};
551
552	for (cases) \|c\| {
553	inline for (Container.list) \|container\| { // for each wrapping
554	var cw = io.countingWriter(io.null_writer);
555	const cww = cw.writer();
556	var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
557
558	_ = try df.write(c.data);
559	try df.flush();
560
561	// df.token_writer.show();
562	try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
563	try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
564
565	try testing.expectEqual(container.headerSize(), cw.bytes_written);
566	try df.finish();
567	try testing.expectEqual(container.size(), cw.bytes_written);
568	}
569	}
570	}
571
572	// Tests that tokens writen are equal to expected token list.
573	const TestTokenWriter = struct {
574	const Self = @This();
575	//expected: []const Token,
576	pos: usize = 0,
577	actual: [1024]Token = undefined,
578
579	pub fn init(_: anytype) Self {
580	return .{};
581	}
582	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
583	for (tokens) \|t\| {
584	self.actual[self.pos] = t;
585	self.pos += 1;
586	}
587	}
588
589	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
590
591	pub fn get(self: *Self) []Token {
592	return self.actual[0..self.pos];
593	}
594
595	pub fn show(self: *Self) void {
596	print("\n", .{});
597	for (self.get()) \|t\| {
598	t.show();
599	}
600	}
601
602	pub fn flush(_: *Self) !void {}
603	};
604
605	test "flate.Deflate struct sizes" {
606	try expect(@sizeOf(Token) == 4);
607
608	// list: (1 << 15) * 4 = 128k + pos: 8
609	const tokens_size = 128 * 1024 + 8;
610	try expect(@sizeOf(Tokens) == tokens_size);
611
612	// head: (1 << 15) * 2 = 64k, chain: (32768 * 2) * 2 = 128k = 192k
613	const lookup_size = 192 * 1024;
614	try expect(@sizeOf(Lookup) == lookup_size);
615
616	// buffer: (32k * 2), wp: 8, rp: 8, fp: 8
617	const window_size = 64 * 1024 + 8 + 8 + 8;
618	try expect(@sizeOf(SlidingWindow) == window_size);
619
620	const Bw = BlockWriter(@TypeOf(io.null_writer));
621	// huffman bit writer internal: 11480
622	const hbw_size = 11472; // 11.2k
623	try expect(@sizeOf(Bw) == hbw_size);
624
625	const D = Deflate(.raw, @TypeOf(io.null_writer), Bw);
626	// 404744, 395.26K
627	// ?Token: 6, ?u8: 2, level: 8
628	try expect(@sizeOf(D) == tokens_size + lookup_size + window_size + hbw_size + 24);
629	//print("Delfate size: {d} {d}\n", .{ @sizeOf(D), tokens_size + lookup_size + hbw_size + window_size });
630
631	// current std lib deflate allocation:
632	// 797_901, 779.2k
633	// measured with:
634	// var la = std.heap.logToWriterAllocator(testing.allocator, io.getStdOut().writer());
635	// const allocator = la.allocator();
636	// var cmp = try std.compress.deflate.compressor(allocator, io.null_writer, .{});
637	// defer cmp.deinit();
638
639	const HC = huffman.Compressor(.raw, @TypeOf(io.null_writer));
640	//print("size of HOC {d}\n", .{@sizeOf(HOC)});
641	try expect(@sizeOf(HC) == 77024);
642	// 64K buffer
643	// 11480 huffman_encoded
644	// 8 buffer write pointer
645	}
646
647	test "flate deflate file tokenization" {
648	const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
649	const cases = [_]struct {
650	data: []const u8, // uncompressed content
651	// expected number of tokens producet in deflate tokenization
652	tokens_count: [levels.len]usize = .{0} ** levels.len,
653	}{
654	.{
655	.data = @embedFile("testdata/rfc1951.txt"),
656	.tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
657	},
658
659	.{
660	.data = @embedFile("testdata/block_writer/huffman-null-max.input"),
661	.tokens_count = .{ 257, 257, 257, 257, 257, 257 },
662	},
663	.{
664	.data = @embedFile("testdata/block_writer/huffman-pi.input"),
665	.tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
666	},
667	.{
668	.data = @embedFile("testdata/block_writer/huffman-text.input"),
669	.tokens_count = .{ 235, 234, 234, 234, 234, 234 },
670	},
671	.{
672	.data = @embedFile("testdata/fuzz/roundtrip1.input"),
673	.tokens_count = .{ 333, 331, 331, 331, 331, 331 },
674	},
675	.{
676	.data = @embedFile("testdata/fuzz/roundtrip2.input"),
677	.tokens_count = .{ 334, 334, 334, 334, 334, 334 },
678	},
679	};
680
681	for (cases) \|case\| { // for each case
682	const data = case.data;
683
684	for (levels, 0..) \|level, i\| { // for each compression level
685	var original = io.fixedBufferStream(data);
686
687	// buffer for decompressed data
688	var al = std.ArrayList(u8).init(testing.allocator);
689	defer al.deinit();
690	const writer = al.writer();
691
692	// create compressor
693	const WriterType = @TypeOf(writer);
694	const TokenWriter = TokenDecoder(@TypeOf(writer));
695	var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
696
697	// Stream uncompressed `orignal` data to the compressor. It will
698	// produce tokens list and pass that list to the TokenDecoder. This
699	// TokenDecoder uses CircularBuffer from inflate to convert list of
700	// tokens back to the uncompressed stream.
701	try cmp.compress(original.reader());
702	try cmp.flush();
703	const expected_count = case.tokens_count[i];
704	const actual = cmp.block_writer.tokens_count;
705	if (expected_count == 0) {
706	print("actual token count {d}\n", .{actual});
707	} else {
708	try testing.expectEqual(expected_count, actual);
709	}
710
711	try testing.expectEqual(data.len, al.items.len);
712	try testing.expectEqualSlices(u8, data, al.items);
713	}
714	}
715	}
716
717	fn TokenDecoder(comptime WriterType: type) type {
718	return struct {
719	const CircularBuffer = @import("CircularBuffer.zig");
720	hist: CircularBuffer = .{},
721	wrt: WriterType,
722	tokens_count: usize = 0,
723
724	const Self = @This();
725
726	pub fn init(wrt: WriterType) Self {
727	return .{ .wrt = wrt };
728	}
729
730	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
731	self.tokens_count += tokens.len;
732	for (tokens) \|t\| {
733	switch (t.kind) {
734	.literal => self.hist.write(t.literal()),
735	.match => try self.hist.writeMatch(t.length(), t.distance()),
736	}
737	if (self.hist.free() < 285) try self.flushWin();
738	}
739	try self.flushWin();
740	}
741
742	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
743
744	fn flushWin(self: *Self) !void {
745	while (true) {
746	const buf = self.hist.read();
747	if (buf.len == 0) break;
748	try self.wrt.writeAll(buf);
749	}
750	}
751
752	pub fn flush(_: *Self) !void {}
753	};
754	}
755
756	test "flate.Deflate store simple compressor" {
757	const data = "Hello world!";
758	const expected = [_]u8{
759	0x1, // block type 0, final bit set
760	0xc, 0x0, // len = 12
761	0xf3, 0xff, // ~len
762	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
763	//0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
764	};
765
766	var fbs = std.io.fixedBufferStream(data);
767	var al = std.ArrayList(u8).init(testing.allocator);
768	defer al.deinit();
769
770	var cmp = try store.compressor(.raw, al.writer());
771	try cmp.compress(fbs.reader());
772	try cmp.finish();
773	try testing.expectEqualSlices(u8, &expected, al.items);
774
775	fbs.reset();
776	try al.resize(0);
777
778	// huffman only compresoor will also emit store block for this small sample
779	var hc = try huffman.compressor(.raw, al.writer());
780	try hc.compress(fbs.reader());
781	try hc.finish();
782	try testing.expectEqualSlices(u8, &expected, al.items);
783	}

lib/std/compress/flate/flate.zig created+236

...	@@ -0,0 +1,236 @@
1	/// Deflate is a lossless data compression file format that uses a combination
2	/// of LZ77 and Huffman coding.
3	pub const deflate = @import("deflate.zig");
4
5	/// Inflate is the decoding process that takes a Deflate bitstream for
6	/// decompression and correctly produces the original full-size data or file.
7	pub const inflate = @import("inflate.zig");
8
9	/// Decompress compressed data from reader and write plain data to the writer.
10	pub fn decompress(reader: anytype, writer: anytype) !void {
11	try inflate.decompress(.raw, reader, writer);
12	}
13
14	/// Decompressor type
15	pub fn Decompressor(comptime ReaderType: type) type {
16	return inflate.Inflate(.raw, ReaderType);
17	}
18
19	/// Create Decompressor which will read compressed data from reader.
20	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
21	return inflate.decompressor(.raw, reader);
22	}
23
24	/// Compression level, trades between speed and compression size.
25	pub const Options = deflate.Options;
26
27	/// Compress plain data from reader and write compressed data to the writer.
28	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
29	try deflate.compress(.raw, reader, writer, options);
30	}
31
32	/// Compressor type
33	pub fn Compressor(comptime WriterType: type) type {
34	return deflate.Compressor(.raw, WriterType);
35	}
36
37	/// Create Compressor which outputs compressed data to the writer.
38	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
39	return try deflate.compressor(.raw, writer, options);
40	}
41
42	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
43	/// compression, less memory requirements but bigger compressed sizes.
44	pub const huffman = struct {
45	pub fn compress(reader: anytype, writer: anytype) !void {
46	try deflate.huffman.compress(.raw, reader, writer);
47	}
48
49	pub fn Compressor(comptime WriterType: type) type {
50	return deflate.huffman.Compressor(.raw, WriterType);
51	}
52
53	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
54	return deflate.huffman.compressor(.raw, writer);
55	}
56	};
57
58	// No compression store only. Compressed size is slightly bigger than plain.
59	pub const store = struct {
60	pub fn compress(reader: anytype, writer: anytype) !void {
61	try deflate.store.compress(.raw, reader, writer);
62	}
63
64	pub fn Compressor(comptime WriterType: type) type {
65	return deflate.store.Compressor(.raw, WriterType);
66	}
67
68	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
69	return deflate.store.compressor(.raw, writer);
70	}
71	};
72
73	/// Container defines header/footer arround deflate bit stream. Gzip and zlib
74	/// compression algorithms are containers arround deflate bit stream body.
75	const Container = @import("container.zig").Container;
76	const std = @import("std");
77	const testing = std.testing;
78	const fixedBufferStream = std.io.fixedBufferStream;
79	const print = std.debug.print;
80
81	test "flate compress/decompress" {
82	var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer
83	var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer
84
85	const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
86	const cases = [_]struct {
87	data: []const u8, // uncompressed content
88	// compressed data sizes per level 4-9
89	gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len,
90	huffman_only_size: usize = 0,
91	store_size: usize = 0,
92	}{
93	.{
94	.data = @embedFile("testdata/rfc1951.txt"),
95	.gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 },
96	.huffman_only_size = 20287,
97	.store_size = 36967,
98	},
99	.{
100	.data = @embedFile("testdata/fuzz/roundtrip1.input"),
101	.gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 },
102	.huffman_only_size = 393,
103	.store_size = 393,
104	},
105	.{
106	.data = @embedFile("testdata/fuzz/roundtrip2.input"),
107	.gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 },
108	.huffman_only_size = 394,
109	.store_size = 394,
110	},
111	.{
112	.data = @embedFile("testdata/fuzz/deflate-stream.expect"),
113	.gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 },
114	.huffman_only_size = 498,
115	.store_size = 747,
116	},
117	};
118
119	for (cases, 0..) \|case, case_no\| { // for each case
120	const data = case.data;
121
122	for (levels, 0..) \|level, i\| { // for each compression level
123
124	inline for (Container.list) \|container\| { // for each wrapping
125	var compressed_size: usize = if (case.gzip_sizes[i] > 0)
126	case.gzip_sizes[i] - Container.gzip.size() + container.size()
127	else
128	0;
129
130	// compress original stream to compressed stream
131	{
132	var original = fixedBufferStream(data);
133	var compressed = fixedBufferStream(&cmp_buf);
134	try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level });
135	if (compressed_size == 0) {
136	if (container == .gzip)
137	print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
138	compressed_size = compressed.pos;
139	}
140	try testing.expectEqual(compressed_size, compressed.pos);
141	}
142	// decompress compressed stream to decompressed stream
143	{
144	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
145	var decompressed = fixedBufferStream(&dcm_buf);
146	try inflate.decompress(container, compressed.reader(), decompressed.writer());
147	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
148	}
149
150	// compressor writer interface
151	{
152	var compressed = fixedBufferStream(&cmp_buf);
153	var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level });
154	var cmp_wrt = cmp.writer();
155	try cmp_wrt.writeAll(data);
156	try cmp.finish();
157
158	try testing.expectEqual(compressed_size, compressed.pos);
159	}
160	// decompressor reader interface
161	{
162	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
163	var dcm = inflate.decompressor(container, compressed.reader());
164	var dcm_rdr = dcm.reader();
165	const n = try dcm_rdr.readAll(&dcm_buf);
166	try testing.expectEqual(data.len, n);
167	try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);
168	}
169	}
170	}
171	// huffman only compression
172	{
173	inline for (Container.list) \|container\| { // for each wrapping
174	var compressed_size: usize = if (case.huffman_only_size > 0)
175	case.huffman_only_size - Container.gzip.size() + container.size()
176	else
177	0;
178
179	// compress original stream to compressed stream
180	{
181	var original = fixedBufferStream(data);
182	var compressed = fixedBufferStream(&cmp_buf);
183	var cmp = try deflate.huffman.compressor(container, compressed.writer());
184	try cmp.compress(original.reader());
185	try cmp.finish();
186	if (compressed_size == 0) {
187	if (container == .gzip)
188	print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos });
189	compressed_size = compressed.pos;
190	}
191	try testing.expectEqual(compressed_size, compressed.pos);
192	}
193	// decompress compressed stream to decompressed stream
194	{
195	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
196	var decompressed = fixedBufferStream(&dcm_buf);
197	try inflate.decompress(container, compressed.reader(), decompressed.writer());
198	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
199	}
200	}
201	}
202
203	// store only
204	{
205	inline for (Container.list) \|container\| { // for each wrapping
206	var compressed_size: usize = if (case.store_size > 0)
207	case.store_size - Container.gzip.size() + container.size()
208	else
209	0;
210
211	// compress original stream to compressed stream
212	{
213	var original = fixedBufferStream(data);
214	var compressed = fixedBufferStream(&cmp_buf);
215	var cmp = try deflate.store.compressor(container, compressed.writer());
216	try cmp.compress(original.reader());
217	try cmp.finish();
218	if (compressed_size == 0) {
219	if (container == .gzip)
220	print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos });
221	compressed_size = compressed.pos;
222	}
223
224	try testing.expectEqual(compressed_size, compressed.pos);
225	}
226	// decompress compressed stream to decompressed stream
227	{
228	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
229	var decompressed = fixedBufferStream(&dcm_buf);
230	try inflate.decompress(container, compressed.reader(), decompressed.writer());
231	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
232	}
233	}
234	}
235	}
236	}

lib/std/compress/flate/gzip.zig created+66

...	@@ -0,0 +1,66 @@
1	const deflate = @import("deflate.zig");
2	const inflate = @import("inflate.zig");
3
4	/// Decompress compressed data from reader and write plain data to the writer.
5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	try inflate.decompress(.gzip, reader, writer);
7	}
8
9	/// Decompressor type
10	pub fn Decompressor(comptime ReaderType: type) type {
11	return inflate.Inflate(.gzip, ReaderType);
12	}
13
14	/// Create Decompressor which will read compressed data from reader.
15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16	return inflate.decompressor(.gzip, reader);
17	}
18
19	/// Compression level, trades between speed and compression size.
20	pub const Options = deflate.Options;
21
22	/// Compress plain data from reader and write compressed data to the writer.
23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24	try deflate.compress(.gzip, reader, writer, options);
25	}
26
27	/// Compressor type
28	pub fn Compressor(comptime WriterType: type) type {
29	return deflate.Compressor(.gzip, WriterType);
30	}
31
32	/// Create Compressor which outputs compressed data to the writer.
33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34	return try deflate.compressor(.gzip, writer, options);
35	}
36
37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38	/// compression, less memory requirements but bigger compressed sizes.
39	pub const huffman = struct {
40	pub fn compress(reader: anytype, writer: anytype) !void {
41	try deflate.huffman.compress(.gzip, reader, writer);
42	}
43
44	pub fn Compressor(comptime WriterType: type) type {
45	return deflate.huffman.Compressor(.gzip, WriterType);
46	}
47
48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49	return deflate.huffman.compressor(.gzip, writer);
50	}
51	};
52
53	// No compression store only. Compressed size is slightly bigger than plain.
54	pub const store = struct {
55	pub fn compress(reader: anytype, writer: anytype) !void {
56	try deflate.store.compress(.gzip, reader, writer);
57	}
58
59	pub fn Compressor(comptime WriterType: type) type {
60	return deflate.store.Compressor(.gzip, WriterType);
61	}
62
63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64	return deflate.store.compressor(.gzip, writer);
65	}
66	};

lib/std/compress/flate/huffman_decoder.zig created+308

...	@@ -0,0 +1,308 @@
1	const std = @import("std");
2	const testing = std.testing;
3
4	pub const Symbol = packed struct {
5	pub const Kind = enum(u2) {
6	literal,
7	end_of_block,
8	match,
9	};
10
11	symbol: u8 = 0, // symbol from alphabet
12	code_bits: u4 = 0, // number of bits in code 0-15
13	kind: Kind = .literal,
14
15	code: u16 = 0, // huffman code of the symbol
16	next: u16 = 0, // pointer to the next symbol in linked list
17	// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
18
19	// Sorting less than function.
20	pub fn asc(_: void, a: Symbol, b: Symbol) bool {
21	if (a.code_bits == b.code_bits) {
22	if (a.kind == b.kind) {
23	return a.symbol < b.symbol;
24	}
25	return @intFromEnum(a.kind) < @intFromEnum(b.kind);
26	}
27	return a.code_bits < b.code_bits;
28	}
29	};
30
31	pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
32	pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
33	pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
34
35	pub const Error = error{
36	InvalidCode,
37	OversubscribedHuffmanTree,
38	IncompleteHuffmanTree,
39	MissingEndOfBlockCode,
40	};
41
42	/// Creates huffman tree codes from list of code lengths (in `build`).
43	///
44	/// `find` then finds symbol for code bits. Code can be any length between 1 and
45	/// 15 bits. When calling `find` we don't know how many bits will be used to
46	/// find symbol. When symbol is returned it has code_bits field which defines
47	/// how much we should advance in bit stream.
48	///
49	/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
50	/// many times in this table; 32K places for 286 (at most) symbols.
51	/// Small lookup table is optimization for faster search.
52	/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
53	/// with difference that we here use statically allocated arrays.
54	///
55	fn HuffmanDecoder(
56	comptime alphabet_size: u16,
57	comptime max_code_bits: u4,
58	comptime lookup_bits: u4,
59	) type {
60	const lookup_shift = max_code_bits - lookup_bits;
61
62	return struct {
63	// all symbols in alaphabet, sorted by code_len, symbol
64	symbols: [alphabet_size]Symbol = undefined,
65	// lookup table code -> symbol
66	lookup: [1 << lookup_bits]Symbol = undefined,
67
68	const Self = @This();
69
70	/// Generates symbols and lookup tables from list of code lens for each symbol.
71	pub fn generate(self: *Self, lens: []const u4) !void {
72	try checkCompletnes(lens);
73
74	// init alphabet with code_bits
75	for (self.symbols, 0..) \|_, i\| {
76	const cb: u4 = if (i < lens.len) lens[i] else 0;
77	self.symbols[i] = if (i < 256)
78	.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
79	else if (i == 256)
80	.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
81	else
82	.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
83	}
84	std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
85
86	// reset lookup table
87	for (0..self.lookup.len) \|i\| {
88	self.lookup[i] = .{};
89	}
90
91	// assign code to symbols
92	// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
93	var code: u16 = 0;
94	var idx: u16 = 0;
95	for (&self.symbols, 0..) \|*sym, pos\| {
96	//print("sym: {}\n", .{sym});
97	if (sym.code_bits == 0) continue; // skip unused
98	sym.code = code;
99
100	const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
101	const next_idx = next_code >> lookup_shift;
102
103	if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
104	if (sym.code_bits <= lookup_bits) {
105	// fill small lookup table
106	for (idx..next_idx) \|j\|
107	self.lookup[j] = sym.*;
108	} else {
109	// insert into linked table starting at root
110	const root = &self.lookup[idx];
111	const root_next = root.next;
112	root.next = @intCast(pos);
113	sym.next = root_next;
114	}
115
116	idx = next_idx;
117	code = next_code;
118	}
119	//print("decoder generate, code: {d}, idx: {d}\n", .{ code, idx });
120	}
121
122	/// Given the list of code lengths check that it represents a canonical
123	/// Huffman code for n symbols.
124	///
125	/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
126	fn checkCompletnes(lens: []const u4) !void {
127	if (alphabet_size == 286)
128	if (lens[256] == 0) return error.MissingEndOfBlockCode;
129
130	var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
131	var max: usize = 0;
132	for (lens) \|n\| {
133	if (n == 0) continue;
134	if (n > max) max = n;
135	count[n] += 1;
136	}
137	if (max == 0) // emtpy tree
138	return;
139
140	// check for an over-subscribed or incomplete set of lengths
141	var left: usize = 1; // one possible code of zero length
142	for (1..count.len) \|len\| {
143	left <<= 1; // one more bit, double codes left
144	if (count[len] > left)
145	return error.OversubscribedHuffmanTree;
146	left -= count[len]; // deduct count from possible codes
147	}
148	if (left > 0) { // left > 0 means incomplete
149	// incomplete code ok only for single length 1 code
150	if (max_code_bits > 7 and max == count[0] + count[1]) return;
151	return error.IncompleteHuffmanTree;
152	}
153	}
154
155	/// Finds symbol for lookup table code.
156	pub fn find(self: *Self, code: u16) !Symbol {
157	// try to find in lookup table
158	const idx = code >> lookup_shift;
159	const sym = self.lookup[idx];
160	if (sym.code_bits != 0) return sym;
161	// if not use linked list of symbols with same prefix
162	return self.findLinked(code, sym.next);
163	}
164
165	inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
166	var pos = start;
167	while (pos > 0) {
168	const sym = self.symbols[pos];
169	const shift = max_code_bits - sym.code_bits;
170	// compare code_bits number of upper bits
171	if ((code ^ sym.code) >> shift == 0) return sym;
172	pos = sym.next;
173	}
174	return error.InvalidCode;
175	}
176	};
177	}
178
179	test "flate.HuffmanDecoder init/find" {
180	// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
181	const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
182	var h: CodegenDecoder = .{};
183	try h.generate(&code_lens);
184
185	const expected = [_]struct {
186	sym: Symbol,
187	code: u16,
188	}{
189	.{
190	.code = 0b00_00000,
191	.sym = .{ .symbol = 3, .code_bits = 2 },
192	},
193	.{
194	.code = 0b01_00000,
195	.sym = .{ .symbol = 18, .code_bits = 2 },
196	},
197	.{
198	.code = 0b100_0000,
199	.sym = .{ .symbol = 1, .code_bits = 3 },
200	},
201	.{
202	.code = 0b101_0000,
203	.sym = .{ .symbol = 4, .code_bits = 3 },
204	},
205	.{
206	.code = 0b110_0000,
207	.sym = .{ .symbol = 17, .code_bits = 3 },
208	},
209	.{
210	.code = 0b1110_000,
211	.sym = .{ .symbol = 0, .code_bits = 4 },
212	},
213	.{
214	.code = 0b1111_000,
215	.sym = .{ .symbol = 16, .code_bits = 4 },
216	},
217	};
218
219	// unused symbols
220	for (0..12) \|i\| {
221	try testing.expectEqual(0, h.symbols[i].code_bits);
222	}
223	// used, from index 12
224	for (expected, 12..) \|e, i\| {
225	try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
226	try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
227	const sym_from_code = try h.find(e.code);
228	try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
229	}
230
231	// All possible codes for each symbol.
232	// Lookup table has 126 elements, to cover all possible 7 bit codes.
233	for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
234	try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
235
236	for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
237	try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
238
239	for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
240	try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
241
242	for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
243	try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
244
245	for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
246	try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
247
248	for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
249	try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
250
251	for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
252	try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
253	}
254
255	const print = std.debug.print;
256	const assert = std.debug.assert;
257	const expect = std.testing.expect;
258
259	test "flate.HuffmanDecoder encode/decode literals" {
260	const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder;
261
262	for (1..286) \|j\| { // for all different number of codes
263	var enc: LiteralEncoder = .{};
264	// create freqencies
265	var freq = [_]u16{0} ** 286;
266	freq[256] = 1; // ensure we have end of block code
267	for (&freq, 1..) \|*f, i\| {
268	if (i % j == 0)
269	f.* = @intCast(i);
270	}
271
272	// encoder from freqencies
273	enc.generate(&freq, 15);
274
275	// get code_lens from encoder
276	var code_lens = [_]u4{0} ** 286;
277	for (code_lens, 0..) \|_, i\| {
278	code_lens[i] = @intCast(enc.codes[i].len);
279	}
280	// generate decoder from code lens
281	var dec: LiteralDecoder = .{};
282	try dec.generate(&code_lens);
283
284	// expect decoder code to match original encoder code
285	for (dec.symbols) \|s\| {
286	if (s.code_bits == 0) continue;
287	const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
288	const symbol: u16 = switch (s.kind) {
289	.literal => s.symbol,
290	.end_of_block => 256,
291	.match => @as(u16, s.symbol) + 257,
292	};
293
294	const c = enc.codes[symbol];
295	try expect(c.code == c_code);
296	}
297
298	// find each symbol by code
299	for (enc.codes) \|c\| {
300	if (c.len == 0) continue;
301
302	const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
303	const s = try dec.find(s_code);
304	try expect(s.code == s_code);
305	try expect(s.code_bits == c.len);
306	}
307	}
308	}

lib/std/compress/flate/huffman_encoder.zig created+536

...	@@ -0,0 +1,536 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5	const sort = std.sort;
6	const testing = std.testing;
7
8	const consts = @import("consts.zig").huffman;
9
10	const LiteralNode = struct {
11	literal: u16,
12	freq: u16,
13	};
14
15	// Describes the state of the constructed tree for a given depth.
16	const LevelInfo = struct {
17	// Our level. for better printing
18	level: u32,
19
20	// The frequency of the last node at this level
21	last_freq: u32,
22
23	// The frequency of the next character to add to this level
24	next_char_freq: u32,
25
26	// The frequency of the next pair (from level below) to add to this level.
27	// Only valid if the "needed" value of the next lower level is 0.
28	next_pair_freq: u32,
29
30	// The number of chains remaining to generate for this level before moving
31	// up to the next level
32	needed: u32,
33	};
34
35	// hcode is a huffman code with a bit code and bit length.
36	pub const HuffCode = struct {
37	code: u16 = 0,
38	len: u16 = 0,
39
40	// set sets the code and length of an hcode.
41	fn set(self: *HuffCode, code: u16, length: u16) void {
42	self.len = length;
43	self.code = code;
44	}
45	};
46
47	pub fn HuffmanEncoder(comptime size: usize) type {
48	return struct {
49	codes: [size]HuffCode = undefined,
50	// Reusable buffer with the longest possible frequency table.
51	freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined,
52	bit_count: [17]u32 = undefined,
53	lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
54	lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
55
56	const Self = @This();
57
58	// Update this Huffman Code object to be the minimum code for the specified frequency count.
59	//
60	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
61	// max_bits The maximum number of bits to use for any literal.
62	pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
63	var list = self.freq_cache[0 .. freq.len + 1];
64	// Number of non-zero literals
65	var count: u32 = 0;
66	// Set list to be the set of all non-zero literals and their frequencies
67	for (freq, 0..) \|f, i\| {
68	if (f != 0) {
69	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
70	count += 1;
71	} else {
72	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
73	self.codes[i].len = 0;
74	}
75	}
76	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
77
78	list = list[0..count];
79	if (count <= 2) {
80	// Handle the small cases here, because they are awkward for the general case code. With
81	// two or fewer literals, everything has bit length 1.
82	for (list, 0..) \|node, i\| {
83	// "list" is in order of increasing literal value.
84	self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
85	}
86	return;
87	}
88	self.lfs = list;
89	mem.sort(LiteralNode, self.lfs, {}, byFreq);
90
91	// Get the number of literals for each bit count
92	const bit_count = self.bitCounts(list, max_bits);
93	// And do the assignment
94	self.assignEncodingAndSize(bit_count, list);
95	}
96
97	pub fn bitLength(self: *Self, freq: []u16) u32 {
98	var total: u32 = 0;
99	for (freq, 0..) \|f, i\| {
100	if (f != 0) {
101	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
102	}
103	}
104	return total;
105	}
106
107	// Return the number of literals assigned to each bit size in the Huffman encoding
108	//
109	// This method is only called when list.len >= 3
110	// The cases of 0, 1, and 2 literals are handled by special case code.
111	//
112	// list: An array of the literals with non-zero frequencies
113	// and their associated frequencies. The array is in order of increasing
114	// frequency, and has as its last element a special element with frequency
115	// std.math.maxInt(i32)
116	//
117	// max_bits: The maximum number of bits that should be used to encode any literal.
118	// Must be less than 16.
119	//
120	// Returns an integer array in which array[i] indicates the number of literals
121	// that should be encoded in i bits.
122	fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
123	var max_bits = max_bits_to_use;
124	const n = list.len;
125	const max_bits_limit = 16;
126
127	assert(max_bits < max_bits_limit);
128
129	// The tree can't have greater depth than n - 1, no matter what. This
130	// saves a little bit of work in some small cases
131	max_bits = @min(max_bits, n - 1);
132
133	// Create information about each of the levels.
134	// A bogus "Level 0" whose sole purpose is so that
135	// level1.prev.needed == 0. This makes level1.next_pair_freq
136	// be a legitimate value that never gets chosen.
137	var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
138	// leaf_counts[i] counts the number of literals at the left
139	// of ancestors of the rightmost node at level i.
140	// leaf_counts[i][j] is the number of literals at the left
141	// of the level j ancestor.
142	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
143
144	{
145	var level = @as(u32, 1);
146	while (level <= max_bits) : (level += 1) {
147	// For every level, the first two items are the first two characters.
148	// We initialize the levels as if we had already figured this out.
149	levels[level] = LevelInfo{
150	.level = level,
151	.last_freq = list[1].freq,
152	.next_char_freq = list[2].freq,
153	.next_pair_freq = list[0].freq + list[1].freq,
154	.needed = 0,
155	};
156	leaf_counts[level][level] = 2;
157	if (level == 1) {
158	levels[level].next_pair_freq = math.maxInt(i32);
159	}
160	}
161	}
162
163	// We need a total of 2*n - 2 items at top level and have already generated 2.
164	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
165
166	{
167	var level = max_bits;
168	while (true) {
169	var l = &levels[level];
170	if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
171	// We've run out of both leafs and pairs.
172	// End all calculations for this level.
173	// To make sure we never come back to this level or any lower level,
174	// set next_pair_freq impossibly large.
175	l.needed = 0;
176	levels[level + 1].next_pair_freq = math.maxInt(i32);
177	level += 1;
178	continue;
179	}
180
181	const prev_freq = l.last_freq;
182	if (l.next_char_freq < l.next_pair_freq) {
183	// The next item on this row is a leaf node.
184	const next = leaf_counts[level][level] + 1;
185	l.last_freq = l.next_char_freq;
186	// Lower leaf_counts are the same of the previous node.
187	leaf_counts[level][level] = next;
188	if (next >= list.len) {
189	l.next_char_freq = maxNode().freq;
190	} else {
191	l.next_char_freq = list[next].freq;
192	}
193	} else {
194	// The next item on this row is a pair from the previous row.
195	// next_pair_freq isn't valid until we generate two
196	// more values in the level below
197	l.last_freq = l.next_pair_freq;
198	// Take leaf counts from the lower level, except counts[level] remains the same.
199	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
200	levels[l.level - 1].needed = 2;
201	}
202
203	l.needed -= 1;
204	if (l.needed == 0) {
205	// We've done everything we need to do for this level.
206	// Continue calculating one level up. Fill in next_pair_freq
207	// of that level with the sum of the two nodes we've just calculated on
208	// this level.
209	if (l.level == max_bits) {
210	// All done!
211	break;
212	}
213	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
214	level += 1;
215	} else {
216	// If we stole from below, move down temporarily to replenish it.
217	while (levels[level - 1].needed > 0) {
218	level -= 1;
219	if (level == 0) {
220	break;
221	}
222	}
223	}
224	}
225	}
226
227	// Somethings is wrong if at the end, the top level is null or hasn't used
228	// all of the leaves.
229	assert(leaf_counts[max_bits][max_bits] == n);
230
231	var bit_count = self.bit_count[0 .. max_bits + 1];
232	var bits: u32 = 1;
233	const counts = &leaf_counts[max_bits];
234	{
235	var level = max_bits;
236	while (level > 0) : (level -= 1) {
237	// counts[level] gives the number of literals requiring at least "bits"
238	// bits to encode.
239	bit_count[bits] = counts[level] - counts[level - 1];
240	bits += 1;
241	if (level == 0) {
242	break;
243	}
244	}
245	}
246	return bit_count;
247	}
248
249	// Look at the leaves and assign them a bit count and an encoding as specified
250	// in RFC 1951 3.2.2
251	fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
252	var code = @as(u16, 0);
253	var list = list_arg;
254
255	for (bit_count, 0..) \|bits, n\| {
256	code <<= 1;
257	if (n == 0 or bits == 0) {
258	continue;
259	}
260	// The literals list[list.len-bits] .. list[list.len-bits]
261	// are encoded using "bits" bits, and get the values
262	// code, code + 1, .... The code values are
263	// assigned in literal order (not frequency order).
264	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
265
266	self.lns = chunk;
267	mem.sort(LiteralNode, self.lns, {}, byLiteral);
268
269	for (chunk) \|node\| {
270	self.codes[node.literal] = HuffCode{
271	.code = bitReverse(u16, code, @as(u5, @intCast(n))),
272	.len = @as(u16, @intCast(n)),
273	};
274	code += 1;
275	}
276	list = list[0 .. list.len - @as(u32, @intCast(bits))];
277	}
278	}
279	};
280	}
281
282	fn maxNode() LiteralNode {
283	return LiteralNode{
284	.literal = math.maxInt(u16),
285	.freq = math.maxInt(u16),
286	};
287	}
288
289	pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
290	return .{};
291	}
292
293	pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies);
294	pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count);
295	pub const CodegenEncoder = HuffmanEncoder(19);
296
297	// Generates a HuffmanCode corresponding to the fixed literal table
298	pub fn fixedLiteralEncoder() LiteralEncoder {
299	var h: LiteralEncoder = undefined;
300	var ch: u16 = 0;
301
302	while (ch < consts.max_num_frequencies) : (ch += 1) {
303	var bits: u16 = undefined;
304	var size: u16 = undefined;
305	switch (ch) {
306	0...143 => {
307	// size 8, 000110000 .. 10111111
308	bits = ch + 48;
309	size = 8;
310	},
311	144...255 => {
312	// size 9, 110010000 .. 111111111
313	bits = ch + 400 - 144;
314	size = 9;
315	},
316	256...279 => {
317	// size 7, 0000000 .. 0010111
318	bits = ch - 256;
319	size = 7;
320	},
321	else => {
322	// size 8, 11000000 .. 11000111
323	bits = ch + 192 - 280;
324	size = 8;
325	},
326	}
327	h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
328	}
329	return h;
330	}
331
332	pub fn fixedDistanceEncoder() DistanceEncoder {
333	var h: DistanceEncoder = undefined;
334	for (h.codes, 0..) \|_, ch\| {
335	h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
336	}
337	return h;
338	}
339
340	pub fn huffmanDistanceEncoder() DistanceEncoder {
341	var distance_freq = [1]u16{0} ** consts.distance_code_count;
342	distance_freq[0] = 1;
343	// huff_distance is a static distance encoder used for huffman only encoding.
344	// It can be reused since we will not be encoding distance values.
345	var h: DistanceEncoder = .{};
346	h.generate(distance_freq[0..], 15);
347	return h;
348	}
349
350	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
351	_ = context;
352	return a.literal < b.literal;
353	}
354
355	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
356	_ = context;
357	if (a.freq == b.freq) {
358	return a.literal < b.literal;
359	}
360	return a.freq < b.freq;
361	}
362
363	test "flate.HuffmanEncoder generate a Huffman code from an array of frequencies" {
364	var freqs: [19]u16 = [_]u16{
365	8, // 0
366	1, // 1
367	1, // 2
368	2, // 3
369	5, // 4
370	10, // 5
371	9, // 6
372	1, // 7
373	0, // 8
374	0, // 9
375	0, // 10
376	0, // 11
377	0, // 12
378	0, // 13
379	0, // 14
380	0, // 15
381	1, // 16
382	3, // 17
383	5, // 18
384	};
385
386	var enc = huffmanEncoder(19);
387	enc.generate(freqs[0..], 7);
388
389	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422	}
423
424	test "flate.HuffmanEncoder generate a Huffman code for the fixed literal table specific to Deflate" {
425	const enc = fixedLiteralEncoder();
426	for (enc.codes) \|c\| {
427	switch (c.len) {
428	7 => {
429	const v = @bitReverse(@as(u7, @intCast(c.code)));
430	try testing.expect(v <= 0b0010111);
431	},
432	8 => {
433	const v = @bitReverse(@as(u8, @intCast(c.code)));
434	try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
435	(v >= 0b11000000 and v <= 11000111));
436	},
437	9 => {
438	const v = @bitReverse(@as(u9, @intCast(c.code)));
439	try testing.expect(v >= 0b110010000 and v <= 0b111111111);
440	},
441	else => unreachable,
442	}
443	}
444	}
445
446	test "flate.HuffmanEncoder generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
447	const enc = fixedDistanceEncoder();
448	for (enc.codes) \|c\| {
449	const v = @bitReverse(@as(u5, @intCast(c.code)));
450	try testing.expect(v <= 29);
451	try testing.expect(c.len == 5);
452	}
453	}
454
455	// Reverse bit-by-bit a N-bit code.
456	fn bitReverse(comptime T: type, value: T, n: usize) T {
457	const r = @bitReverse(value);
458	return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).Int.bits - n));
459	}
460
461	test "flate bitReverse" {
462	const ReverseBitsTest = struct {
463	in: u16,
464	bit_count: u5,
465	out: u16,
466	};
467
468	const reverse_bits_tests = [_]ReverseBitsTest{
469	.{ .in = 1, .bit_count = 1, .out = 1 },
470	.{ .in = 1, .bit_count = 2, .out = 2 },
471	.{ .in = 1, .bit_count = 3, .out = 4 },
472	.{ .in = 1, .bit_count = 4, .out = 8 },
473	.{ .in = 1, .bit_count = 5, .out = 16 },
474	.{ .in = 17, .bit_count = 5, .out = 17 },
475	.{ .in = 257, .bit_count = 9, .out = 257 },
476	.{ .in = 29, .bit_count = 5, .out = 23 },
477	};
478
479	for (reverse_bits_tests) \|h\| {
480	const v = bitReverse(u16, h.in, h.bit_count);
481	try std.testing.expectEqual(h.out, v);
482	}
483	}
484
485	test "flate.HuffmanEncoder fixedLiteralEncoder codes" {
486	var al = std.ArrayList(u8).init(testing.allocator);
487	defer al.deinit();
488	var bw = std.io.bitWriter(.little, al.writer());
489
490	const f = fixedLiteralEncoder();
491	for (f.codes) \|c\| {
492	try bw.writeBits(c.code, c.len);
493	}
494	try testing.expectEqualSlices(u8, &fixed_codes, al.items);
495	}
496
497	pub const fixed_codes = [_]u8{
498	0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
499	0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
500	0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
501	0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
502	0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
503	0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
504	0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
505	0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
506	0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
507	0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
508	0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
509	0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
510	0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
511	0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
512	0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
513	0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
514	0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
515	0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
516	0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
517	0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
518	0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
519	0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
520	0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
521	0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
522	0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
523	0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
524	0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
525	0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
526	0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
527	0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
528	0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
529	0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
530	0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
531	0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
532	0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
533	0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
534	0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
535	0b10100011,
536	};

lib/std/compress/flate/inflate.zig created+546

...	@@ -0,0 +1,546 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	const hfd = @import("huffman_decoder.zig");
6	const BitReader = @import("bit_reader.zig").BitReader;
7	const CircularBuffer = @import("CircularBuffer.zig");
8	const Container = @import("container.zig").Container;
9	const Token = @import("Token.zig");
10	const codegen_order = @import("consts.zig").huffman.codegen_order;
11
12	/// Decompresses deflate bit stream `reader` and writes uncompressed data to the
13	/// `writer` stream.
14	pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void {
15	var d = decompressor(container, reader);
16	try d.decompress(writer);
17	}
18
19	/// Inflate decompressor for the reader type.
20	pub fn decompressor(comptime container: Container, reader: anytype) Inflate(container, @TypeOf(reader)) {
21	return Inflate(container, @TypeOf(reader)).init(reader);
22	}
23
24	/// Inflate decompresses deflate bit stream. Reads compressed data from reader
25	/// provided in init. Decompressed data are stored in internal hist buffer and
26	/// can be accesses iterable `next` or reader interface.
27	///
28	/// Container defines header/footer wrapper around deflate bit stream. Can be
29	/// gzip or zlib.
30	///
31	/// Deflate bit stream consists of multiple blocks. Block can be one of three types:
32	/// * stored, non compressed, max 64k in size
33	/// * fixed, huffman codes are predefined
34	/// * dynamic, huffman code tables are encoded at the block start
35	///
36	/// `step` function runs decoder until internal `hist` buffer is full. Client
37	/// than needs to read that data in order to proceed with decoding.
38	///
39	/// Allocates 74.5K of internal buffers, most important are:
40	/// * 64K for history (CircularBuffer)
41	/// * ~10K huffman decoders (Literal and DistanceDecoder)
42	///
43	pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {
44	return struct {
45	const BitReaderType = BitReader(ReaderType);
46	const F = BitReaderType.flag;
47
48	bits: BitReaderType = .{},
49	hist: CircularBuffer = .{},
50	// Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.
51	hasher: container.Hasher() = .{},
52
53	// dynamic block huffman code decoders
54	lit_dec: hfd.LiteralDecoder = .{}, // literals
55	dst_dec: hfd.DistanceDecoder = .{}, // distances
56
57	// current read state
58	bfinal: u1 = 0,
59	block_type: u2 = 0b11,
60	state: ReadState = .protocol_header,
61
62	const ReadState = enum {
63	protocol_header,
64	block_header,
65	block,
66	protocol_footer,
67	end,
68	};
69
70	const Self = @This();
71
72	pub const Error = BitReaderType.Error \|\| Container.Error \|\| hfd.Error \|\| error{
73	InvalidCode,
74	InvalidMatch,
75	InvalidBlockType,
76	WrongStoredBlockNlen,
77	InvalidDynamicBlockHeader,
78	};
79
80	pub fn init(rt: ReaderType) Self {
81	return .{ .bits = BitReaderType.init(rt) };
82	}
83
84	fn blockHeader(self: *Self) !void {
85	self.bfinal = try self.bits.read(u1);
86	self.block_type = try self.bits.read(u2);
87	}
88
89	fn storedBlock(self: *Self) !bool {
90	self.bits.alignToByte(); // skip padding until byte boundary
91	// everyting after this is byte aligned in stored block
92	var len = try self.bits.read(u16);
93	const nlen = try self.bits.read(u16);
94	if (len != ~nlen) return error.WrongStoredBlockNlen;
95
96	while (len > 0) {
97	const buf = self.hist.getWritable(len);
98	try self.bits.readAll(buf);
99	len -= @intCast(buf.len);
100	}
101	return true;
102	}
103
104	fn fixedBlock(self: *Self) !bool {
105	while (!self.hist.full()) {
106	const code = try self.bits.readFixedCode();
107	switch (code) {
108	0...255 => self.hist.write(@intCast(code)),
109	256 => return true, // end of block
110	257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
111	else => return error.InvalidCode,
112	}
113	}
114	return false;
115	}
116
117	// Handles fixed block non literal (length) code.
118	// Length code is followed by 5 bits of distance code.
119	fn fixedDistanceCode(self: *Self, code: u8) !void {
120	try self.bits.fill(5 + 5 + 13);
121	const length = try self.decodeLength(code);
122	const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered \| F.reverse));
123	try self.hist.writeMatch(length, distance);
124	}
125
126	inline fn decodeLength(self: *Self, code: u8) !u16 {
127	if (code > 28) return error.InvalidCode;
128	const ml = Token.matchLength(code);
129	return if (ml.extra_bits == 0) // 0 - 5 extra bits
130	ml.base
131	else
132	ml.base + try self.bits.readN(ml.extra_bits, F.buffered);
133	}
134
135	fn decodeDistance(self: *Self, code: u8) !u16 {
136	if (code > 29) return error.InvalidCode;
137	const md = Token.matchDistance(code);
138	return if (md.extra_bits == 0) // 0 - 13 extra bits
139	md.base
140	else
141	md.base + try self.bits.readN(md.extra_bits, F.buffered);
142	}
143
144	fn dynamicBlockHeader(self: *Self) !void {
145	const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
146	const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
147	const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lenths are encoded
148
149	if (hlit > 286 or hdist > 30)
150	return error.InvalidDynamicBlockHeader;
151
152	// lengths for code lengths
153	var cl_lens = [_]u4{0} ** 19;
154	for (0..hclen) \|i\| {
155	cl_lens[codegen_order[i]] = try self.bits.read(u3);
156	}
157	var cl_dec: hfd.CodegenDecoder = .{};
158	try cl_dec.generate(&cl_lens);
159
160	// literal code lengths
161	var lit_lens = [_]u4{0} ** (286);
162	var pos: usize = 0;
163	while (pos < hlit) {
164	const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
165	try self.bits.shift(sym.code_bits);
166	pos += try self.dynamicCodeLength(sym.symbol, &lit_lens, pos);
167	}
168	if (pos > hlit)
169	return error.InvalidDynamicBlockHeader;
170
171	// distance code lenths
172	var dst_lens = [_]u4{0} ** (30);
173	pos = 0;
174	while (pos < hdist) {
175	const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
176	try self.bits.shift(sym.code_bits);
177	pos += try self.dynamicCodeLength(sym.symbol, &dst_lens, pos);
178	}
179	if (pos > hdist)
180	return error.InvalidDynamicBlockHeader;
181
182	try self.lit_dec.generate(&lit_lens);
183	try self.dst_dec.generate(&dst_lens);
184	}
185
186	// Decode code length symbol to code length. Writes decoded length into
187	// lens slice starting at position pos. Returns number of positions
188	// advanced.
189	fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize {
190	if (pos >= lens.len)
191	return error.InvalidDynamicBlockHeader;
192
193	switch (code) {
194	0...15 => {
195	// Represent code lengths of 0 - 15
196	lens[pos] = @intCast(code);
197	return 1;
198	},
199	16 => {
200	// Copy the previous code length 3 - 6 times.
201	// The next 2 bits indicate repeat length
202	const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
203	if (pos == 0 or pos + n > lens.len)
204	return error.InvalidDynamicBlockHeader;
205	for (0..n) \|i\| {
206	lens[pos + i] = lens[pos + i - 1];
207	}
208	return n;
209	},
210	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
211	17 => return @as(u8, try self.bits.read(u3)) + 3,
212	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
213	18 => return @as(u8, try self.bits.read(u7)) + 11,
214	else => return error.InvalidDynamicBlockHeader,
215	}
216	}
217
218	// In larger archives most blocks are usually dynamic, so decompression
219	// performance depends on this function.
220	fn dynamicBlock(self: *Self) !bool {
221	// Hot path loop!
222	while (!self.hist.full()) {
223	try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path)
224	const sym = try self.decodeSymbol(&self.lit_dec);
225
226	switch (sym.kind) {
227	.literal => self.hist.write(sym.symbol),
228	.match => { // Decode match backreference <length, distance>
229	try self.bits.fill(5 + 15 + 13); // so we can use buffered reads
230	const length = try self.decodeLength(sym.symbol);
231	const dsm = try self.decodeSymbol(&self.dst_dec);
232	const distance = try self.decodeDistance(dsm.symbol);
233	try self.hist.writeMatch(length, distance);
234	},
235	.end_of_block => return true,
236	}
237	}
238	return false;
239	}
240
241	// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
242	// decoder to find symbol for that code. We then know how many bits is
243	// used. Shift bit reader for that much bits, those bits are used. And
244	// return symbol.
245	fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
246	const sym = try decoder.find(try self.bits.peekF(u15, F.buffered \| F.reverse));
247	try self.bits.shift(sym.code_bits);
248	return sym;
249	}
250
251	fn step(self: *Self) !void {
252	switch (self.state) {
253	.protocol_header => {
254	try container.parseHeader(&self.bits);
255	self.state = .block_header;
256	},
257	.block_header => {
258	try self.blockHeader();
259	self.state = .block;
260	if (self.block_type == 2) try self.dynamicBlockHeader();
261	},
262	.block => {
263	const done = switch (self.block_type) {
264	0 => try self.storedBlock(),
265	1 => try self.fixedBlock(),
266	2 => try self.dynamicBlock(),
267	else => return error.InvalidBlockType,
268	};
269	if (done) {
270	self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
271	}
272	},
273	.protocol_footer => {
274	self.bits.alignToByte();
275	try container.parseFooter(&self.hasher, &self.bits);
276	self.state = .end;
277	},
278	.end => {},
279	}
280	}
281
282	/// Replaces the inner reader with new reader.
283	pub fn setReader(self: *Self, new_reader: ReaderType) void {
284	self.bits.forward_reader = new_reader;
285	if (self.state == .end or self.state == .protocol_footer) {
286	self.state = .protocol_header;
287	}
288	}
289
290	// Reads all compressed data from the internal reader and outputs plain
291	// (uncompressed) data to the provided writer.
292	pub fn decompress(self: *Self, writer: anytype) !void {
293	while (try self.next()) \|buf\| {
294	try writer.writeAll(buf);
295	}
296	}
297
298	// Iterator interface
299
300	/// Can be used in iterator like loop without memcpy to another buffer:
301	/// while (try inflate.next()) \|buf\| { ... }
302	pub fn next(self: *Self) Error!?[]const u8 {
303	const out = try self.get(0);
304	if (out.len == 0) return null;
305	return out;
306	}
307
308	/// Returns decompressed data from internal sliding window buffer.
309	/// Returned buffer can be any length between 0 and `limit` bytes.
310	/// 0 returned bytes means end of stream reached.
311	/// With limit=0 returns as much data can. It newer will be more
312	/// than 65536 bytes, which is limit of internal buffer.
313	pub fn get(self: *Self, limit: usize) Error![]const u8 {
314	while (true) {
315	const out = self.hist.readAtMost(limit);
316	if (out.len > 0) {
317	self.hasher.update(out);
318	return out;
319	}
320	if (self.state == .end) return out;
321	try self.step();
322	}
323	}
324
325	// Reader interface
326
327	pub const Reader = std.io.Reader(*Self, Error, read);
328
329	/// Returns the number of bytes read. It may be less than buffer.len.
330	/// If the number of bytes read is 0, it means end of stream.
331	/// End of stream is not an error condition.
332	pub fn read(self: *Self, buffer: []u8) Error!usize {
333	const out = try self.get(buffer.len);
334	@memcpy(buffer[0..out.len], out);
335	return out.len;
336	}
337
338	pub fn reader(self: *Self) Reader {
339	return .{ .context = self };
340	}
341	};
342	}
343
344	test "flate.Inflate struct sizes" {
345	var fbs = std.io.fixedBufferStream("");
346	const ReaderType = @TypeOf(fbs.reader());
347	const inflate_size = @sizeOf(Inflate(.gzip, ReaderType));
348
349	try testing.expectEqual(76320, inflate_size);
350	try testing.expectEqual(
351	@sizeOf(CircularBuffer) + @sizeOf(hfd.LiteralDecoder) + @sizeOf(hfd.DistanceDecoder) + 48,
352	inflate_size,
353	);
354	try testing.expectEqual(65536 + 8 + 8, @sizeOf(CircularBuffer));
355	try testing.expectEqual(8, @sizeOf(Container.raw.Hasher()));
356	try testing.expectEqual(24, @sizeOf(BitReader(ReaderType)));
357	try testing.expectEqual(6384, @sizeOf(hfd.LiteralDecoder));
358	try testing.expectEqual(4336, @sizeOf(hfd.DistanceDecoder));
359	}
360
361	test "flate.Inflate decompress" {
362	const cases = [_]struct {
363	in: []const u8,
364	out: []const u8,
365	}{
366	// non compressed block (type 0)
367	.{
368	.in = &[_]u8{
369	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
370	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
371	},
372	.out = "Hello world\n",
373	},
374	// fixed code block (type 1)
375	.{
376	.in = &[_]u8{
377	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
378	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
379	},
380	.out = "Hello world\n",
381	},
382	// dynamic block (type 2)
383	.{
384	.in = &[_]u8{
385	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
386	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
387	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
388	},
389	.out = "ABCDEABCD ABCDEABCD",
390	},
391	};
392	for (cases) \|c\| {
393	var fb = std.io.fixedBufferStream(c.in);
394	var al = std.ArrayList(u8).init(testing.allocator);
395	defer al.deinit();
396
397	try decompress(.raw, fb.reader(), al.writer());
398	try testing.expectEqualStrings(c.out, al.items);
399	}
400	}
401
402	test "flate.Inflate gzip decompress" {
403	const cases = [_]struct {
404	in: []const u8,
405	out: []const u8,
406	}{
407	// non compressed block (type 0)
408	.{
409	.in = &[_]u8{
410	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
411	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
412	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
413	0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
414	0x0c, 0x00, 0x00, 0x00, // gzip footer: size
415	},
416	.out = "Hello world\n",
417	},
418	// fixed code block (type 1)
419	.{
420	.in = &[_]u8{
421	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
422	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
423	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
424	0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
425	},
426	.out = "Hello world\n",
427	},
428	// dynamic block (type 2)
429	.{
430	.in = &[_]u8{
431	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
432	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
433	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
434	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
435	0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
436	},
437	.out = "ABCDEABCD ABCDEABCD",
438	},
439	// gzip header with name
440	.{
441	.in = &[_]u8{
442	0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
443	0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
444	0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
445	},
446	.out = "Hello world\n",
447	},
448	};
449	for (cases) \|c\| {
450	var fb = std.io.fixedBufferStream(c.in);
451	var al = std.ArrayList(u8).init(testing.allocator);
452	defer al.deinit();
453
454	try decompress(.gzip, fb.reader(), al.writer());
455	try testing.expectEqualStrings(c.out, al.items);
456	}
457	}
458
459	test "flate.Inflate zlib decompress" {
460	const cases = [_]struct {
461	in: []const u8,
462	out: []const u8,
463	}{
464	// non compressed block (type 0)
465	.{
466	.in = &[_]u8{
467	0x78, 0b10_0_11100, // zlib header (2 bytes)
468	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
469	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
470	0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
471	},
472	.out = "Hello world\n",
473	},
474	};
475	for (cases) \|c\| {
476	var fb = std.io.fixedBufferStream(c.in);
477	var al = std.ArrayList(u8).init(testing.allocator);
478	defer al.deinit();
479
480	try decompress(.zlib, fb.reader(), al.writer());
481	try testing.expectEqualStrings(c.out, al.items);
482	}
483	}
484
485	test "flate.Inflate fuzzing tests" {
486	const cases = [_]struct {
487	input: []const u8,
488	out: []const u8 = "",
489	err: ?anyerror = null,
490	}{
491	.{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
492	.{ .input = "empty-distance-alphabet01" },
493	.{ .input = "empty-distance-alphabet02" },
494	.{ .input = "end-of-stream", .err = error.EndOfStream },
495	.{ .input = "invalid-distance", .err = error.InvalidMatch },
496	.{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
497	.{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
498	.{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
499	.{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
500	.{ .input = "out-of-codes", .err = error.InvalidCode },
501	.{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
502	.{ .input = "puff02", .err = error.EndOfStream },
503	.{ .input = "puff03", .out = &[_]u8{0xa} },
504	.{ .input = "puff04", .err = error.InvalidCode },
505	.{ .input = "puff05", .err = error.EndOfStream },
506	.{ .input = "puff06", .err = error.EndOfStream },
507	.{ .input = "puff08", .err = error.InvalidCode },
508	.{ .input = "puff09", .out = "P" },
509	.{ .input = "puff10", .err = error.InvalidCode },
510	.{ .input = "puff11", .err = error.InvalidMatch },
511	.{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
512	.{ .input = "puff13", .err = error.IncompleteHuffmanTree },
513	.{ .input = "puff14", .err = error.EndOfStream },
514	.{ .input = "puff15", .err = error.IncompleteHuffmanTree },
515	.{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
516	.{ .input = "puff17", .err = error.InvalidDynamicBlockHeader }, // 25
517	.{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
518	.{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
519	.{ .input = "fuzz3", .err = error.InvalidMatch },
520	.{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
521	.{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
522	.{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
523	.{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
524	.{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
525	.{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
526	.{ .input = "puff23", .err = error.InvalidDynamicBlockHeader }, // 35
527	.{ .input = "puff24", .err = error.InvalidDynamicBlockHeader },
528	.{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
529	.{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
530	.{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
531	};
532
533	inline for (cases, 0..) \|c, case_no\| {
534	var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
535	var out = std.ArrayList(u8).init(testing.allocator);
536	defer out.deinit();
537	errdefer std.debug.print("test case failed {}\n", .{case_no});
538
539	if (c.err) \|expected_err\| {
540	try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
541	} else {
542	try decompress(.raw, in.reader(), out.writer());
543	try testing.expectEqualStrings(c.out, out.items);
544	}
545	}
546	}

lib/std/compress/flate/root.zig created+133

...	@@ -0,0 +1,133 @@
1	pub const flate = @import("flate.zig");
2	pub const gzip = @import("gzip.zig");
3	pub const zlib = @import("zlib.zig");
4
5	test "flate" {
6	_ = @import("deflate.zig");
7	_ = @import("inflate.zig");
8	}
9
10	test "flate public interface" {
11	const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };
12
13	// deflate final stored block, header + plain (stored) data
14	const deflate_block = [_]u8{
15	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
16	} ++ plain_data;
17
18	// gzip header/footer + deflate block
19	const gzip_data =
20	[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
21	deflate_block ++
22	[_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
23
24	// zlib header/footer + deflate block
25	const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
26	deflate_block ++
27	[_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
28
29	try testInterface(gzip, &gzip_data, &plain_data);
30	try testInterface(zlib, &zlib_data, &plain_data);
31	try testInterface(flate, &deflate_block, &plain_data);
32	}
33
34	fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void {
35	const std = @import("std");
36	const testing = std.testing;
37	const fixedBufferStream = std.io.fixedBufferStream;
38
39	var buffer1: [64]u8 = undefined;
40	var buffer2: [64]u8 = undefined;
41
42	var compressed = fixedBufferStream(&buffer1);
43	var plain = fixedBufferStream(&buffer2);
44
45	// decompress
46	{
47	var in = fixedBufferStream(gzip_data);
48	try pkg.decompress(in.reader(), plain.writer());
49	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
50	}
51	plain.reset();
52	compressed.reset();
53
54	// compress/decompress
55	{
56	var in = fixedBufferStream(plain_data);
57	try pkg.compress(in.reader(), compressed.writer(), .{});
58	compressed.reset();
59	try pkg.decompress(compressed.reader(), plain.writer());
60	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
61	}
62	plain.reset();
63	compressed.reset();
64
65	// compressor/decompressor
66	{
67	var in = fixedBufferStream(plain_data);
68	var cmp = try pkg.compressor(compressed.writer(), .{});
69	try cmp.compress(in.reader());
70	try cmp.finish();
71
72	compressed.reset();
73	var dcp = pkg.decompressor(compressed.reader());
74	try dcp.decompress(plain.writer());
75	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
76	}
77	plain.reset();
78	compressed.reset();
79
80	// huffman
81	{
82	// huffman compress/decompress
83	{
84	var in = fixedBufferStream(plain_data);
85	try pkg.huffman.compress(in.reader(), compressed.writer());
86	compressed.reset();
87	try pkg.decompress(compressed.reader(), plain.writer());
88	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
89	}
90	plain.reset();
91	compressed.reset();
92
93	// huffman compressor/decompressor
94	{
95	var in = fixedBufferStream(plain_data);
96	var cmp = try pkg.huffman.compressor(compressed.writer());
97	try cmp.compress(in.reader());
98	try cmp.finish();
99
100	compressed.reset();
101	try pkg.decompress(compressed.reader(), plain.writer());
102	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
103	}
104	}
105	plain.reset();
106	compressed.reset();
107
108	// store
109	{
110	// store compress/decompress
111	{
112	var in = fixedBufferStream(plain_data);
113	try pkg.store.compress(in.reader(), compressed.writer());
114	compressed.reset();
115	try pkg.decompress(compressed.reader(), plain.writer());
116	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
117	}
118	plain.reset();
119	compressed.reset();
120
121	// store compressor/decompressor
122	{
123	var in = fixedBufferStream(plain_data);
124	var cmp = try pkg.store.compressor(compressed.writer());
125	try cmp.compress(in.reader());
126	try cmp.finish();
127
128	compressed.reset();
129	try pkg.decompress(compressed.reader(), plain.writer());
130	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
131	}
132	}
133	}

lib/std/compress/flate/testdata/block_writer.zig created+606

...	@@ -0,0 +1,606 @@
1	const Token = @import("../Token.zig");
2
3	pub const TestCase = struct {
4	tokens: []const Token,
5	input: []const u8 = "", // File name of input data matching the tokens.
6	want: []const u8 = "", // File name of data with the expected output with input available.
7	want_no_input: []const u8 = "", // File name of the expected output when no input is available.
8	};
9
10	pub const testCases = blk: {
11	@setEvalBranchQuota(4096 * 2);
12
13	const L = Token.initLiteral;
14	const M = Token.initMatch;
15	const ml = M(1, 258); // Maximum length token. Used to reduce the size of writeBlockTests
16
17	break :blk &[_]TestCase{
18	TestCase{
19	.input = "huffman-null-max.input",
20	.want = "huffman-null-max.{s}.expect",
21	.want_no_input = "huffman-null-max.{s}.expect-noinput",
22	.tokens = &[_]Token{
23	L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
24	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
25	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
26	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
27	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
28	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
29	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
30	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
31	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
32	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
33	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
34	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
35	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, L(0x0), L(0x0),
36	},
37	},
38	TestCase{
39	.input = "huffman-pi.input",
40	.want = "huffman-pi.{s}.expect",
41	.want_no_input = "huffman-pi.{s}.expect-noinput",
42	.tokens = &[_]Token{
43	L('3'), L('.'), L('1'), L('4'), L('1'), L('5'), L('9'), L('2'),
44	L('6'), L('5'), L('3'), L('5'), L('8'), L('9'), L('7'), L('9'),
45	L('3'), L('2'), L('3'), L('8'), L('4'), L('6'), L('2'), L('6'),
46	L('4'), L('3'), L('3'), L('8'), L('3'), L('2'), L('7'), L('9'),
47	L('5'), L('0'), L('2'), L('8'), L('8'), L('4'), L('1'), L('9'),
48	L('7'), L('1'), L('6'), L('9'), L('3'), L('9'), L('9'), L('3'),
49	L('7'), L('5'), L('1'), L('0'), L('5'), L('8'), L('2'), L('0'),
50	L('9'), L('7'), L('4'), L('9'), L('4'), L('4'), L('5'), L('9'),
51	L('2'), L('3'), L('0'), L('7'), L('8'), L('1'), L('6'), L('4'),
52	L('0'), L('6'), L('2'), L('8'), L('6'), L('2'), L('0'), L('8'),
53	L('9'), L('9'), L('8'), L('6'), L('2'), L('8'), L('0'), L('3'),
54	L('4'), L('8'), L('2'), L('5'), L('3'), L('4'), L('2'), L('1'),
55	L('1'), L('7'), L('0'), L('6'), L('7'), L('9'), L('8'), L('2'),
56	L('1'), L('4'), L('8'), L('0'), L('8'), L('6'), L('5'), L('1'),
57	L('3'), L('2'), L('8'), L('2'), L('3'), L('0'), L('6'), L('6'),
58	L('4'), L('7'), L('0'), L('9'), L('3'), L('8'), L('4'), L('4'),
59	L('6'), L('0'), L('9'), L('5'), L('5'), L('0'), L('5'), L('8'),
60	L('2'), L('2'), L('3'), L('1'), L('7'), L('2'), L('5'), L('3'),
61	L('5'), L('9'), L('4'), L('0'), L('8'), L('1'), L('2'), L('8'),
62	L('4'), L('8'), L('1'), L('1'), L('1'), L('7'), L('4'), M(127, 4),
63	L('4'), L('1'), L('0'), L('2'), L('7'), L('0'), L('1'), L('9'),
64	L('3'), L('8'), L('5'), L('2'), L('1'), L('1'), L('0'), L('5'),
65	L('5'), L('5'), L('9'), L('6'), L('4'), L('4'), L('6'), L('2'),
66	L('2'), L('9'), L('4'), L('8'), L('9'), L('5'), L('4'), L('9'),
67	L('3'), L('0'), L('3'), L('8'), L('1'), M(19, 4), L('2'), L('8'),
68	L('8'), L('1'), L('0'), L('9'), L('7'), L('5'), L('6'), L('6'),
69	L('5'), L('9'), L('3'), L('3'), L('4'), L('4'), L('6'), M(72, 4),
70	L('7'), L('5'), L('6'), L('4'), L('8'), L('2'), L('3'), L('3'),
71	L('7'), L('8'), L('6'), L('7'), L('8'), L('3'), L('1'), L('6'),
72	L('5'), L('2'), L('7'), L('1'), L('2'), L('0'), L('1'), L('9'),
73	L('0'), L('9'), L('1'), L('4'), M(27, 4), L('5'), L('6'), L('6'),
74	L('9'), L('2'), L('3'), L('4'), L('6'), M(179, 4), L('6'), L('1'),
75	L('0'), L('4'), L('5'), L('4'), L('3'), L('2'), L('6'), M(51, 4),
76	L('1'), L('3'), L('3'), L('9'), L('3'), L('6'), L('0'), L('7'),
77	L('2'), L('6'), L('0'), L('2'), L('4'), L('9'), L('1'), L('4'),
78	L('1'), L('2'), L('7'), L('3'), L('7'), L('2'), L('4'), L('5'),
79	L('8'), L('7'), L('0'), L('0'), L('6'), L('6'), L('0'), L('6'),
80	L('3'), L('1'), L('5'), L('5'), L('8'), L('8'), L('1'), L('7'),
81	L('4'), L('8'), L('8'), L('1'), L('5'), L('2'), L('0'), L('9'),
82	L('2'), L('0'), L('9'), L('6'), L('2'), L('8'), L('2'), L('9'),
83	L('2'), L('5'), L('4'), L('0'), L('9'), L('1'), L('7'), L('1'),
84	L('5'), L('3'), L('6'), L('4'), L('3'), L('6'), L('7'), L('8'),
85	L('9'), L('2'), L('5'), L('9'), L('0'), L('3'), L('6'), L('0'),
86	L('0'), L('1'), L('1'), L('3'), L('3'), L('0'), L('5'), L('3'),
87	L('0'), L('5'), L('4'), L('8'), L('8'), L('2'), L('0'), L('4'),
88	L('6'), L('6'), L('5'), L('2'), L('1'), L('3'), L('8'), L('4'),
89	L('1'), L('4'), L('6'), L('9'), L('5'), L('1'), L('9'), L('4'),
90	L('1'), L('5'), L('1'), L('1'), L('6'), L('0'), L('9'), L('4'),
91	L('3'), L('3'), L('0'), L('5'), L('7'), L('2'), L('7'), L('0'),
92	L('3'), L('6'), L('5'), L('7'), L('5'), L('9'), L('5'), L('9'),
93	L('1'), L('9'), L('5'), L('3'), L('0'), L('9'), L('2'), L('1'),
94	L('8'), L('6'), L('1'), L('1'), L('7'), M(234, 4), L('3'), L('2'),
95	M(10, 4), L('9'), L('3'), L('1'), L('0'), L('5'), L('1'), L('1'),
96	L('8'), L('5'), L('4'), L('8'), L('0'), L('7'), M(271, 4), L('3'),
97	L('7'), L('9'), L('9'), L('6'), L('2'), L('7'), L('4'), L('9'),
98	L('5'), L('6'), L('7'), L('3'), L('5'), L('1'), L('8'), L('8'),
99	L('5'), L('7'), L('5'), L('2'), L('7'), L('2'), L('4'), L('8'),
100	L('9'), L('1'), L('2'), L('2'), L('7'), L('9'), L('3'), L('8'),
101	L('1'), L('8'), L('3'), L('0'), L('1'), L('1'), L('9'), L('4'),
102	L('9'), L('1'), L('2'), L('9'), L('8'), L('3'), L('3'), L('6'),
103	L('7'), L('3'), L('3'), L('6'), L('2'), L('4'), L('4'), L('0'),
104	L('6'), L('5'), L('6'), L('6'), L('4'), L('3'), L('0'), L('8'),
105	L('6'), L('0'), L('2'), L('1'), L('3'), L('9'), L('4'), L('9'),
106	L('4'), L('6'), L('3'), L('9'), L('5'), L('2'), L('2'), L('4'),
107	L('7'), L('3'), L('7'), L('1'), L('9'), L('0'), L('7'), L('0'),
108	L('2'), L('1'), L('7'), L('9'), L('8'), M(154, 5), L('7'), L('0'),
109	L('2'), L('7'), L('7'), L('0'), L('5'), L('3'), L('9'), L('2'),
110	L('1'), L('7'), L('1'), L('7'), L('6'), L('2'), L('9'), L('3'),
111	L('1'), L('7'), L('6'), L('7'), L('5'), M(563, 5), L('7'), L('4'),
112	L('8'), L('1'), M(7, 4), L('6'), L('6'), L('9'), L('4'), L('0'),
113	M(488, 4), L('0'), L('0'), L('0'), L('5'), L('6'), L('8'), L('1'),
114	L('2'), L('7'), L('1'), L('4'), L('5'), L('2'), L('6'), L('3'),
115	L('5'), L('6'), L('0'), L('8'), L('2'), L('7'), L('7'), L('8'),
116	L('5'), L('7'), L('7'), L('1'), L('3'), L('4'), L('2'), L('7'),
117	L('5'), L('7'), L('7'), L('8'), L('9'), L('6'), M(298, 4), L('3'),
118	L('6'), L('3'), L('7'), L('1'), L('7'), L('8'), L('7'), L('2'),
119	L('1'), L('4'), L('6'), L('8'), L('4'), L('4'), L('0'), L('9'),
120	L('0'), L('1'), L('2'), L('2'), L('4'), L('9'), L('5'), L('3'),
121	L('4'), L('3'), L('0'), L('1'), L('4'), L('6'), L('5'), L('4'),
122	L('9'), L('5'), L('8'), L('5'), L('3'), L('7'), L('1'), L('0'),
123	L('5'), L('0'), L('7'), L('9'), M(203, 4), L('6'), M(340, 4), L('8'),
124	L('9'), L('2'), L('3'), L('5'), L('4'), M(458, 4), L('9'), L('5'),
125	L('6'), L('1'), L('1'), L('2'), L('1'), L('2'), L('9'), L('0'),
126	L('2'), L('1'), L('9'), L('6'), L('0'), L('8'), L('6'), L('4'),
127	L('0'), L('3'), L('4'), L('4'), L('1'), L('8'), L('1'), L('5'),
128	L('9'), L('8'), L('1'), L('3'), L('6'), L('2'), L('9'), L('7'),
129	L('7'), L('4'), M(117, 4), L('0'), L('9'), L('9'), L('6'), L('0'),
130	L('5'), L('1'), L('8'), L('7'), L('0'), L('7'), L('2'), L('1'),
131	L('1'), L('3'), L('4'), L('9'), M(1, 5), L('8'), L('3'), L('7'),
132	L('2'), L('9'), L('7'), L('8'), L('0'), L('4'), L('9'), L('9'),
133	M(731, 4), L('9'), L('7'), L('3'), L('1'), L('7'), L('3'), L('2'),
134	L('8'), M(395, 4), L('6'), L('3'), L('1'), L('8'), L('5'), M(770, 4),
135	M(745, 4), L('4'), L('5'), L('5'), L('3'), L('4'), L('6'), L('9'),
136	L('0'), L('8'), L('3'), L('0'), L('2'), L('6'), L('4'), L('2'),
137	L('5'), L('2'), L('2'), L('3'), L('0'), M(740, 4), M(616, 4), L('8'),
138	L('5'), L('0'), L('3'), L('5'), L('2'), L('6'), L('1'), L('9'),
139	L('3'), L('1'), L('1'), M(531, 4), L('1'), L('0'), L('1'), L('0'),
140	L('0'), L('0'), L('3'), L('1'), L('3'), L('7'), L('8'), L('3'),
141	L('8'), L('7'), L('5'), L('2'), L('8'), L('8'), L('6'), L('5'),
142	L('8'), L('7'), L('5'), L('3'), L('3'), L('2'), L('0'), L('8'),
143	L('3'), L('8'), L('1'), L('4'), L('2'), L('0'), L('6'), M(321, 4),
144	M(300, 4), L('1'), L('4'), L('7'), L('3'), L('0'), L('3'), L('5'),
145	L('9'), M(815, 5), L('9'), L('0'), L('4'), L('2'), L('8'), L('7'),
146	L('5'), L('5'), L('4'), L('6'), L('8'), L('7'), L('3'), L('1'),
147	L('1'), L('5'), L('9'), L('5'), M(854, 4), L('3'), L('8'), L('8'),
148	L('2'), L('3'), L('5'), L('3'), L('7'), L('8'), L('7'), L('5'),
149	M(896, 5), L('9'), M(315, 4), L('1'), M(329, 4), L('8'), L('0'), L('5'),
150	L('3'), M(395, 4), L('2'), L('2'), L('6'), L('8'), L('0'), L('6'),
151	L('6'), L('1'), L('3'), L('0'), L('0'), L('1'), L('9'), L('2'),
152	L('7'), L('8'), L('7'), L('6'), L('6'), L('1'), L('1'), L('1'),
153	L('9'), L('5'), L('9'), M(568, 4), L('6'), M(293, 5), L('8'), L('9'),
154	L('3'), L('8'), L('0'), L('9'), L('5'), L('2'), L('5'), L('7'),
155	L('2'), L('0'), L('1'), L('0'), L('6'), L('5'), L('4'), L('8'),
156	L('5'), L('8'), L('6'), L('3'), L('2'), L('7'), M(155, 4), L('9'),
157	L('3'), L('6'), L('1'), L('5'), L('3'), M(545, 4), M(349, 5), L('2'),
158	L('3'), L('0'), L('3'), L('0'), L('1'), L('9'), L('5'), L('2'),
159	L('0'), L('3'), L('5'), L('3'), L('0'), L('1'), L('8'), L('5'),
160	L('2'), M(370, 4), M(118, 4), L('3'), L('6'), L('2'), L('2'), L('5'),
161	L('9'), L('9'), L('4'), L('1'), L('3'), M(597, 4), L('4'), L('9'),
162	L('7'), L('2'), L('1'), L('7'), M(223, 4), L('3'), L('4'), L('7'),
163	L('9'), L('1'), L('3'), L('1'), L('5'), L('1'), L('5'), L('5'),
164	L('7'), L('4'), L('8'), L('5'), L('7'), L('2'), L('4'), L('2'),
165	L('4'), L('5'), L('4'), L('1'), L('5'), L('0'), L('6'), L('9'),
166	M(320, 4), L('8'), L('2'), L('9'), L('5'), L('3'), L('3'), L('1'),
167	L('1'), L('6'), L('8'), L('6'), L('1'), L('7'), L('2'), L('7'),
168	L('8'), M(824, 4), L('9'), L('0'), L('7'), L('5'), L('0'), L('9'),
169	M(270, 4), L('7'), L('5'), L('4'), L('6'), L('3'), L('7'), L('4'),
170	L('6'), L('4'), L('9'), L('3'), L('9'), L('3'), L('1'), L('9'),
171	L('2'), L('5'), L('5'), L('0'), L('6'), L('0'), L('4'), L('0'),
172	L('0'), L('9'), M(620, 4), L('1'), L('6'), L('7'), L('1'), L('1'),
173	L('3'), L('9'), L('0'), L('0'), L('9'), L('8'), M(822, 4), L('4'),
174	L('0'), L('1'), L('2'), L('8'), L('5'), L('8'), L('3'), L('6'),
175	L('1'), L('6'), L('0'), L('3'), L('5'), L('6'), L('3'), L('7'),
176	L('0'), L('7'), L('6'), L('6'), L('0'), L('1'), L('0'), L('4'),
177	M(371, 4), L('8'), L('1'), L('9'), L('4'), L('2'), L('9'), M(1055, 5),
178	M(240, 4), M(652, 4), L('7'), L('8'), L('3'), L('7'), L('4'), M(1193, 4),
179	L('8'), L('2'), L('5'), L('5'), L('3'), L('7'), M(522, 5), L('2'),
180	L('6'), L('8'), M(47, 4), L('4'), L('0'), L('4'), L('7'), M(466, 4),
181	L('4'), M(1206, 4), M(910, 4), L('8'), L('4'), M(937, 4), L('6'), M(800, 6),
182	L('3'), L('3'), L('1'), L('3'), L('6'), L('7'), L('7'), L('0'),
183	L('2'), L('8'), L('9'), L('8'), L('9'), L('1'), L('5'), L('2'),
184	M(99, 4), L('5'), L('2'), L('1'), L('6'), L('2'), L('0'), L('5'),
185	L('6'), L('9'), L('6'), M(1042, 4), L('0'), L('5'), L('8'), M(1144, 4),
186	L('5'), M(1177, 4), L('5'), L('1'), L('1'), M(522, 4), L('8'), L('2'),
187	L('4'), L('3'), L('0'), L('0'), L('3'), L('5'), L('5'), L('8'),
188	L('7'), L('6'), L('4'), L('0'), L('2'), L('4'), L('7'), L('4'),
189	L('9'), L('6'), L('4'), L('7'), L('3'), L('2'), L('6'), L('3'),
190	M(1087, 4), L('9'), L('9'), L('2'), M(1100, 4), L('4'), L('2'), L('6'),
191	L('9'), M(710, 6), L('7'), M(471, 4), L('4'), M(1342, 4), M(1054, 4), L('9'),
192	L('3'), L('4'), L('1'), L('7'), M(430, 4), L('1'), L('2'), M(43, 4),
193	L('4'), M(415, 4), L('1'), L('5'), L('0'), L('3'), L('0'), L('2'),
194	L('8'), L('6'), L('1'), L('8'), L('2'), L('9'), L('7'), L('4'),
195	L('5'), L('5'), L('5'), L('7'), L('0'), L('6'), L('7'), L('4'),
196	M(310, 4), L('5'), L('0'), L('5'), L('4'), L('9'), L('4'), L('5'),
197	L('8'), M(454, 4), L('9'), M(82, 4), L('5'), L('6'), M(493, 4), L('7'),
198	L('2'), L('1'), L('0'), L('7'), L('9'), M(346, 4), L('3'), L('0'),
199	M(267, 4), L('3'), L('2'), L('1'), L('1'), L('6'), L('5'), L('3'),
200	L('4'), L('4'), L('9'), L('8'), L('7'), L('2'), L('0'), L('2'),
201	L('7'), M(284, 4), L('0'), L('2'), L('3'), L('6'), L('4'), M(559, 4),
202	L('5'), L('4'), L('9'), L('9'), L('1'), L('1'), L('9'), L('8'),
203	M(1049, 4), L('4'), M(284, 4), L('5'), L('3'), L('5'), L('6'), L('6'),
204	L('3'), L('6'), L('9'), M(1105, 4), L('2'), L('6'), L('5'), M(741, 4),
205	L('7'), L('8'), L('6'), L('2'), L('5'), L('5'), L('1'), M(987, 4),
206	L('1'), L('7'), L('5'), L('7'), L('4'), L('6'), L('7'), L('2'),
207	L('8'), L('9'), L('0'), L('9'), L('7'), L('7'), L('7'), L('7'),
208	M(1108, 5), L('0'), L('0'), L('0'), M(1534, 4), L('7'), L('0'), M(1248, 4),
209	L('6'), M(1002, 4), L('4'), L('9'), L('1'), M(1055, 4), M(664, 4), L('2'),
210	L('1'), L('4'), L('7'), L('7'), L('2'), L('3'), L('5'), L('0'),
211	L('1'), L('4'), L('1'), L('4'), M(1604, 4), L('3'), L('5'), L('6'),
212	M(1200, 4), L('1'), L('6'), L('1'), L('3'), L('6'), L('1'), L('1'),
213	L('5'), L('7'), L('3'), L('5'), L('2'), L('5'), M(1285, 4), L('3'),
214	L('4'), M(92, 4), L('1'), L('8'), M(1148, 4), L('8'), L('4'), M(1512, 4),
215	L('3'), L('3'), L('2'), L('3'), L('9'), L('0'), L('7'), L('3'),
216	L('9'), L('4'), L('1'), L('4'), L('3'), L('3'), L('3'), L('4'),
217	L('5'), L('4'), L('7'), L('7'), L('6'), L('2'), L('4'), M(579, 4),
218	L('2'), L('5'), L('1'), L('8'), L('9'), L('8'), L('3'), L('5'),
219	L('6'), L('9'), L('4'), L('8'), L('5'), L('5'), L('6'), L('2'),
220	L('0'), L('9'), L('9'), L('2'), L('1'), L('9'), L('2'), L('2'),
221	L('2'), L('1'), L('8'), L('4'), L('2'), L('7'), M(575, 4), L('2'),
222	M(187, 4), L('6'), L('8'), L('8'), L('7'), L('6'), L('7'), L('1'),
223	L('7'), L('9'), L('0'), M(86, 4), L('0'), M(263, 5), L('6'), L('6'),
224	M(1000, 4), L('8'), L('8'), L('6'), L('2'), L('7'), L('2'), M(1757, 4),
225	L('1'), L('7'), L('8'), L('6'), L('0'), L('8'), L('5'), L('7'),
226	M(116, 4), L('3'), M(765, 5), L('7'), L('9'), L('7'), L('6'), L('6'),
227	L('8'), L('1'), M(702, 4), L('0'), L('0'), L('9'), L('5'), L('3'),
228	L('8'), L('8'), M(1593, 4), L('3'), M(1702, 4), L('0'), L('6'), L('8'),
229	L('0'), L('0'), L('6'), L('4'), L('2'), L('2'), L('5'), L('1'),
230	L('2'), L('5'), L('2'), M(1404, 4), L('7'), L('3'), L('9'), L('2'),
231	M(664, 4), M(1141, 4), L('4'), M(1716, 5), L('8'), L('6'), L('2'), L('6'),
232	L('9'), L('4'), L('5'), M(486, 4), L('4'), L('1'), L('9'), L('6'),
233	L('5'), L('2'), L('8'), L('5'), L('0'), M(154, 4), M(925, 4), L('1'),
234	L('8'), L('6'), L('3'), M(447, 4), L('4'), M(341, 5), L('2'), L('0'),
235	L('3'), L('9'), M(1420, 4), L('4'), L('5'), M(701, 4), L('2'), L('3'),
236	L('7'), M(1069, 4), L('6'), M(1297, 4), L('5'), L('6'), M(1593, 4), L('7'),
237	L('1'), L('9'), L('1'), L('7'), L('2'), L('8'), M(370, 4), L('7'),
238	L('6'), L('4'), L('6'), L('5'), L('7'), L('5'), L('7'), L('3'),
239	L('9'), M(258, 4), L('3'), L('8'), L('9'), M(1865, 4), L('8'), L('3'),
240	L('2'), L('6'), L('4'), L('5'), L('9'), L('9'), L('5'), L('8'),
241	M(1704, 4), L('0'), L('4'), L('7'), L('8'), M(479, 4), M(809, 4), L('9'),
242	M(46, 4), L('6'), L('4'), L('0'), L('7'), L('8'), L('9'), L('5'),
243	L('1'), M(143, 4), L('6'), L('8'), L('3'), M(304, 4), L('2'), L('5'),
244	L('9'), L('5'), L('7'), L('0'), M(1129, 4), L('8'), L('2'), L('2'),
245	M(713, 4), L('2'), M(1564, 4), L('4'), L('0'), L('7'), L('7'), L('2'),
246	L('6'), L('7'), L('1'), L('9'), L('4'), L('7'), L('8'), M(794, 4),
247	L('8'), L('2'), L('6'), L('0'), L('1'), L('4'), L('7'), L('6'),
248	L('9'), L('9'), L('0'), L('9'), M(1257, 4), L('0'), L('1'), L('3'),
249	L('6'), L('3'), L('9'), L('4'), L('4'), L('3'), M(640, 4), L('3'),
250	L('0'), M(262, 4), L('2'), L('0'), L('3'), L('4'), L('9'), L('6'),
251	L('2'), L('5'), L('2'), L('4'), L('5'), L('1'), L('7'), M(950, 4),
252	L('9'), L('6'), L('5'), L('1'), L('4'), L('3'), L('1'), L('4'),
253	L('2'), L('9'), L('8'), L('0'), L('9'), L('1'), L('9'), L('0'),
254	L('6'), L('5'), L('9'), L('2'), M(643, 4), L('7'), L('2'), L('2'),
255	L('1'), L('6'), L('9'), L('6'), L('4'), L('6'), M(1050, 4), M(123, 4),
256	L('5'), M(1295, 4), L('4'), M(1382, 5), L('8'), M(1370, 4), L('9'), L('7'),
257	M(1404, 4), L('5'), L('4'), M(1182, 4), M(575, 4), L('7'), M(1627, 4), L('8'),
258	L('4'), L('6'), L('8'), L('1'), L('3'), M(141, 4), L('6'), L('8'),
259	L('3'), L('8'), L('6'), L('8'), L('9'), L('4'), L('2'), L('7'),
260	L('7'), L('4'), L('1'), L('5'), L('5'), L('9'), L('9'), L('1'),
261	L('8'), L('5'), M(91, 4), L('2'), L('4'), L('5'), L('9'), L('5'),
262	L('3'), L('9'), L('5'), L('9'), L('4'), L('3'), L('1'), M(1464, 4),
263	L('7'), M(19, 4), L('6'), L('8'), L('0'), L('8'), L('4'), L('5'),
264	M(744, 4), L('7'), L('3'), M(2079, 4), L('9'), L('5'), L('8'), L('4'),
265	L('8'), L('6'), L('5'), L('3'), L('8'), M(1769, 4), L('6'), L('2'),
266	M(243, 4), L('6'), L('0'), L('9'), M(1207, 4), L('6'), L('0'), L('8'),
267	L('0'), L('5'), L('1'), L('2'), L('4'), L('3'), L('8'), L('8'),
268	L('4'), M(315, 4), M(12, 4), L('4'), L('1'), L('3'), M(784, 4), L('7'),
269	L('6'), L('2'), L('7'), L('8'), M(834, 4), L('7'), L('1'), L('5'),
270	M(1436, 4), L('3'), L('5'), L('9'), L('9'), L('7'), L('7'), L('0'),
271	L('0'), L('1'), L('2'), L('9'), M(1139, 4), L('8'), L('9'), L('4'),
272	L('4'), L('1'), M(632, 4), L('6'), L('8'), L('5'), L('5'), M(96, 4),
273	L('4'), L('0'), L('6'), L('3'), M(2279, 4), L('2'), L('0'), L('7'),
274	L('2'), L('2'), M(345, 4), M(516, 5), L('4'), L('8'), L('1'), L('5'),
275	L('8'), M(518, 4), M(511, 4), M(635, 4), M(665, 4), L('3'), L('9'), L('4'),
276	L('5'), L('2'), L('2'), L('6'), L('7'), M(1175, 6), L('8'), M(1419, 4),
277	L('2'), L('1'), M(747, 4), L('2'), M(904, 4), L('5'), L('4'), L('6'),
278	L('6'), L('6'), M(1308, 4), L('2'), L('3'), L('9'), L('8'), L('6'),
279	L('4'), L('5'), L('6'), M(1221, 4), L('1'), L('6'), L('3'), L('5'),
280	M(596, 5), M(2066, 4), L('7'), M(2222, 4), L('9'), L('8'), M(1119, 4), L('9'),
281	L('3'), L('6'), L('3'), L('4'), M(1884, 4), L('7'), L('4'), L('3'),
282	L('2'), L('4'), M(1148, 4), L('1'), L('5'), L('0'), L('7'), L('6'),
283	M(1212, 4), L('7'), L('9'), L('4'), L('5'), L('1'), L('0'), L('9'),
284	M(63, 4), L('0'), L('9'), L('4'), L('0'), M(1703, 4), L('8'), L('8'),
285	L('7'), L('9'), L('7'), L('1'), L('0'), L('8'), L('9'), L('3'),
286	M(2289, 4), L('6'), L('9'), L('1'), L('3'), L('6'), L('8'), L('6'),
287	L('7'), L('2'), M(604, 4), M(511, 4), L('5'), M(1344, 4), M(1129, 4), M(2050, 4),
288	L('1'), L('7'), L('9'), L('2'), L('8'), L('6'), L('8'), M(2253, 4),
289	L('8'), L('7'), L('4'), L('7'), M(1951, 5), L('8'), L('2'), L('4'),
290	M(2427, 4), L('8'), M(604, 4), L('7'), L('1'), L('4'), L('9'), L('0'),
291	L('9'), L('6'), L('7'), L('5'), L('9'), L('8'), M(1776, 4), L('3'),
292	L('6'), L('5'), M(309, 4), L('8'), L('1'), M(93, 4), M(1862, 4), M(2359, 4),
293	L('6'), L('8'), L('2'), L('9'), M(1407, 4), L('8'), L('7'), L('2'),
294	L('2'), L('6'), L('5'), L('8'), L('8'), L('0'), M(1554, 4), L('5'),
295	M(586, 4), L('4'), L('2'), L('7'), L('0'), L('4'), L('7'), L('7'),
296	L('5'), L('5'), M(2079, 4), L('3'), L('7'), L('9'), L('6'), L('4'),
297	L('1'), L('4'), L('5'), L('1'), L('5'), L('2'), M(1534, 4), L('2'),
298	L('3'), L('4'), L('3'), L('6'), L('4'), L('5'), L('4'), M(1503, 4),
299	L('4'), L('4'), L('4'), L('7'), L('9'), L('5'), M(61, 4), M(1316, 4),
300	M(2279, 5), L('4'), L('1'), M(1323, 4), L('3'), M(773, 4), L('5'), L('2'),
301	L('3'), L('1'), M(2114, 5), L('1'), L('6'), L('6'), L('1'), M(2227, 4),
302	L('5'), L('9'), L('6'), L('9'), L('5'), L('3'), L('6'), L('2'),
303	L('3'), L('1'), L('4'), M(1536, 4), L('2'), L('4'), L('8'), L('4'),
304	L('9'), L('3'), L('7'), L('1'), L('8'), L('7'), L('1'), L('1'),
305	L('0'), L('1'), L('4'), L('5'), L('7'), L('6'), L('5'), L('4'),
306	M(1890, 4), L('0'), L('2'), L('7'), L('9'), L('9'), L('3'), L('4'),
307	L('4'), L('0'), L('3'), L('7'), L('4'), L('2'), L('0'), L('0'),
308	L('7'), M(2368, 4), L('7'), L('8'), L('5'), L('3'), L('9'), L('0'),
309	L('6'), L('2'), L('1'), L('9'), M(666, 5), M(838, 4), L('8'), L('4'),
310	L('7'), M(979, 5), L('8'), L('3'), L('3'), L('2'), L('1'), L('4'),
311	L('4'), L('5'), L('7'), L('1'), M(645, 4), M(1911, 4), L('4'), L('3'),
312	L('5'), L('0'), M(2345, 4), M(1129, 4), L('5'), L('3'), L('1'), L('9'),
313	L('1'), L('0'), L('4'), L('8'), L('4'), L('8'), L('1'), L('0'),
314	L('0'), L('5'), L('3'), L('7'), L('0'), L('6'), M(2237, 4), M(1438, 5),
315	M(1922, 5), L('1'), M(1370, 4), L('7'), M(796, 4), L('5'), M(2029, 4), M(1037, 4),
316	L('6'), L('3'), M(2013, 5), L('4'), M(2418, 4), M(847, 5), M(1014, 5), L('8'),
317	M(1326, 5), M(2184, 5), L('9'), M(392, 4), L('9'), L('1'), M(2255, 4), L('8'),
318	L('1'), L('4'), L('6'), L('7'), L('5'), L('1'), M(1580, 4), L('1'),
319	L('2'), L('3'), L('9'), M(426, 6), L('9'), L('0'), L('7'), L('1'),
320	L('8'), L('6'), L('4'), L('9'), L('4'), L('2'), L('3'), L('1'),
321	L('9'), L('6'), L('1'), L('5'), L('6'), M(493, 4), M(1725, 4), L('9'),
322	L('5'), M(2343, 4), M(1130, 4), M(284, 4), L('6'), L('0'), L('3'), L('8'),
323	M(2598, 4), M(368, 4), M(901, 4), L('6'), L('2'), M(1115, 4), L('5'), M(2125, 4),
324	L('6'), L('3'), L('8'), L('9'), L('3'), L('7'), L('7'), L('8'),
325	L('7'), M(2246, 4), M(249, 4), L('9'), L('7'), L('9'), L('2'), L('0'),
326	L('7'), L('7'), L('3'), M(1496, 4), L('2'), L('1'), L('8'), L('2'),
327	L('5'), L('6'), M(2016, 4), L('6'), L('6'), M(1751, 4), L('4'), L('2'),
328	M(1663, 5), L('6'), M(1767, 4), L('4'), L('4'), M(37, 4), L('5'), L('4'),
329	L('9'), L('2'), L('0'), L('2'), L('6'), L('0'), L('5'), M(2740, 4),
330	M(997, 5), L('2'), L('0'), L('1'), L('4'), L('9'), M(1235, 4), L('8'),
331	L('5'), L('0'), L('7'), L('3'), M(1434, 4), L('6'), L('6'), L('6'),
332	L('0'), M(405, 4), L('2'), L('4'), L('3'), L('4'), L('0'), M(136, 4),
333	L('0'), M(1900, 4), L('8'), L('6'), L('3'), M(2391, 4), M(2021, 4), M(1068, 4),
334	M(373, 4), L('5'), L('7'), L('9'), L('6'), L('2'), L('6'), L('8'),
335	L('5'), L('6'), M(321, 4), L('5'), L('0'), L('8'), M(1316, 4), L('5'),
336	L('8'), L('7'), L('9'), L('6'), L('9'), L('9'), M(1810, 4), L('5'),
337	L('7'), L('4'), M(2585, 4), L('8'), L('4'), L('0'), M(2228, 4), L('1'),
338	L('4'), L('5'), L('9'), L('1'), M(1933, 4), L('7'), L('0'), M(565, 4),
339	L('0'), L('1'), M(3048, 4), L('1'), L('2'), M(3189, 4), L('0'), M(964, 4),
340	L('3'), L('9'), M(2859, 4), M(275, 4), L('7'), L('1'), L('5'), M(945, 4),
341	L('4'), L('2'), L('0'), M(3059, 5), L('9'), M(3011, 4), L('0'), L('7'),
342	M(834, 4), M(1942, 4), M(2736, 4), M(3171, 4), L('2'), L('1'), M(2401, 4), L('2'),
343	L('5'), L('1'), M(1404, 4), M(2373, 4), L('9'), L('2'), M(435, 4), L('8'),
344	L('2'), L('6'), M(2919, 4), L('2'), M(633, 4), L('3'), L('2'), L('1'),
345	L('5'), L('7'), L('9'), L('1'), L('9'), L('8'), L('4'), L('1'),
346	L('4'), M(2172, 5), L('9'), L('1'), L('6'), L('4'), M(1769, 5), L('9'),
347	M(2905, 5), M(2268, 4), L('7'), L('2'), L('2'), M(802, 4), L('5'), M(2213, 4),
348	M(322, 4), L('9'), L('1'), L('0'), M(189, 4), M(3164, 4), L('5'), L('2'),
349	L('8'), L('0'), L('1'), L('7'), M(562, 4), L('7'), L('1'), L('2'),
350	M(2325, 4), L('8'), L('3'), L('2'), M(884, 4), L('1'), M(1418, 4), L('0'),
351	L('9'), L('3'), L('5'), L('3'), L('9'), L('6'), L('5'), L('7'),
352	M(1612, 4), L('1'), L('0'), L('8'), L('3'), M(106, 4), L('5'), L('1'),
353	M(1915, 4), M(3419, 4), L('1'), L('4'), L('4'), L('4'), L('2'), L('1'),
354	L('0'), L('0'), M(515, 4), L('0'), L('3'), M(413, 4), L('1'), L('1'),
355	L('0'), L('3'), M(3202, 4), M(10, 4), M(39, 4), M(1539, 6), L('5'), L('1'),
356	L('6'), M(1498, 4), M(2180, 5), M(2347, 4), L('5'), M(3139, 5), L('8'), L('5'),
357	L('1'), L('7'), L('1'), L('4'), L('3'), L('7'), M(1542, 4), M(110, 4),
358	L('1'), L('5'), L('5'), L('6'), L('5'), L('0'), L('8'), L('8'),
359	M(954, 4), L('9'), L('8'), L('9'), L('8'), L('5'), L('9'), L('9'),
360	L('8'), L('2'), L('3'), L('8'), M(464, 4), M(2491, 4), L('3'), M(365, 4),
361	M(1087, 4), M(2500, 4), L('8'), M(3590, 5), L('3'), L('2'), M(264, 4), L('5'),
362	M(774, 4), L('3'), M(459, 4), L('9'), M(1052, 4), L('9'), L('8'), M(2174, 4),
363	L('4'), M(3257, 4), L('7'), M(1612, 4), L('0'), L('7'), M(230, 4), L('4'),
364	L('8'), L('1'), L('4'), L('1'), M(1338, 4), L('8'), L('5'), L('9'),
365	L('4'), L('6'), L('1'), M(3018, 4), L('8'), L('0'),
366	},
367	},
368	TestCase{
369	.input = "huffman-rand-1k.input",
370	.want = "huffman-rand-1k.{s}.expect",
371	.want_no_input = "huffman-rand-1k.{s}.expect-noinput",
372	.tokens = &[_]Token{
373	L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xd), L(0x85), L(0x94), L(0x25), L(0x80), L(0xaf), L(0xc2), L(0xfe), L(0x8d),
374	L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b),
375	L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4),
376	L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde),
377	L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14),
378	L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xd), L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c),
379	L(0x99), L(0xdf), L(0xfd), L(0x70), L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5),
380	L(0xd4), L(0x80), L(0x59), L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20),
381	L(0x1b), L(0x46), L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4),
382	L(0x7b), L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
383	L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), L(0xa0),
384	L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), L(0x75), L(0xda),
385	L(0xea), L(0x9b), L(0xa), L(0x64), L(0xb), L(0xe0), L(0x23), L(0x29), L(0xbd), L(0xf7), L(0xe7), L(0x83), L(0x3c), L(0xfb),
386	L(0xdf), L(0xb3), L(0xae), L(0x4f), L(0xa4), L(0x47), L(0x55), L(0x99), L(0xde), L(0x2f), L(0x96), L(0x6e), L(0x1c), L(0x43),
387	L(0x4c), L(0x87), L(0xe2), L(0x7c), L(0xd9), L(0x5f), L(0x4c), L(0x7c), L(0xe8), L(0x90), L(0x3), L(0xdb), L(0x30), L(0x95),
388	L(0xd6), L(0x22), L(0xc), L(0x47), L(0xb8), L(0x4d), L(0x6b), L(0xbd), L(0x24), L(0x11), L(0xab), L(0x2c), L(0xd7), L(0xbe),
389	L(0x6e), L(0x7a), L(0xd6), L(0x8), L(0xa3), L(0x98), L(0xd8), L(0xdd), L(0x15), L(0x6a), L(0xfa), L(0x93), L(0x30), L(0x1),
390	L(0x25), L(0x1d), L(0xa2), L(0x74), L(0x86), L(0x4b), L(0x6a), L(0x95), L(0xe8), L(0xe1), L(0x4e), L(0xe), L(0x76), L(0xb9),
391	L(0x49), L(0xa9), L(0x5f), L(0xa0), L(0xa6), L(0x63), L(0x3c), L(0x7e), L(0x7e), L(0x20), L(0x13), L(0x4f), L(0xbb), L(0x66),
392	L(0x92), L(0xb8), L(0x2e), L(0xa4), L(0xfa), L(0x48), L(0xcb), L(0xae), L(0xb9), L(0x3c), L(0xaf), L(0xd3), L(0x1f), L(0xe1),
393	L(0xd5), L(0x8d), L(0x42), L(0x6d), L(0xf0), L(0xfc), L(0x8c), L(0xc), L(0x0), L(0xde), L(0x40), L(0xab), L(0x8b), L(0x47),
394	L(0x97), L(0x4e), L(0xa8), L(0xcf), L(0x8e), L(0xdb), L(0xa6), L(0x8b), L(0x20), L(0x9), L(0x84), L(0x7a), L(0x66), L(0xe5),
395	L(0x98), L(0x29), L(0x2), L(0x95), L(0xe6), L(0x38), L(0x32), L(0x60), L(0x3), L(0xe3), L(0x9a), L(0x1e), L(0x54), L(0xe8),
396	L(0x63), L(0x80), L(0x48), L(0x9c), L(0xe7), L(0x63), L(0x33), L(0x6e), L(0xa0), L(0x65), L(0x83), L(0xfa), L(0xc6), L(0xba),
397	L(0x7a), L(0x43), L(0x71), L(0x5), L(0xf5), L(0x68), L(0x69), L(0x85), L(0x9c), L(0xba), L(0x45), L(0xcd), L(0x6b), L(0xb),
398	L(0x19), L(0xd1), L(0xbb), L(0x7f), L(0x70), L(0x85), L(0x92), L(0xd1), L(0xb4), L(0x64), L(0x82), L(0xb1), L(0xe4), L(0x62),
399	L(0xc5), L(0x3c), L(0x46), L(0x1f), L(0x92), L(0x31), L(0x1c), L(0x4e), L(0x41), L(0x77), L(0xf7), L(0xe7), L(0x87), L(0xa2),
400	L(0xf), L(0x6e), L(0xe8), L(0x92), L(0x3), L(0x6b), L(0xa), L(0xe7), L(0xa9), L(0x3b), L(0x11), L(0xda), L(0x66), L(0x8a),
401	L(0x29), L(0xda), L(0x79), L(0xe1), L(0x64), L(0x8d), L(0xe3), L(0x54), L(0xd4), L(0xf5), L(0xef), L(0x64), L(0x87), L(0x3b),
402	L(0xf4), L(0xc2), L(0xf4), L(0x71), L(0x13), L(0xa9), L(0xe9), L(0xe0), L(0xa2), L(0x6), L(0x14), L(0xab), L(0x5d), L(0xa7),
403	L(0x96), L(0x0), L(0xd6), L(0xc3), L(0xcc), L(0x57), L(0xed), L(0x39), L(0x6a), L(0x25), L(0xcd), L(0x76), L(0xea), L(0xba),
404	L(0x3a), L(0xf2), L(0xa1), L(0x95), L(0x5d), L(0xe5), L(0x71), L(0xcf), L(0x9c), L(0x62), L(0x9e), L(0x6a), L(0xfa), L(0xd5),
405	L(0x31), L(0xd1), L(0xa8), L(0x66), L(0x30), L(0x33), L(0xaa), L(0x51), L(0x17), L(0x13), L(0x82), L(0x99), L(0xc8), L(0x14),
406	L(0x60), L(0x9f), L(0x4d), L(0x32), L(0x6d), L(0xda), L(0x19), L(0x26), L(0x21), L(0xdc), L(0x7e), L(0x2e), L(0x25), L(0x67),
407	L(0x72), L(0xca), L(0xf), L(0x92), L(0xcd), L(0xf6), L(0xd6), L(0xcb), L(0x97), L(0x8a), L(0x33), L(0x58), L(0x73), L(0x70),
408	L(0x91), L(0x1d), L(0xbf), L(0x28), L(0x23), L(0xa3), L(0xc), L(0xf1), L(0x83), L(0xc3), L(0xc8), L(0x56), L(0x77), L(0x68),
409	L(0xe3), L(0x82), L(0xba), L(0xb9), L(0x57), L(0x56), L(0x57), L(0x9c), L(0xc3), L(0xd6), L(0x14), L(0x5), L(0x3c), L(0xb1),
410	L(0xaf), L(0x93), L(0xc8), L(0x8a), L(0x57), L(0x7f), L(0x53), L(0xfa), L(0x2f), L(0xaa), L(0x6e), L(0x66), L(0x83), L(0xfa),
411	L(0x33), L(0xd1), L(0x21), L(0xab), L(0x1b), L(0x71), L(0xb4), L(0x7c), L(0xda), L(0xfd), L(0xfb), L(0x7f), L(0x20), L(0xab),
412	L(0x5e), L(0xd5), L(0xca), L(0xfd), L(0xdd), L(0xe0), L(0xee), L(0xda), L(0xba), L(0xa8), L(0x27), L(0x99), L(0x97), L(0x69),
413	L(0xc1), L(0x3c), L(0x82), L(0x8c), L(0xa), L(0x5c), L(0x2d), L(0x5b), L(0x88), L(0x3e), L(0x34), L(0x35), L(0x86), L(0x37),
414	L(0x46), L(0x79), L(0xe1), L(0xaa), L(0x19), L(0xfb), L(0xaa), L(0xde), L(0x15), L(0x9), L(0xd), L(0x1a), L(0x57), L(0xff),
415	L(0xb5), L(0xf), L(0xf3), L(0x2b), L(0x5a), L(0x6a), L(0x4d), L(0x19), L(0x77), L(0x71), L(0x45), L(0xdf), L(0x4f), L(0xb3),
416	L(0xec), L(0xf1), L(0xeb), L(0x18), L(0x53), L(0x3e), L(0x3b), L(0x47), L(0x8), L(0x9a), L(0x73), L(0xa0), L(0x5c), L(0x8c),
417	L(0x5f), L(0xeb), L(0xf), L(0x3a), L(0xc2), L(0x43), L(0x67), L(0xb4), L(0x66), L(0x67), L(0x80), L(0x58), L(0xe), L(0xc1),
418	L(0xec), L(0x40), L(0xd4), L(0x22), L(0x94), L(0xca), L(0xf9), L(0xe8), L(0x92), L(0xe4), L(0x69), L(0x38), L(0xbe), L(0x67),
419	L(0x64), L(0xca), L(0x50), L(0xc7), L(0x6), L(0x67), L(0x42), L(0x6e), L(0xa3), L(0xf0), L(0xb7), L(0x6c), L(0xf2), L(0xe8),
420	L(0x5f), L(0xb1), L(0xaf), L(0xe7), L(0xdb), L(0xbb), L(0x77), L(0xb5), L(0xf8), L(0xcb), L(0x8), L(0xc4), L(0x75), L(0x7e),
421	L(0xc0), L(0xf9), L(0x1c), L(0x7f), L(0x3c), L(0x89), L(0x2f), L(0xd2), L(0x58), L(0x3a), L(0xe2), L(0xf8), L(0x91), L(0xb6),
422	L(0x7b), L(0x24), L(0x27), L(0xe9), L(0xae), L(0x84), L(0x8b), L(0xde), L(0x74), L(0xac), L(0xfd), L(0xd9), L(0xb7), L(0x69),
423	L(0x2a), L(0xec), L(0x32), L(0x6f), L(0xf0), L(0x92), L(0x84), L(0xf1), L(0x40), L(0xc), L(0x8a), L(0xbc), L(0x39), L(0x6e),
424	L(0x2e), L(0x73), L(0xd4), L(0x6e), L(0x8a), L(0x74), L(0x2a), L(0xdc), L(0x60), L(0x1f), L(0xa3), L(0x7), L(0xde), L(0x75),
425	L(0x8b), L(0x74), L(0xc8), L(0xfe), L(0x63), L(0x75), L(0xf6), L(0x3d), L(0x63), L(0xac), L(0x33), L(0x89), L(0xc3), L(0xf0),
426	L(0xf8), L(0x2d), L(0x6b), L(0xb4), L(0x9e), L(0x74), L(0x8b), L(0x5c), L(0x33), L(0xb4), L(0xca), L(0xa8), L(0xe4), L(0x99),
427	L(0xb6), L(0x90), L(0xa1), L(0xef), L(0xf), L(0xd3), L(0x61), L(0xb2), L(0xc6), L(0x1a), L(0x94), L(0x7c), L(0x44), L(0x55),
428	L(0xf4), L(0x45), L(0xff), L(0x9e), L(0xa5), L(0x5a), L(0xc6), L(0xa0), L(0xe8), L(0x2a), L(0xc1), L(0x8d), L(0x6f), L(0x34),
429	L(0x11), L(0xb9), L(0xbe), L(0x4e), L(0xd9), L(0x87), L(0x97), L(0x73), L(0xcf), L(0x3d), L(0x23), L(0xae), L(0xd5), L(0x1a),
430	L(0x5e), L(0xae), L(0x5d), L(0x6a), L(0x3), L(0xf9), L(0x22), L(0xd), L(0x10), L(0xd9), L(0x47), L(0x69), L(0x15), L(0x3f),
431	L(0xee), L(0x52), L(0xa3), L(0x8), L(0xd2), L(0x3c), L(0x51), L(0xf4), L(0xf8), L(0x9d), L(0xe4), L(0x98), L(0x89), L(0xc8),
432	L(0x67), L(0x39), L(0xd5), L(0x5e), L(0x35), L(0x78), L(0x27), L(0xe8), L(0x3c), L(0x80), L(0xae), L(0x79), L(0x71), L(0xd2),
433	L(0x93), L(0xf4), L(0xaa), L(0x51), L(0x12), L(0x1c), L(0x4b), L(0x1b), L(0xe5), L(0x6e), L(0x15), L(0x6f), L(0xe4), L(0xbb),
434	L(0x51), L(0x9b), L(0x45), L(0x9f), L(0xf9), L(0xc4), L(0x8c), L(0x2a), L(0xfb), L(0x1a), L(0xdf), L(0x55), L(0xd3), L(0x48),
435	L(0x93), L(0x27), L(0x1), L(0x26), L(0xc2), L(0x6b), L(0x55), L(0x6d), L(0xa2), L(0xfb), L(0x84), L(0x8b), L(0xc9), L(0x9e),
436	L(0x28), L(0xc2), L(0xef), L(0x1a), L(0x24), L(0xec), L(0x9b), L(0xae), L(0xbd), L(0x60), L(0xe9), L(0x15), L(0x35), L(0xee),
437	L(0x42), L(0xa4), L(0x33), L(0x5b), L(0xfa), L(0xf), L(0xb6), L(0xf7), L(0x1), L(0xa6), L(0x2), L(0x4c), L(0xca), L(0x90),
438	L(0x58), L(0x3a), L(0x96), L(0x41), L(0xe7), L(0xcb), L(0x9), L(0x8c), L(0xdb), L(0x85), L(0x4d), L(0xa8), L(0x89), L(0xf3),
439	L(0xb5), L(0x8e), L(0xfd), L(0x75), L(0x5b), L(0x4f), L(0xed), L(0xde), L(0x3f), L(0xeb), L(0x38), L(0xa3), L(0xbe), L(0xb0),
440	L(0x73), L(0xfc), L(0xb8), L(0x54), L(0xf7), L(0x4c), L(0x30), L(0x67), L(0x2e), L(0x38), L(0xa2), L(0x54), L(0x18), L(0xba),
441	L(0x8), L(0xbf), L(0xf2), L(0x39), L(0xd5), L(0xfe), L(0xa5), L(0x41), L(0xc6), L(0x66), L(0x66), L(0xba), L(0x81), L(0xef),
442	L(0x67), L(0xe4), L(0xe6), L(0x3c), L(0xc), L(0xca), L(0xa4), L(0xa), L(0x79), L(0xb3), L(0x57), L(0x8b), L(0x8a), L(0x75),
443	L(0x98), L(0x18), L(0x42), L(0x2f), L(0x29), L(0xa3), L(0x82), L(0xef), L(0x9f), L(0x86), L(0x6), L(0x23), L(0xe1), L(0x75),
444	L(0xfa), L(0x8), L(0xb1), L(0xde), L(0x17), L(0x4a),
445	},
446	},
447	TestCase{
448	.input = "huffman-rand-limit.input",
449	.want = "huffman-rand-limit.{s}.expect",
450	.want_no_input = "huffman-rand-limit.{s}.expect-noinput",
451	.tokens = &[_]Token{
452	L(0x61), M(1, 74), L(0xa), L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xa), L(0x85), L(0x94), L(0x25), L(0x80),
453	L(0xaf), L(0xc2), L(0xfe), L(0x8d), L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde),
454	L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73),
455	L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15),
456	L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9),
457	L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xa),
458	L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), L(0x99), L(0xdf), L(0xfd), L(0x70),
459	L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), L(0xd4), L(0x80), L(0x59),
460	L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), L(0x1b), L(0x46),
461	L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), L(0x7b),
462	L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
463	L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f),
464	L(0xa0), L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4),
465	L(0x75), L(0xda), L(0xea), L(0x9b), L(0xa),
466	},
467	},
468	TestCase{
469	.input = "huffman-shifts.input",
470	.want = "huffman-shifts.{s}.expect",
471	.want_no_input = "huffman-shifts.{s}.expect-noinput",
472	.tokens = &[_]Token{
473	L('1'), L('0'), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
474	M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
475	M(2, 258), M(2, 76), L(0xd), L(0xa), L('2'), L('3'), M(2, 258), M(2, 258),
476	M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 256),
477	},
478	},
479	TestCase{
480	.input = "huffman-text-shift.input",
481	.want = "huffman-text-shift.{s}.expect",
482	.want_no_input = "huffman-text-shift.{s}.expect-noinput",
483	.tokens = &[_]Token{
484	L('/'), L('/'), L('C'), L('o'), L('p'), L('y'), L('r'), L('i'),
485	L('g'), L('h'), L('t'), L('2'), L('0'), L('0'), L('9'), L('T'),
486	L('h'), L('G'), L('o'), L('A'), L('u'), L('t'), L('h'), L('o'),
487	L('r'), L('.'), L('A'), L('l'), L('l'), M(23, 5), L('r'), L('r'),
488	L('v'), L('d'), L('.'), L(0xd), L(0xa), L('/'), L('/'), L('U'),
489	L('o'), L('f'), L('t'), L('h'), L('i'), L('o'), L('u'), L('r'),
490	L('c'), L('c'), L('o'), L('d'), L('i'), L('g'), L('o'), L('v'),
491	L('r'), L('n'), L('d'), L('b'), L('y'), L('B'), L('S'), L('D'),
492	L('-'), L('t'), L('y'), L('l'), M(33, 4), L('l'), L('i'), L('c'),
493	L('n'), L('t'), L('h'), L('t'), L('c'), L('n'), L('b'), L('f'),
494	L('o'), L('u'), L('n'), L('d'), L('i'), L('n'), L('t'), L('h'),
495	L('L'), L('I'), L('C'), L('E'), L('N'), L('S'), L('E'), L('f'),
496	L('i'), L('l'), L('.'), L(0xd), L(0xa), L(0xd), L(0xa), L('p'),
497	L('c'), L('k'), L('g'), L('m'), L('i'), L('n'), M(11, 4), L('i'),
498	L('m'), L('p'), L('o'), L('r'), L('t'), L('"'), L('o'), L('"'),
499	M(13, 4), L('f'), L('u'), L('n'), L('c'), L('m'), L('i'), L('n'),
500	L('('), L(')'), L('{'), L(0xd), L(0xa), L(0x9), L('v'), L('r'),
501	L('b'), L('='), L('m'), L('k'), L('('), L('['), L(']'), L('b'),
502	L('y'), L('t'), L(','), L('6'), L('5'), L('5'), L('3'), L('5'),
503	L(')'), L(0xd), L(0xa), L(0x9), L('f'), L(','), L('_'), L(':'),
504	L('='), L('o'), L('.'), L('C'), L('r'), L('t'), L('('), L('"'),
505	L('h'), L('u'), L('f'), L('f'), L('m'), L('n'), L('-'), L('n'),
506	L('u'), L('l'), L('l'), L('-'), L('m'), L('x'), L('.'), L('i'),
507	L('n'), L('"'), M(34, 5), L('.'), L('W'), L('r'), L('i'), L('t'),
508	L('('), L('b'), L(')'), L(0xd), L(0xa), L('}'), L(0xd), L(0xa),
509	L('A'), L('B'), L('C'), L('D'), L('E'), L('F'), L('G'), L('H'),
510	L('I'), L('J'), L('K'), L('L'), L('M'), L('N'), L('O'), L('P'),
511	L('Q'), L('R'), L('S'), L('T'), L('U'), L('V'), L('X'), L('x'),
512	L('y'), L('z'), L('!'), L('"'), L('#'), L(0xc2), L(0xa4), L('%'),
513	L('&'), L('/'), L('?'), L('"'),
514	},
515	},
516	TestCase{
517	.input = "huffman-text.input",
518	.want = "huffman-text.{s}.expect",
519	.want_no_input = "huffman-text.{s}.expect-noinput",
520	.tokens = &[_]Token{
521	L('/'), L('/'), L(' '), L('z'), L('i'), L('g'), L(' '), L('v'),
522	L('0'), L('.'), L('1'), L('0'), L('.'), L('0'), L(0xa), L('/'),
523	L('/'), L(' '), L('c'), L('r'), L('e'), L('a'), L('t'), L('e'),
524	L(' '), L('a'), L(' '), L('f'), L('i'), L('l'), L('e'), M(5, 4),
525	L('l'), L('e'), L('d'), L(' '), L('w'), L('i'), L('t'), L('h'),
526	L(' '), L('0'), L('x'), L('0'), L('0'), L(0xa), L('c'), L('o'),
527	L('n'), L('s'), L('t'), L(' '), L('s'), L('t'), L('d'), L(' '),
528	L('='), L(' '), L('@'), L('i'), L('m'), L('p'), L('o'), L('r'),
529	L('t'), L('('), L('"'), L('s'), L('t'), L('d'), L('"'), L(')'),
530	L(';'), L(0xa), L(0xa), L('p'), L('u'), L('b'), L(' '), L('f'),
531	L('n'), L(' '), L('m'), L('a'), L('i'), L('n'), L('('), L(')'),
532	L(' '), L('!'), L('v'), L('o'), L('i'), L('d'), L(' '), L('{'),
533	L(0xa), L(' '), L(' '), L(' '), L(' '), L('v'), L('a'), L('r'),
534	L(' '), L('b'), L(' '), L('='), L(' '), L('['), L('1'), L(']'),
535	L('u'), L('8'), L('{'), L('0'), L('}'), L(' '), L(''), L(''),
536	L(' '), L('6'), L('5'), L('5'), L('3'), L('5'), L(';'), M(31, 5),
537	M(86, 6), L('f'), L(' '), L('='), L(' '), L('t'), L('r'), L('y'),
538	M(94, 4), L('.'), L('f'), L('s'), L('.'), L('c'), L('w'), L('d'),
539	L('('), L(')'), L('.'), M(144, 6), L('F'), L('i'), L('l'), L('e'),
540	L('('), M(43, 5), M(1, 4), L('"'), L('h'), L('u'), L('f'), L('f'),
541	L('m'), L('a'), L('n'), L('-'), L('n'), L('u'), L('l'), L('l'),
542	L('-'), L('m'), L('a'), L('x'), L('.'), L('i'), L('n'), L('"'),
543	L(','), M(31, 9), L('.'), L('{'), L(' '), L('.'), L('r'), L('e'),
544	L('a'), L('d'), M(79, 5), L('u'), L('e'), L(' '), L('}'), M(27, 6),
545	L(')'), M(108, 6), L('d'), L('e'), L('f'), L('e'), L('r'), L(' '),
546	L('f'), L('.'), L('c'), L('l'), L('o'), L('s'), L('e'), L('('),
547	M(183, 4), M(22, 4), L('_'), M(124, 7), L('f'), L('.'), L('w'), L('r'),
548	L('i'), L('t'), L('e'), L('A'), L('l'), L('l'), L('('), L('b'),
549	L('['), L('0'), L('.'), L('.'), L(']'), L(')'), L(';'), L(0xa),
550	L('}'), L(0xa),
551	},
552	},
553	TestCase{
554	.input = "huffman-zero.input",
555	.want = "huffman-zero.{s}.expect",
556	.want_no_input = "huffman-zero.{s}.expect-noinput",
557	.tokens = &[_]Token{ L(0x30), ml, M(1, 49) },
558	},
559	TestCase{
560	.input = "",
561	.want = "",
562	.want_no_input = "null-long-match.{s}.expect-noinput",
563	.tokens = &[_]Token{
564	L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
565	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
566	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
567	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
568	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
569	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
570	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
571	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
572	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
573	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
574	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
575	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
576	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
577	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
578	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
579	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
580	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
581	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
582	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
583	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
584	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
585	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
586	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
587	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
588	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
589	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
590	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
591	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
592	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
593	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
594	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
595	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
596	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
597	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
598	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
599	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
600	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
601	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
602	ml, ml, ml, M(1, 8),
603	},
604	},
605	};
606	};

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.input differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.input created+1

...	@@ -0,0 +1 @@
1	3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input created+4

...	@@ -0,0 +1,4 @@
1	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2	��vH
3	��%�� ɷ��}��>��ls��m�IGH��1Y�4�[�� 0[\|]o#�
4	�-#��ul��pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O��{��k�*�w(��b� ��kQC9/��lu>�5�C.��u�

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.input created+2

...	@@ -0,0 +1,2 @@
1	101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2	232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input created+14

...	@@ -0,0 +1,14 @@
1	//Copyright2009ThGoAuthor.Allrightrrvd.
2	//UofthiourccodigovrndbyBSD-tyl
3	//licnthtcnbfoundinthLICENSEfil.
4
5	pckgmin
6
7	import"o"
8
9	funcmin(){
10	vrb=mk([]byt,65535)
11	f,_:=o.Crt("huffmn-null-mx.in")
12	f.Writ(b)
13	}
14	ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text.input created+14

...	@@ -0,0 +1,14 @@
1	// zig v0.10.0
2	// create a file filled with 0x00
3	const std = @import("std");
4
5	pub fn main() !void {
6	var b = [1]u8{0} ** 65535;
7	const f = try std.fs.cwd().createFile(
8	"huffman-null-max.in",
9	.{ .read = true },
10	);
11	defer f.close();
12
13	_ = try f.writeAll(b[0..]);
14	}

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.input created+1

...	@@ -0,0 +1 @@
1	00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput differ

lib/std/compress/flate/testdata/fuzz/deflate-stream.expect created+22

...	@@ -0,0 +1,22 @@
1	[
2	{ id: "brieflz",
3	name: "BriefLZ",
4	libraryUrl: "https://github.com/jibsen/brieflz",
5	license: "MIT",
6	revision: "bcaa6a1ee7ccf005512b5c23aa92b40cf75f9ed1",
7	codecs: [ { name: "brieflz" } ], },
8	{ id: "brotli",
9	name: "Brotli",
10	libraryUrl: "https://github.com/google/brotli",
11	license: "Apache 2.0",
12	revision: "1dd66ef114fd244778d9dcb5da09c28b49a0df33",
13	codecs: [ { name: "brotli",
14	levels: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
15	streaming: true } ], },
16	{ id: "bsc",
17	name: "bsc",
18	libraryUrl: "http://libbsc.com/",
19	license: "Apache 2.0",
20	revision: "b2b07421381b19b2fada8b291f3cdead10578abc",
21	codecs: [ { name: "bsc" } ] }
22	]

lib/std/compress/flate/testdata/fuzz/deflate-stream.input created+3

...	@@ -0,0 +1,3 @@
1	��=o�0��+N��ڭR��K�}>W!AI@j+�{
2	�\|��<��F�x[�ش�\�9f;P�%�0৷#��iu��V��UWDQ��L�YF��tTG��U��_��\|S�Q��D��<M�4)�Xk%M��e�SdűK�0�]Cas[v��;�ɕMSV��J��@N�5Ea�tJN��Y�$Y�[eѤVs�27��ܺ8��}wӆ��.H1��A�`� c�3PW��%��uY@߮�jN��]$
3	,<��L�4<K��sa�2�i�s#�p1Z�V�4˵��؎��o
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet01.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet01.input differ

lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet02.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet02.input differ

lib/std/compress/flate/testdata/fuzz/end-of-stream.input created+1

...	@@ -0,0 +1 @@
1	��=o�0�
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/fuzz1.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz1.input differ

lib/std/compress/flate/testdata/fuzz/fuzz2.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz2.input differ

lib/std/compress/flate/testdata/fuzz/fuzz3.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz3.input differ

lib/std/compress/flate/testdata/fuzz/fuzz4.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz4.input differ

lib/std/compress/flate/testdata/fuzz/invalid-distance.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/invalid-distance.input differ

lib/std/compress/flate/testdata/fuzz/invalid-tree01.input created+1

...	@@ -0,0 +1 @@
1	000
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/invalid-tree02.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/invalid-tree02.input differ

lib/std/compress/flate/testdata/fuzz/invalid-tree03.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/invalid-tree03.input differ

lib/std/compress/flate/testdata/fuzz/lengths-overflow.input created+1

...	@@ -0,0 +1 @@
1	�$��9
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/out-of-codes.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/out-of-codes.input differ

lib/std/compress/flate/testdata/fuzz/puff01.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff01.input differ

lib/std/compress/flate/testdata/fuzz/puff02.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff02.input differ

lib/std/compress/flate/testdata/fuzz/puff03.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff03.input differ

lib/std/compress/flate/testdata/fuzz/puff04.input created+1

...	@@ -0,0 +1 @@
1	~��
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff05.input created+1

...	@@ -0,0 +1 @@
1
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff06.input created+1

...	@@ -0,0 +1 @@
1	�I�$I�$��
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff07.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff07.input differ

lib/std/compress/flate/testdata/fuzz/puff08.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff08.input differ

lib/std/compress/flate/testdata/fuzz/puff09.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff09.input differ

lib/std/compress/flate/testdata/fuzz/puff10.input created+1

...	@@ -0,0 +1 @@
1
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff11.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff11.input differ

lib/std/compress/flate/testdata/fuzz/puff12.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff12.input differ

lib/std/compress/flate/testdata/fuzz/puff13.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff13.input differ

lib/std/compress/flate/testdata/fuzz/puff14.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff14.input differ

lib/std/compress/flate/testdata/fuzz/puff15.input created+1

...	@@ -0,0 +1 @@
1	�I�$I�$��Ä
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff16.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff16.input differ

lib/std/compress/flate/testdata/fuzz/puff17.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff17.input differ

lib/std/compress/flate/testdata/fuzz/puff18.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff18.input differ

lib/std/compress/flate/testdata/fuzz/puff19.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff19.input differ

lib/std/compress/flate/testdata/fuzz/puff20.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff20.input differ

lib/std/compress/flate/testdata/fuzz/puff21.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff21.input differ

lib/std/compress/flate/testdata/fuzz/puff22.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff22.input differ

lib/std/compress/flate/testdata/fuzz/puff23.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff23.input differ

lib/std/compress/flate/testdata/fuzz/puff24.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff24.input differ

lib/std/compress/flate/testdata/fuzz/puff25.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff25.input differ

lib/std/compress/flate/testdata/fuzz/puff26.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff26.input differ

lib/std/compress/flate/testdata/fuzz/puff27.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff27.input differ

lib/std/compress/flate/testdata/fuzz/roundtrip1.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/roundtrip1.input differ

lib/std/compress/flate/testdata/fuzz/roundtrip2.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/roundtrip2.input differ

lib/std/compress/flate/testdata/rfc1951.txt created+955

...	@@ -0,0 +1,955 @@
1
2
3
4
5
6
7	Network Working Group P. Deutsch
8	Request for Comments: 1951 Aladdin Enterprises
9	Category: Informational May 1996
10
11
12	DEFLATE Compressed Data Format Specification version 1.3
13
14	Status of This Memo
15
16	This memo provides information for the Internet community. This memo
17	does not specify an Internet standard of any kind. Distribution of
18	this memo is unlimited.
19
20	IESG Note:
21
22	The IESG takes no position on the validity of any Intellectual
23	Property Rights statements contained in this document.
24
25	Notices
26
27	Copyright (c) 1996 L. Peter Deutsch
28
29	Permission is granted to copy and distribute this document for any
30	purpose and without charge, including translations into other
31	languages and incorporation into compilations, provided that the
32	copyright notice and this notice are preserved, and that any
33	substantive changes or deletions from the original are clearly
34	marked.
35
36	A pointer to the latest version of this and related documentation in
37	HTML format can be found at the URL
38	<ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40	Abstract
41
42	This specification defines a lossless compressed data format that
43	compresses data using a combination of the LZ77 algorithm and Huffman
44	coding, with efficiency comparable to the best currently available
45	general-purpose compression methods. The data can be produced or
46	consumed, even for an arbitrarily long sequentially presented input
47	data stream, using only an a priori bounded amount of intermediate
48	storage. The format can be implemented readily in a manner not
49	covered by patents.
50
51
52
53
54
55
56
57
58	Deutsch Informational [Page 1]
59
60	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
61
62
63	Table of Contents
64
65	1. Introduction ................................................... 2
66	1.1. Purpose ................................................... 2
67	1.2. Intended audience ......................................... 3
68	1.3. Scope ..................................................... 3
69	1.4. Compliance ................................................ 3
70	1.5. Definitions of terms and conventions used ................ 3
71	1.6. Changes from previous versions ............................ 4
72	2. Compressed representation overview ............................. 4
73	3. Detailed specification ......................................... 5
74	3.1. Overall conventions ....................................... 5
75	3.1.1. Packing into bytes .................................. 5
76	3.2. Compressed block format ................................... 6
77	3.2.1. Synopsis of prefix and Huffman coding ............... 6
78	3.2.2. Use of Huffman coding in the "deflate" format ....... 7
79	3.2.3. Details of block format ............................. 9
80	3.2.4. Non-compressed blocks (BTYPE=00) ................... 11
81	3.2.5. Compressed blocks (length and distance codes) ...... 11
82	3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12
83	3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13
84	3.3. Compliance ............................................... 14
85	4. Compression algorithm details ................................. 14
86	5. References .................................................... 16
87	6. Security Considerations ....................................... 16
88	7. Source code ................................................... 16
89	8. Acknowledgements .............................................. 16
90	9. Author's Address .............................................. 17
91
92	1. Introduction
93
94	1.1. Purpose
95
96	The purpose of this specification is to define a lossless
97	compressed data format that:
98	* Is independent of CPU type, operating system, file system,
99	and character set, and hence can be used for interchange;
100	* Can be produced or consumed, even for an arbitrarily long
101	sequentially presented input data stream, using only an a
102	priori bounded amount of intermediate storage, and hence
103	can be used in data communications or similar structures
104	such as Unix filters;
105	* Compresses data with efficiency comparable to the best
106	currently available general-purpose compression methods,
107	and in particular considerably better than the "compress"
108	program;
109	* Can be implemented readily in a manner not covered by
110	patents, and hence can be practiced freely;
111
112
113
114	Deutsch Informational [Page 2]
115
116	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
117
118
119	* Is compatible with the file format produced by the current
120	widely used gzip utility, in that conforming decompressors
121	will be able to read data produced by the existing gzip
122	compressor.
123
124	The data format defined by this specification does not attempt to:
125
126	* Allow random access to compressed data;
127	* Compress specialized data (e.g., raster graphics) as well
128	as the best currently available specialized algorithms.
129
130	A simple counting argument shows that no lossless compression
131	algorithm can compress every possible input data set. For the
132	format defined here, the worst case expansion is 5 bytes per 32K-
133	byte block, i.e., a size increase of 0.015% for large data sets.
134	English text usually compresses by a factor of 2.5 to 3;
135	executable files usually compress somewhat less; graphical data
136	such as raster images may compress much more.
137
138	1.2. Intended audience
139
140	This specification is intended for use by implementors of software
141	to compress data into "deflate" format and/or decompress data from
142	"deflate" format.
143
144	The text of the specification assumes a basic background in
145	programming at the level of bits and other primitive data
146	representations. Familiarity with the technique of Huffman coding
147	is helpful but not required.
148
149	1.3. Scope
150
151	The specification specifies a method for representing a sequence
152	of bytes as a (usually shorter) sequence of bits, and a method for
153	packing the latter bit sequence into bytes.
154
155	1.4. Compliance
156
157	Unless otherwise indicated below, a compliant decompressor must be
158	able to accept and decompress any data set that conforms to all
159	the specifications presented here; a compliant compressor must
160	produce data sets that conform to all the specifications presented
161	here.
162
163	1.5. Definitions of terms and conventions used
164
165	Byte: 8 bits stored or transmitted as a unit (same as an octet).
166	For this specification, a byte is exactly 8 bits, even on machines
167
168
169
170	Deutsch Informational [Page 3]
171
172	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
173
174
175	which store a character on a number of bits different from eight.
176	See below, for the numbering of bits within a byte.
177
178	String: a sequence of arbitrary bytes.
179
180	1.6. Changes from previous versions
181
182	There have been no technical changes to the deflate format since
183	version 1.1 of this specification. In version 1.2, some
184	terminology was changed. Version 1.3 is a conversion of the
185	specification to RFC style.
186
187	2. Compressed representation overview
188
189	A compressed data set consists of a series of blocks, corresponding
190	to successive blocks of input data. The block sizes are arbitrary,
191	except that non-compressible blocks are limited to 65,535 bytes.
192
193	Each block is compressed using a combination of the LZ77 algorithm
194	and Huffman coding. The Huffman trees for each block are independent
195	of those for previous or subsequent blocks; the LZ77 algorithm may
196	use a reference to a duplicated string occurring in a previous block,
197	up to 32K input bytes before.
198
199	Each block consists of two parts: a pair of Huffman code trees that
200	describe the representation of the compressed data part, and a
201	compressed data part. (The Huffman trees themselves are compressed
202	using Huffman encoding.) The compressed data consists of a series of
203	elements of two types: literal bytes (of strings that have not been
204	detected as duplicated within the previous 32K input bytes), and
205	pointers to duplicated strings, where a pointer is represented as a
206	pair <length, backward distance>. The representation used in the
207	"deflate" format limits distances to 32K bytes and lengths to 258
208	bytes, but does not limit the size of a block, except for
209	uncompressible blocks, which are limited as noted above.
210
211	Each type of value (literals, distances, and lengths) in the
212	compressed data is represented using a Huffman code, using one code
213	tree for literals and lengths and a separate code tree for distances.
214	The code trees for each block appear in a compact form just before
215	the compressed data for that block.
216
217
218
219
220
221
222
223
224
225
226	Deutsch Informational [Page 4]
227
228	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
229
230
231	3. Detailed specification
232
233	3.1. Overall conventions In the diagrams below, a box like this:
234
235	+---+
236	\| \| <-- the vertical bars might be missing
237	+---+
238
239	represents one byte; a box like this:
240
241	+==============+
242	\| \|
243	+==============+
244
245	represents a variable number of bytes.
246
247	Bytes stored within a computer do not have a "bit order", since
248	they are always treated as a unit. However, a byte considered as
249	an integer between 0 and 255 does have a most- and least-
250	significant bit, and since we write numbers with the most-
251	significant digit on the left, we also write bytes with the most-
252	significant bit on the left. In the diagrams below, we number the
253	bits of a byte so that bit 0 is the least-significant bit, i.e.,
254	the bits are numbered:
255
256	+--------+
257	\|76543210\|
258	+--------+
259
260	Within a computer, a number may occupy multiple bytes. All
261	multi-byte numbers in the format described here are stored with
262	the least-significant byte first (at the lower memory address).
263	For example, the decimal number 520 is stored as:
264
265	0 1
266	+--------+--------+
267	\|00001000\|00000010\|
268	+--------+--------+
269	^ ^
270	\| \|
271	\| + more significant byte = 2 x 256
272	+ less significant byte = 8
273
274	3.1.1. Packing into bytes
275
276	This document does not address the issue of the order in which
277	bits of a byte are transmitted on a bit-sequential medium,
278	since the final data format described here is byte- rather than
279
280
281
282	Deutsch Informational [Page 5]
283
284	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
285
286
287	bit-oriented. However, we describe the compressed block format
288	in below, as a sequence of data elements of various bit
289	lengths, not a sequence of bytes. We must therefore specify
290	how to pack these data elements into bytes to form the final
291	compressed byte sequence:
292
293	* Data elements are packed into bytes in order of
294	increasing bit number within the byte, i.e., starting
295	with the least-significant bit of the byte.
296	* Data elements other than Huffman codes are packed
297	starting with the least-significant bit of the data
298	element.
299	* Huffman codes are packed starting with the most-
300	significant bit of the code.
301
302	In other words, if one were to print out the compressed data as
303	a sequence of bytes, starting with the first byte at the
304	right margin and proceeding to the left, with the most-
305	significant bit of each byte on the left as usual, one would be
306	able to parse the result from right to left, with fixed-width
307	elements in the correct MSB-to-LSB order and Huffman codes in
308	bit-reversed order (i.e., with the first bit of the code in the
309	relative LSB position).
310
311	3.2. Compressed block format
312
313	3.2.1. Synopsis of prefix and Huffman coding
314
315	Prefix coding represents symbols from an a priori known
316	alphabet by bit sequences (codes), one code for each symbol, in
317	a manner such that different symbols may be represented by bit
318	sequences of different lengths, but a parser can always parse
319	an encoded string unambiguously symbol-by-symbol.
320
321	We define a prefix code in terms of a binary tree in which the
322	two edges descending from each non-leaf node are labeled 0 and
323	1 and in which the leaf nodes correspond one-for-one with (are
324	labeled with) the symbols of the alphabet; then the code for a
325	symbol is the sequence of 0's and 1's on the edges leading from
326	the root to the leaf labeled with that symbol. For example:
327
328
329
330
331
332
333
334
335
336
337
338	Deutsch Informational [Page 6]
339
340	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
341
342
343	/\ Symbol Code
344	0 1 ------ ----
345	/ \ A 00
346	/\ B B 1
347	0 1 C 011
348	/ \ D 010
349	A /\
350	0 1
351	/ \
352	D C
353
354	A parser can decode the next symbol from an encoded input
355	stream by walking down the tree from the root, at each step
356	choosing the edge corresponding to the next input bit.
357
358	Given an alphabet with known symbol frequencies, the Huffman
359	algorithm allows the construction of an optimal prefix code
360	(one which represents strings with those symbol frequencies
361	using the fewest bits of any possible prefix codes for that
362	alphabet). Such a code is called a Huffman code. (See
363	reference [1] in Chapter 5, references for additional
364	information on Huffman codes.)
365
366	Note that in the "deflate" format, the Huffman codes for the
367	various alphabets must not exceed certain maximum code lengths.
368	This constraint complicates the algorithm for computing code
369	lengths from symbol frequencies. Again, see Chapter 5,
370	references for details.
371
372	3.2.2. Use of Huffman coding in the "deflate" format
373
374	The Huffman codes used for each alphabet in the "deflate"
375	format have two additional rules:
376
377	* All codes of a given bit length have lexicographically
378	consecutive values, in the same order as the symbols
379	they represent;
380
381	* Shorter codes lexicographically precede longer codes.
382
383
384
385
386
387
388
389
390
391
392
393
394	Deutsch Informational [Page 7]
395
396	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
397
398
399	We could recode the example above to follow this rule as
400	follows, assuming that the order of the alphabet is ABCD:
401
402	Symbol Code
403	------ ----
404	A 10
405	B 0
406	C 110
407	D 111
408
409	I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are
410	lexicographically consecutive.
411
412	Given this rule, we can define the Huffman code for an alphabet
413	just by giving the bit lengths of the codes for each symbol of
414	the alphabet in order; this is sufficient to determine the
415	actual codes. In our example, the code is completely defined
416	by the sequence of bit lengths (2, 1, 3, 3). The following
417	algorithm generates the codes as integers, intended to be read
418	from most- to least-significant bit. The code lengths are
419	initially in tree[I].Len; the codes are produced in
420	tree[I].Code.
421
422	1) Count the number of codes for each code length. Let
423	bl_count[N] be the number of codes of length N, N >= 1.
424
425	2) Find the numerical value of the smallest code for each
426	code length:
427
428	code = 0;
429	bl_count[0] = 0;
430	for (bits = 1; bits <= MAX_BITS; bits++) {
431	code = (code + bl_count[bits-1]) << 1;
432	next_code[bits] = code;
433	}
434
435	3) Assign numerical values to all codes, using consecutive
436	values for all codes of the same length with the base
437	values determined at step 2. Codes that are never used
438	(which have a bit length of zero) must not be assigned a
439	value.
440
441	for (n = 0; n <= max_code; n++) {
442	len = tree[n].Len;
443	if (len != 0) {
444	tree[n].Code = next_code[len];
445	next_code[len]++;
446	}
447
448
449
450	Deutsch Informational [Page 8]
451
452	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
453
454
455	}
456
457	Example:
458
459	Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3,
460	3, 2, 4, 4). After step 1, we have:
461
462	N bl_count[N]
463	- -----------
464	2 1
465	3 5
466	4 2
467
468	Step 2 computes the following next_code values:
469
470	N next_code[N]
471	- ------------
472	1 0
473	2 0
474	3 2
475	4 14
476
477	Step 3 produces the following code values:
478
479	Symbol Length Code
480	------ ------ ----
481	A 3 010
482	B 3 011
483	C 3 100
484	D 3 101
485	E 3 110
486	F 2 00
487	G 4 1110
488	H 4 1111
489
490	3.2.3. Details of block format
491
492	Each block of compressed data begins with 3 header bits
493	containing the following data:
494
495	first bit BFINAL
496	next 2 bits BTYPE
497
498	Note that the header bits do not necessarily begin on a byte
499	boundary, since a block does not necessarily occupy an integral
500	number of bytes.
501
502
503
504
505
506	Deutsch Informational [Page 9]
507
508	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
509
510
511	BFINAL is set if and only if this is the last block of the data
512	set.
513
514	BTYPE specifies how the data are compressed, as follows:
515
516	00 - no compression
517	01 - compressed with fixed Huffman codes
518	10 - compressed with dynamic Huffman codes
519	11 - reserved (error)
520
521	The only difference between the two compressed cases is how the
522	Huffman codes for the literal/length and distance alphabets are
523	defined.
524
525	In all cases, the decoding algorithm for the actual data is as
526	follows:
527
528	do
529	read block header from input stream.
530	if stored with no compression
531	skip any remaining bits in current partially
532	processed byte
533	read LEN and NLEN (see next section)
534	copy LEN bytes of data to output
535	otherwise
536	if compressed with dynamic Huffman codes
537	read representation of code trees (see
538	subsection below)
539	loop (until end of block code recognized)
540	decode literal/length value from input stream
541	if value < 256
542	copy value (literal byte) to output stream
543	otherwise
544	if value = end of block (256)
545	break from loop
546	otherwise (value = 257..285)
547	decode distance from input stream
548
549	move backwards distance bytes in the output
550	stream, and copy length bytes from this
551	position to the output stream.
552	end loop
553	while not last block
554
555	Note that a duplicated string reference may refer to a string
556	in a previous block; i.e., the backward distance may cross one
557	or more block boundaries. However a distance cannot refer past
558	the beginning of the output stream. (An application using a
559
560
561
562	Deutsch Informational [Page 10]
563
564	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
565
566
567	preset dictionary might discard part of the output stream; a
568	distance can refer to that part of the output stream anyway)
569	Note also that the referenced string may overlap the current
570	position; for example, if the last 2 bytes decoded have values
571	X and Y, a string reference with <length = 5, distance = 2>
572	adds X,Y,X,Y,X to the output stream.
573
574	We now specify each compression method in turn.
575
576	3.2.4. Non-compressed blocks (BTYPE=00)
577
578	Any bits of input up to the next byte boundary are ignored.
579	The rest of the block consists of the following information:
580
581	0 1 2 3 4...
582	+---+---+---+---+================================+
583	\| LEN \| NLEN \|... LEN bytes of literal data...\|
584	+---+---+---+---+================================+
585
586	LEN is the number of data bytes in the block. NLEN is the
587	one's complement of LEN.
588
589	3.2.5. Compressed blocks (length and distance codes)
590
591	As noted above, encoded data blocks in the "deflate" format
592	consist of sequences of symbols drawn from three conceptually
593	distinct alphabets: either literal bytes, from the alphabet of
594	byte values (0..255), or <length, backward distance> pairs,
595	where the length is drawn from (3..258) and the distance is
596	drawn from (1..32,768). In fact, the literal and length
597	alphabets are merged into a single alphabet (0..285), where
598	values 0..255 represent literal bytes, the value 256 indicates
599	end-of-block, and values 257..285 represent length codes
600	(possibly in conjunction with extra bits following the symbol
601	code) as follows:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618	Deutsch Informational [Page 11]
619
620	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
621
622
623	Extra Extra Extra
624	Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
625	---- ---- ------ ---- ---- ------- ---- ---- -------
626	257 0 3 267 1 15,16 277 4 67-82
627	258 0 4 268 1 17,18 278 4 83-98
628	259 0 5 269 2 19-22 279 4 99-114
629	260 0 6 270 2 23-26 280 4 115-130
630	261 0 7 271 2 27-30 281 5 131-162
631	262 0 8 272 2 31-34 282 5 163-194
632	263 0 9 273 3 35-42 283 5 195-226
633	264 0 10 274 3 43-50 284 5 227-257
634	265 1 11,12 275 3 51-58 285 0 258
635	266 1 13,14 276 3 59-66
636
637	The extra bits should be interpreted as a machine integer
638	stored with the most-significant bit first, e.g., bits 1110
639	represent the value 14.
640
641	Extra Extra Extra
642	Code Bits Dist Code Bits Dist Code Bits Distance
643	---- ---- ---- ---- ---- ------ ---- ---- --------
644	0 0 1 10 4 33-48 20 9 1025-1536
645	1 0 2 11 4 49-64 21 9 1537-2048
646	2 0 3 12 5 65-96 22 10 2049-3072
647	3 0 4 13 5 97-128 23 10 3073-4096
648	4 1 5,6 14 6 129-192 24 11 4097-6144
649	5 1 7,8 15 6 193-256 25 11 6145-8192
650	6 2 9-12 16 7 257-384 26 12 8193-12288
651	7 2 13-16 17 7 385-512 27 12 12289-16384
652	8 3 17-24 18 8 513-768 28 13 16385-24576
653	9 3 25-32 19 8 769-1024 29 13 24577-32768
654
655	3.2.6. Compression with fixed Huffman codes (BTYPE=01)
656
657	The Huffman codes for the two alphabets are fixed, and are not
658	represented explicitly in the data. The Huffman code lengths
659	for the literal/length alphabet are:
660
661	Lit Value Bits Codes
662	--------- ---- -----
663	0 - 143 8 00110000 through
664	10111111
665	144 - 255 9 110010000 through
666	111111111
667	256 - 279 7 0000000 through
668	0010111
669	280 - 287 8 11000000 through
670	11000111
671
672
673
674	Deutsch Informational [Page 12]
675
676	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
677
678
679	The code lengths are sufficient to generate the actual codes,
680	as described above; we show the codes in the table for added
681	clarity. Literal/length values 286-287 will never actually
682	occur in the compressed data, but participate in the code
683	construction.
684
685	Distance codes 0-31 are represented by (fixed-length) 5-bit
686	codes, with possible additional bits as shown in the table
687	shown in Paragraph 3.2.5, above. Note that distance codes 30-
688	31 will never actually occur in the compressed data.
689
690	3.2.7. Compression with dynamic Huffman codes (BTYPE=10)
691
692	The Huffman codes for the two alphabets appear in the block
693	immediately after the header bits and before the actual
694	compressed data, first the literal/length code and then the
695	distance code. Each code is defined by a sequence of code
696	lengths, as discussed in Paragraph 3.2.2, above. For even
697	greater compactness, the code length sequences themselves are
698	compressed using a Huffman code. The alphabet for code lengths
699	is as follows:
700
701	0 - 15: Represent code lengths of 0 - 15
702	16: Copy the previous code length 3 - 6 times.
703	The next 2 bits indicate repeat length
704	(0 = 3, ... , 3 = 6)
705	Example: Codes 8, 16 (+2 bits 11),
706	16 (+2 bits 10) will expand to
707	12 code lengths of 8 (1 + 6 + 5)
708	17: Repeat a code length of 0 for 3 - 10 times.
709	(3 bits of length)
710	18: Repeat a code length of 0 for 11 - 138 times
711	(7 bits of length)
712
713	A code length of 0 indicates that the corresponding symbol in
714	the literal/length or distance alphabet will not occur in the
715	block, and should not participate in the Huffman code
716	construction algorithm given earlier. If only one distance
717	code is used, it is encoded using one bit, not zero bits; in
718	this case there is a single code length of one, with one unused
719	code. One distance code of zero bits means that there are no
720	distance codes used at all (the data is all literals).
721
722	We can now define the format of the block:
723
724	5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286)
725	5 Bits: HDIST, # of Distance codes - 1 (1 - 32)
726	4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19)
727
728
729
730	Deutsch Informational [Page 13]
731
732	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
733
734
735	(HCLEN + 4) x 3 bits: code lengths for the code length
736	alphabet given just above, in the order: 16, 17, 18,
737	0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
738
739	These code lengths are interpreted as 3-bit integers
740	(0-7); as above, a code length of 0 means the
741	corresponding symbol (literal/length or distance code
742	length) is not used.
743
744	HLIT + 257 code lengths for the literal/length alphabet,
745	encoded using the code length Huffman code
746
747	HDIST + 1 code lengths for the distance alphabet,
748	encoded using the code length Huffman code
749
750	The actual compressed data of the block,
751	encoded using the literal/length and distance Huffman
752	codes
753
754	The literal/length symbol 256 (end of data),
755	encoded using the literal/length Huffman code
756
757	The code length repeat codes can cross from HLIT + 257 to the
758	HDIST + 1 code lengths. In other words, all code lengths form
759	a single sequence of HLIT + HDIST + 258 values.
760
761	3.3. Compliance
762
763	A compressor may limit further the ranges of values specified in
764	the previous section and still be compliant; for example, it may
765	limit the range of backward pointers to some value smaller than
766	32K. Similarly, a compressor may limit the size of blocks so that
767	a compressible block fits in memory.
768
769	A compliant decompressor must accept the full range of possible
770	values defined in the previous section, and must accept blocks of
771	arbitrary size.
772
773	4. Compression algorithm details
774
775	While it is the intent of this document to define the "deflate"
776	compressed data format without reference to any particular
777	compression algorithm, the format is related to the compressed
778	formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below);
779	since many variations of LZ77 are patented, it is strongly
780	recommended that the implementor of a compressor follow the general
781	algorithm presented here, which is known not to be patented per se.
782	The material in this section is not part of the definition of the
783
784
785
786	Deutsch Informational [Page 14]
787
788	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
789
790
791	specification per se, and a compressor need not follow it in order to
792	be compliant.
793
794	The compressor terminates a block when it determines that starting a
795	new block with fresh trees would be useful, or when the block size
796	fills up the compressor's block buffer.
797
798	The compressor uses a chained hash table to find duplicated strings,
799	using a hash function that operates on 3-byte sequences. At any
800	given point during compression, let XYZ be the next 3 input bytes to
801	be examined (not necessarily all different, of course). First, the
802	compressor examines the hash chain for XYZ. If the chain is empty,
803	the compressor simply writes out X as a literal byte and advances one
804	byte in the input. If the hash chain is not empty, indicating that
805	the sequence XYZ (or, if we are unlucky, some other 3 bytes with the
806	same hash function value) has occurred recently, the compressor
807	compares all strings on the XYZ hash chain with the actual input data
808	sequence starting at the current point, and selects the longest
809	match.
810
811	The compressor searches the hash chains starting with the most recent
812	strings, to favor small distances and thus take advantage of the
813	Huffman encoding. The hash chains are singly linked. There are no
814	deletions from the hash chains; the algorithm simply discards matches
815	that are too old. To avoid a worst-case situation, very long hash
816	chains are arbitrarily truncated at a certain length, determined by a
817	run-time parameter.
818
819	To improve overall compression, the compressor optionally defers the
820	selection of matches ("lazy matching"): after a match of length N has
821	been found, the compressor searches for a longer match starting at
822	the next input byte. If it finds a longer match, it truncates the
823	previous match to a length of one (thus producing a single literal
824	byte) and then emits the longer match. Otherwise, it emits the
825	original match, and, as described above, advances N bytes before
826	continuing.
827
828	Run-time parameters also control this "lazy match" procedure. If
829	compression ratio is most important, the compressor attempts a
830	complete second search regardless of the length of the first match.
831	In the normal case, if the current match is "long enough", the
832	compressor reduces the search for a longer match, thus speeding up
833	the process. If speed is most important, the compressor inserts new
834	strings in the hash table only when no match was found, or when the
835	match is not "too long". This degrades the compression ratio but
836	saves time since there are both fewer insertions and fewer searches.
837
838
839
840
841
842	Deutsch Informational [Page 15]
843
844	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
845
846
847	5. References
848
849	[1] Huffman, D. A., "A Method for the Construction of Minimum
850	Redundancy Codes", Proceedings of the Institute of Radio
851	Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101.
852
853	[2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data
854	Compression", IEEE Transactions on Information Theory, Vol. 23,
855	No. 3, pp. 337-343.
856
857	[3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources,
858	available in ftp://ftp.uu.net/pub/archiving/zip/doc/
859
860	[4] Gailly, J.-L., and Adler, M., GZIP documentation and sources,
861	available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/
862
863	[5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix
864	encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169.
865
866	[6] Hirschberg and Lelewer, "Efficient decoding of prefix codes,"
867	Comm. ACM, 33,4, April 1990, pp. 449-459.
868
869	6. Security Considerations
870
871	Any data compression method involves the reduction of redundancy in
872	the data. Consequently, any corruption of the data is likely to have
873	severe effects and be difficult to correct. Uncompressed text, on
874	the other hand, will probably still be readable despite the presence
875	of some corrupted bytes.
876
877	It is recommended that systems using this data format provide some
878	means of validating the integrity of the compressed data. See
879	reference [3], for example.
880
881	7. Source code
882
883	Source code for a C language implementation of a "deflate" compliant
884	compressor and decompressor is available within the zlib package at
885	ftp://ftp.uu.net/pub/archiving/zip/zlib/.
886
887	8. Acknowledgements
888
889	Trademarks cited in this document are the property of their
890	respective owners.
891
892	Phil Katz designed the deflate format. Jean-Loup Gailly and Mark
893	Adler wrote the related software described in this specification.
894	Glenn Randers-Pehrson converted this document to RFC and HTML format.
895
896
897
898	Deutsch Informational [Page 16]
899
900	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
901
902
903	9. Author's Address
904
905	L. Peter Deutsch
906	Aladdin Enterprises
907	203 Santa Margarita Ave.
908	Menlo Park, CA 94025
909
910	Phone: (415) 322-0103 (AM only)
911	FAX: (415) 322-1734
912	EMail: <ghost@aladdin.com>
913
914	Questions about the technical content of this specification can be
915	sent by email to:
916
917	Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
918	Mark Adler <madler@alumni.caltech.edu>
919
920	Editorial comments on this specification can be sent by email to:
921
922	L. Peter Deutsch <ghost@aladdin.com> and
923	Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954	Deutsch Informational [Page 17]
955

lib/std/compress/flate/zlib.zig created+66

...	@@ -0,0 +1,66 @@
1	const deflate = @import("deflate.zig");
2	const inflate = @import("inflate.zig");
3
4	/// Decompress compressed data from reader and write plain data to the writer.
5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	try inflate.decompress(.zlib, reader, writer);
7	}
8
9	/// Decompressor type
10	pub fn Decompressor(comptime ReaderType: type) type {
11	return inflate.Inflate(.zlib, ReaderType);
12	}
13
14	/// Create Decompressor which will read compressed data from reader.
15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
16	return inflate.decompressor(.zlib, reader);
17	}
18
19	/// Compression level, trades between speed and compression size.
20	pub const Options = deflate.Options;
21
22	/// Compress plain data from reader and write compressed data to the writer.
23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
24	try deflate.compress(.zlib, reader, writer, options);
25	}
26
27	/// Compressor type
28	pub fn Compressor(comptime WriterType: type) type {
29	return deflate.Compressor(.zlib, WriterType);
30	}
31
32	/// Create Compressor which outputs compressed data to the writer.
33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
34	return try deflate.compressor(.zlib, writer, options);
35	}
36
37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
38	/// compression, less memory requirements but bigger compressed sizes.
39	pub const huffman = struct {
40	pub fn compress(reader: anytype, writer: anytype) !void {
41	try deflate.huffman.compress(.zlib, reader, writer);
42	}
43
44	pub fn Compressor(comptime WriterType: type) type {
45	return deflate.huffman.Compressor(.zlib, WriterType);
46	}
47
48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
49	return deflate.huffman.compressor(.zlib, writer);
50	}
51	};
52
53	// No compression store only. Compressed size is slightly bigger than plain.
54	pub const store = struct {
55	pub fn compress(reader: anytype, writer: anytype) !void {
56	try deflate.store.compress(.zlib, reader, writer);
57	}
58
59	pub fn Compressor(comptime WriterType: type) type {
60	return deflate.store.Compressor(.zlib, WriterType);
61	}
62
63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
64	return deflate.store.compressor(.zlib, writer);
65	}
66	};

lib/std/compress/gzip.zig+1-1

...	@@ -6,7 +6,7 @@ const io = std.io;	...	@@ -6,7 +6,7 @@ const io = std.io;
6	const fs = std.fs;	6	const fs = std.fs;
7	const testing = std.testing;	7	const testing = std.testing;
8	const mem = std.mem;	8	const mem = std.mem;
9	const deflate = std.compress.deflate;	9	const deflate = @import("deflate.zig");
10		10
11	const magic = &[2]u8{ 0x1f, 0x8b };	11	const magic = &[2]u8{ 0x1f, 0x8b };
12		12

lib/std/compress/zlib.zig+1-1

...	@@ -6,7 +6,7 @@ const io = std.io;	...	@@ -6,7 +6,7 @@ const io = std.io;
6	const fs = std.fs;	6	const fs = std.fs;
7	const testing = std.testing;	7	const testing = std.testing;
8	const mem = std.mem;	8	const mem = std.mem;
9	const deflate = std.compress.deflate;	9	const deflate = @import("deflate.zig");
10		10
11	// Zlib header format as specified in RFC1950	11	// Zlib header format as specified in RFC1950
12	const ZLibHeader = packed struct {	12	const ZLibHeader = packed struct {

lib/std/debug.zig+1-2

...	@@ -1212,8 +1212,7 @@ pub fn readElfDebugInfo(	...	@@ -1212,8 +1212,7 @@ pub fn readElfDebugInfo(
1212	const chdr = section_reader.readStruct(elf.Chdr) catch continue;	1212	const chdr = section_reader.readStruct(elf.Chdr) catch continue;
1213	if (chdr.ch_type != .ZLIB) continue;	1213	if (chdr.ch_type != .ZLIB) continue;
1214		1214
1215	var zlib_stream = std.compress.zlib.decompressStream(allocator, section_stream.reader()) catch continue;	1215	var zlib_stream = std.compress.zlib.decompressor(section_stream.reader());
1216	defer zlib_stream.deinit();
1217		1216
1218	const decompressed_section = try allocator.alloc(u8, chdr.ch_size);	1217	const decompressed_section = try allocator.alloc(u8, chdr.ch_size);
1219	errdefer allocator.free(decompressed_section);	1218	errdefer allocator.free(decompressed_section);

lib/std/http/Client.zig+8-8

lib/std/http/Server.zig+6-6

src/Package/Fetch.zig+6-1

 @@ -1099,7 +1099,12 @@ fn unpackResource(
     switch (file_type) {
         .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),
-        .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip),
+        .@"tar.gz" => {
+            const reader = resource.reader();
+            var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
+            var dcp = std.compress.gzip.decompressor(br.reader());
+            try unpackTarball(f, tmp_directory.handle, dcp.reader());
+        },
         .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),
         .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper),
         .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {

src/Package/Fetch/git.zig+3-6

src/link/Elf/Object.zig+1-3

 @@ -902,9 +902,7 @@ pub fn codeDecompressAlloc(self: Object, elf_file: *Elf, atom_index: Atom.Index)
         switch (chdr.ch_type) {
             .ZLIB => {
                 var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]);
-                var zlib_stream = std.compress.zlib.decompressStream(gpa, stream.reader()) catch
+                var zlib_stream = std.compress.zlib.decompressor(stream.reader());
-                    return error.InputOutput;
-                defer zlib_stream.deinit();
                 const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;
                 const decomp = try gpa.alloc(u8, size);
                 const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput;

src/objcopy.zig+1-2

...	@@ -1298,8 +1298,7 @@ const ElfFileHelper = struct {	...	@@ -1298,8 +1298,7 @@ const ElfFileHelper = struct {
1298	try compressed_stream.writer().writeAll(prefix);	1298	try compressed_stream.writer().writeAll(prefix);
1299		1299
1300	{	1300	{
1301	var compressor = try std.compress.zlib.compressStream(allocator, compressed_stream.writer(), .{});	1301	var compressor = try std.compress.zlib.compressor(compressed_stream.writer(), .{});
1302	defer compressor.deinit();
1303		1302
1304	var buf: [8000]u8 = undefined;	1303	var buf: [8000]u8 = undefined;
1305	while (true) {	1304	while (true) {

author	Igor Anić <igor.anic@gmail.com> 2024-02-13 22:14:40+01:00
committer	Igor Anić <igor.anic@gmail.com> 2024-02-14 18:28:20+01:00
log	d645114f7e3cbbfa19bb47ace2671fcac75e32ca
tree	d64330ddf46f3e1fefe7b957459fc509942b9914
parent	a23ab331a28d865e3ea636c9033db4de345f8653

...	@@ -404,8 +404,8 @@ pub const RequestTransfer = union(enum) {	...	@@ -404,8 +404,8 @@ pub const RequestTransfer = union(enum) {
404		404
405	/// The decompressor for response messages.	405	/// The decompressor for response messages.
406	pub const Compression = union(enum) {	406	pub const Compression = union(enum) {
407	pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader);	407	pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader);
408	pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);	408	pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader);
409	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});	409	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
410		410
411	deflate: DeflateDecompressor,	411	deflate: DeflateDecompressor,
...	@@ -601,8 +601,8 @@ pub const Request = struct {	...	@@ -601,8 +601,8 @@ pub const Request = struct {
601	pub fn deinit(req: *Request) void {	601	pub fn deinit(req: *Request) void {
602	switch (req.response.compression) {	602	switch (req.response.compression) {
603	.none => {},	603	.none => {},
604	.deflate => \|*deflate\| deflate.deinit(),	604	.deflate => {},
605	.gzip => \|*gzip\| gzip.deinit(),	605	.gzip => {},
606	.zstd => \|*zstd\| zstd.deinit(),	606	.zstd => \|*zstd\| zstd.deinit(),
607	}	607	}
608		608
...	@@ -632,8 +632,8 @@ pub const Request = struct {	...	@@ -632,8 +632,8 @@ pub const Request = struct {
632		632
633	switch (req.response.compression) {	633	switch (req.response.compression) {
634	.none => {},	634	.none => {},
635	.deflate => \|*deflate\| deflate.deinit(),	635	.deflate => {},
636	.gzip => \|*gzip\| gzip.deinit(),	636	.gzip => {},
637	.zstd => \|*zstd\| zstd.deinit(),	637	.zstd => \|*zstd\| zstd.deinit(),
638	}	638	}
639		639
...	@@ -941,10 +941,10 @@ pub const Request = struct {	...	@@ -941,10 +941,10 @@ pub const Request = struct {
941	.identity => req.response.compression = .none,	941	.identity => req.response.compression = .none,
942	.compress, .@"x-compress" => return error.CompressionNotSupported,	942	.compress, .@"x-compress" => return error.CompressionNotSupported,
943	.deflate => req.response.compression = .{	943	.deflate => req.response.compression = .{
944	.deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,	944	.deflate = std.compress.zlib.decompressor(req.transferReader()),
945	},	945	},
946	.gzip, .@"x-gzip" => req.response.compression = .{	946	.gzip, .@"x-gzip" => req.response.compression = .{
947	.gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,	947	.gzip = std.compress.gzip.decompressor(req.transferReader()),
948	},	948	},
949	.zstd => req.response.compression = .{	949	.zstd => req.response.compression = .{
950	.zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),	950	.zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),

...	@@ -195,8 +195,8 @@ pub const ResponseTransfer = union(enum) {	...	@@ -195,8 +195,8 @@ pub const ResponseTransfer = union(enum) {
195		195
196	/// The decompressor for request messages.	196	/// The decompressor for request messages.
197	pub const Compression = union(enum) {	197	pub const Compression = union(enum) {
198	pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Response.TransferReader);	198	pub const DeflateDecompressor = std.compress.zlib.Decompressor(Response.TransferReader);
199	pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);	199	pub const GzipDecompressor = std.compress.gzip.Decompressor(Response.TransferReader);
200	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});	200	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
201		201
202	deflate: DeflateDecompressor,	202	deflate: DeflateDecompressor,
...	@@ -420,8 +420,8 @@ pub const Response = struct {	...	@@ -420,8 +420,8 @@ pub const Response = struct {
420		420
421	switch (res.request.compression) {	421	switch (res.request.compression) {
422	.none => {},	422	.none => {},
423	.deflate => \|*deflate\| deflate.deinit(),	423	.deflate => {},
424	.gzip => \|*gzip\| gzip.deinit(),	424	.gzip => {},
425	.zstd => \|*zstd\| zstd.deinit(),	425	.zstd => \|*zstd\| zstd.deinit(),
426	}	426	}
427		427
...	@@ -605,10 +605,10 @@ pub const Response = struct {	...	@@ -605,10 +605,10 @@ pub const Response = struct {
605	.identity => res.request.compression = .none,	605	.identity => res.request.compression = .none,
606	.compress, .@"x-compress" => return error.CompressionNotSupported,	606	.compress, .@"x-compress" => return error.CompressionNotSupported,
607	.deflate => res.request.compression = .{	607	.deflate => res.request.compression = .{
608	.deflate = std.compress.zlib.decompressStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,	608	.deflate = std.compress.zlib.decompressor(res.transferReader()),
609	},	609	},
610	.gzip, .@"x-gzip" => res.request.compression = .{	610	.gzip, .@"x-gzip" => res.request.compression = .{
611	.gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,	611	.gzip = std.compress.gzip.decompressor(res.transferReader()),
612	},	612	},
613	.zstd => res.request.compression = .{	613	.zstd => res.request.compression = .{
614	.zstd = std.compress.zstd.decompressStream(res.allocator, res.transferReader()),	614	.zstd = std.compress.zstd.decompressStream(res.allocator, res.transferReader()),

...	@@ -1115,8 +1115,7 @@ fn indexPackFirstPass(	...	@@ -1115,8 +1115,7 @@ fn indexPackFirstPass(
1115	const entry_header = try EntryHeader.read(entry_crc32_reader.reader());	1115	const entry_header = try EntryHeader.read(entry_crc32_reader.reader());
1116	switch (entry_header) {	1116	switch (entry_header) {
1117	inline .commit, .tree, .blob, .tag => \|object, tag\| {	1117	inline .commit, .tree, .blob, .tag => \|object, tag\| {
1118	var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());	1118	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1119	defer entry_decompress_stream.deinit();
1120	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());	1119	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1121	var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));	1120	var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));
1122	const entry_writer = entry_hashed_writer.writer();	1121	const entry_writer = entry_hashed_writer.writer();
...	@@ -1135,8 +1134,7 @@ fn indexPackFirstPass(	...	@@ -1135,8 +1134,7 @@ fn indexPackFirstPass(
1135	});	1134	});
1136	},	1135	},
1137	inline .ofs_delta, .ref_delta => \|delta\| {	1136	inline .ofs_delta, .ref_delta => \|delta\| {
1138	var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());	1137	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1139	defer entry_decompress_stream.deinit();
1140	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());	1138	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1141	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();	1139	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1142	try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);	1140	try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
...	@@ -1257,8 +1255,7 @@ fn resolveDeltaChain(	...	@@ -1257,8 +1255,7 @@ fn resolveDeltaChain(
1257	fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {	1255	fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
1258	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;	1256	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1259	var buffered_reader = std.io.bufferedReader(reader);	1257	var buffered_reader = std.io.bufferedReader(reader);
1260	var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader());	1258	var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader());
1261	defer decompress_stream.deinit();
1262	const data = try allocator.alloc(u8, alloc_size);	1259	const data = try allocator.alloc(u8, alloc_size);
1263	errdefer allocator.free(data);	1260	errdefer allocator.free(data);
1264	try decompress_stream.reader().readNoEof(data);	1261	try decompress_stream.reader().readNoEof(data);

add deflate implemented from first principles