Commit 57d6f789de1d5fed5006aa3cefeb5b005bbdf6d6 | zig

.gitattributes+1

...	@@ -4,6 +4,7 @@	...	@@ -4,6 +4,7 @@
4	langref.html.in text eol=lf	4	langref.html.in text eol=lf
5	lib/std/compress/testdata/** binary	5	lib/std/compress/testdata/** binary
6	lib/std/compress/deflate/testdata/** binary	6	lib/std/compress/deflate/testdata/** binary
		7	lib/std/compress/flate/testdata/** binary
7		8
8	lib/include/** linguist-vendored	9	lib/include/** linguist-vendored
9	lib/libc/** linguist-vendored	10	lib/libc/** linguist-vendored

build.zig+1-1

...	@@ -150,7 +150,7 @@ pub fn build(b: *std.Build) !void {	...	@@ -150,7 +150,7 @@ pub fn build(b: *std.Build) !void {
150	"rfc1951.txt",	150	"rfc1951.txt",
151	"rfc1952.txt",	151	"rfc1952.txt",
152	"rfc8478.txt",	152	"rfc8478.txt",
153	// exclude files from lib/std/compress/deflate/testdata	153	// exclude files from lib/std/compress/flate/testdata
154	".expect",	154	".expect",
155	".expect-noinput",	155	".expect-noinput",
156	".golden",	156	".golden",

lib/std/compress.zig+5-5

 @@ -1,11 +1,11 @@
 const std = @import("std.zig");
-pub const deflate = @import("compress/deflate.zig");
+pub const flate = @import("compress/flate.zig");
 pub const gzip = @import("compress/gzip.zig");
+pub const zlib = @import("compress/zlib.zig");
 pub const lzma = @import("compress/lzma.zig");
 pub const lzma2 = @import("compress/lzma2.zig");
 pub const xz = @import("compress/xz.zig");
-pub const zlib = @import("compress/zlib.zig");
 pub const zstd = @import("compress/zstandard.zig");
 pub fn HashedReader(
 @@ -69,11 +69,11 @@ pub fn hashedWriter(
 }
 test {
-    _ = deflate;
-    _ = gzip;
     _ = lzma;
     _ = lzma2;
     _ = xz;
-    _ = zlib;
     _ = zstd;
+    _ = flate;
+    _ = gzip;
+    _ = zlib;
 }

lib/std/compress/deflate.zig deleted-44

...	@@ -1,44 +0,0 @@
1	//! The deflate package is a translation of the Go code of the compress/flate package from
2	//! https://go.googlesource.com/go/+/refs/tags/go1.17/src/compress/flate/
3
4	const deflate = @import("deflate/compressor.zig");
5	const inflate = @import("deflate/decompressor.zig");
6
7	pub const Compression = deflate.Compression;
8	pub const CompressorOptions = deflate.CompressorOptions;
9	pub const Compressor = deflate.Compressor;
10	pub const Decompressor = inflate.Decompressor;
11
12	pub const compressor = deflate.compressor;
13	pub const decompressor = inflate.decompressor;
14
15	/// Copies elements from a source `src` slice into a destination `dst` slice.
16	/// The copy never returns an error but might not be complete if the destination is too small.
17	/// Returns the number of elements copied, which will be the minimum of `src.len` and `dst.len`.
18	/// TODO: remove this smelly function
19	pub fn copy(dst: []u8, src: []const u8) usize {
20	if (dst.len <= src.len) {
21	@memcpy(dst, src[0..dst.len]);
22	return dst.len;
23	} else {
24	@memcpy(dst[0..src.len], src);
25	return src.len;
26	}
27	}
28
29	test {
30	_ = @import("deflate/token.zig");
31	_ = @import("deflate/bits_utils.zig");
32	_ = @import("deflate/dict_decoder.zig");
33
34	_ = @import("deflate/huffman_code.zig");
35	_ = @import("deflate/huffman_bit_writer.zig");
36
37	_ = @import("deflate/compressor.zig");
38	_ = @import("deflate/compressor_test.zig");
39
40	_ = @import("deflate/deflate_fast.zig");
41	_ = @import("deflate/deflate_fast_test.zig");
42
43	_ = @import("deflate/decompressor.zig");
44	}

lib/std/compress/deflate/bits_utils.zig deleted-33

...	@@ -1,33 +0,0 @@
1	const math = @import("std").math;
2
3	// Reverse bit-by-bit a N-bit code.
4	pub fn bitReverse(comptime T: type, value: T, N: usize) T {
5	const r = @bitReverse(value);
6	return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).Int.bits - N));
7	}
8
9	test "bitReverse" {
10	const std = @import("std");
11
12	const ReverseBitsTest = struct {
13	in: u16,
14	bit_count: u5,
15	out: u16,
16	};
17
18	const reverse_bits_tests = [_]ReverseBitsTest{
19	.{ .in = 1, .bit_count = 1, .out = 1 },
20	.{ .in = 1, .bit_count = 2, .out = 2 },
21	.{ .in = 1, .bit_count = 3, .out = 4 },
22	.{ .in = 1, .bit_count = 4, .out = 8 },
23	.{ .in = 1, .bit_count = 5, .out = 16 },
24	.{ .in = 17, .bit_count = 5, .out = 17 },
25	.{ .in = 257, .bit_count = 9, .out = 257 },
26	.{ .in = 29, .bit_count = 5, .out = 23 },
27	};
28
29	for (reverse_bits_tests) \|h\| {
30	const v = bitReverse(u16, h.in, h.bit_count);
31	try std.testing.expectEqual(h.out, v);
32	}
33	}

lib/std/compress/deflate/compressor.zig deleted-1110

...	@@ -1,1110 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const fmt = std.fmt;
4	const io = std.io;
5	const math = std.math;
6	const mem = std.mem;
7
8	const Allocator = std.mem.Allocator;
9
10	const deflate_const = @import("deflate_const.zig");
11	const fast = @import("deflate_fast.zig");
12	const hm_bw = @import("huffman_bit_writer.zig");
13	const token = @import("token.zig");
14
15	pub const Compression = enum(i5) {
16	/// huffman_only disables Lempel-Ziv match searching and only performs Huffman
17	/// entropy encoding. This mode is useful in compressing data that has
18	/// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
19	/// that lacks an entropy encoder. Compression gains are achieved when
20	/// certain bytes in the input stream occur more frequently than others.
21	///
22	/// Note that huffman_only produces a compressed output that is
23	/// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
24	/// continue to be able to decompress this output.
25	huffman_only = -2,
26	/// Same as level_6
27	default_compression = -1,
28	/// Does not attempt any compression; only adds the necessary DEFLATE framing.
29	no_compression = 0,
30	/// Prioritizes speed over output size, based on Snappy's LZ77-style encoder
31	best_speed = 1,
32	level_2 = 2,
33	level_3 = 3,
34	level_4 = 4,
35	level_5 = 5,
36	level_6 = 6,
37	level_7 = 7,
38	level_8 = 8,
39	/// Prioritizes smaller output size over speed
40	best_compression = 9,
41	};
42
43	const log_window_size = 15;
44	const window_size = 1 << log_window_size;
45	const window_mask = window_size - 1;
46
47	// The LZ77 step produces a sequence of literal tokens and <length, offset>
48	// pair tokens. The offset is also known as distance. The underlying wire
49	// format limits the range of lengths and offsets. For example, there are
50	// 256 legitimate lengths: those in the range [3, 258]. This package's
51	// compressor uses a higher minimum match length, enabling optimizations
52	// such as finding matches via 32-bit loads and compares.
53	const base_match_length = deflate_const.base_match_length; // The smallest match length per the RFC section 3.2.5
54	const min_match_length = 4; // The smallest match length that the compressor actually emits
55	const max_match_length = deflate_const.max_match_length;
56	const base_match_offset = deflate_const.base_match_offset; // The smallest match offset
57	const max_match_offset = deflate_const.max_match_offset; // The largest match offset
58
59	// The maximum number of tokens we put into a single flate block, just to
60	// stop things from getting too large.
61	const max_flate_block_tokens = 1 << 14;
62	const max_store_block_size = deflate_const.max_store_block_size;
63	const hash_bits = 17; // After 17 performance degrades
64	const hash_size = 1 << hash_bits;
65	const hash_mask = (1 << hash_bits) - 1;
66	const max_hash_offset = 1 << 24;
67
68	const skip_never = math.maxInt(u32);
69
70	const CompressionLevel = struct {
71	good: u16,
72	lazy: u16,
73	nice: u16,
74	chain: u16,
75	fast_skip_hashshing: u32,
76	};
77
78	fn levels(compression: Compression) CompressionLevel {
79	switch (compression) {
80	.no_compression,
81	.best_speed, // best_speed uses a custom algorithm; see deflate_fast.zig
82	.huffman_only,
83	=> return .{
84	.good = 0,
85	.lazy = 0,
86	.nice = 0,
87	.chain = 0,
88	.fast_skip_hashshing = 0,
89	},
90	// For levels 2-3 we don't bother trying with lazy matches.
91	.level_2 => return .{
92	.good = 4,
93	.lazy = 0,
94	.nice = 16,
95	.chain = 8,
96	.fast_skip_hashshing = 5,
97	},
98	.level_3 => return .{
99	.good = 4,
100	.lazy = 0,
101	.nice = 32,
102	.chain = 32,
103	.fast_skip_hashshing = 6,
104	},
105
106	// Levels 4-9 use increasingly more lazy matching and increasingly stringent conditions for
107	// "good enough".
108	.level_4 => return .{
109	.good = 4,
110	.lazy = 4,
111	.nice = 16,
112	.chain = 16,
113	.fast_skip_hashshing = skip_never,
114	},
115	.level_5 => return .{
116	.good = 8,
117	.lazy = 16,
118	.nice = 32,
119	.chain = 32,
120	.fast_skip_hashshing = skip_never,
121	},
122	.default_compression,
123	.level_6,
124	=> return .{
125	.good = 8,
126	.lazy = 16,
127	.nice = 128,
128	.chain = 128,
129	.fast_skip_hashshing = skip_never,
130	},
131	.level_7 => return .{
132	.good = 8,
133	.lazy = 32,
134	.nice = 128,
135	.chain = 256,
136	.fast_skip_hashshing = skip_never,
137	},
138	.level_8 => return .{
139	.good = 32,
140	.lazy = 128,
141	.nice = 258,
142	.chain = 1024,
143	.fast_skip_hashshing = skip_never,
144	},
145	.best_compression => return .{
146	.good = 32,
147	.lazy = 258,
148	.nice = 258,
149	.chain = 4096,
150	.fast_skip_hashshing = skip_never,
151	},
152	}
153	}
154
155	// matchLen returns the number of matching bytes in a and b
156	// up to length 'max'. Both slices must be at least 'max'
157	// bytes in size.
158	fn matchLen(a: []u8, b: []u8, max: u32) u32 {
159	const bounded_a = a[0..max];
160	const bounded_b = b[0..max];
161	for (bounded_a, 0..) \|av, i\| {
162	if (bounded_b[i] != av) {
163	return @as(u32, @intCast(i));
164	}
165	}
166	return max;
167	}
168
169	const hash_mul = 0x1e35a7bd;
170
171	// hash4 returns a hash representation of the first 4 bytes
172	// of the supplied slice.
173	// The caller must ensure that b.len >= 4.
174	fn hash4(b: []u8) u32 {
175	return ((@as(u32, b[3]) \|
176	@as(u32, b[2]) << 8 \|
177	@as(u32, b[1]) << 16 \|
178	@as(u32, b[0]) << 24) *% hash_mul) >> (32 - hash_bits);
179	}
180
181	// bulkHash4 will compute hashes using the same
182	// algorithm as hash4
183	fn bulkHash4(b: []u8, dst: []u32) u32 {
184	if (b.len < min_match_length) {
185	return 0;
186	}
187	var hb =
188	@as(u32, b[3]) \|
189	@as(u32, b[2]) << 8 \|
190	@as(u32, b[1]) << 16 \|
191	@as(u32, b[0]) << 24;
192
193	dst[0] = (hb *% hash_mul) >> (32 - hash_bits);
194	const end = b.len - min_match_length + 1;
195	var i: u32 = 1;
196	while (i < end) : (i += 1) {
197	hb = (hb << 8) \| @as(u32, b[i + 3]);
198	dst[i] = (hb *% hash_mul) >> (32 - hash_bits);
199	}
200
201	return hb;
202	}
203
204	pub const CompressorOptions = struct {
205	level: Compression = .default_compression,
206	dictionary: ?[]const u8 = null,
207	};
208
209	/// Returns a new Compressor compressing data at the given level.
210	/// Following zlib, levels range from 1 (best_speed) to 9 (best_compression);
211	/// higher levels typically run slower but compress more. Level 0
212	/// (no_compression) does not attempt any compression; it only adds the
213	/// necessary DEFLATE framing.
214	/// Level -1 (default_compression) uses the default compression level.
215	/// Level -2 (huffman_only) will use Huffman compression only, giving
216	/// a very fast compression for all types of input, but sacrificing considerable
217	/// compression efficiency.
218	///
219	/// `dictionary` is optional and initializes the new `Compressor` with a preset dictionary.
220	/// The returned Compressor behaves as if the dictionary had been written to it without producing
221	/// any compressed output. The compressed data written to hm_bw can only be decompressed by a
222	/// Decompressor initialized with the same dictionary.
223	///
224	/// The compressed data will be passed to the provided `writer`, see `writer()` and `write()`.
225	pub fn compressor(
226	allocator: Allocator,
227	writer: anytype,
228	options: CompressorOptions,
229	) !Compressor(@TypeOf(writer)) {
230	return Compressor(@TypeOf(writer)).init(allocator, writer, options);
231	}
232
233	pub fn Compressor(comptime WriterType: anytype) type {
234	return struct {
235	const Self = @This();
236
237	/// A Writer takes data written to it and writes the compressed
238	/// form of that data to an underlying writer.
239	pub const Writer = io.Writer(*Self, Error, write);
240
241	/// Returns a Writer that takes data written to it and writes the compressed
242	/// form of that data to an underlying writer.
243	pub fn writer(self: *Self) Writer {
244	return .{ .context = self };
245	}
246
247	pub const Error = WriterType.Error;
248
249	allocator: Allocator,
250
251	compression: Compression,
252	compression_level: CompressionLevel,
253
254	// Inner writer wrapped in a HuffmanBitWriter
255	hm_bw: hm_bw.HuffmanBitWriter(WriterType) = undefined,
256	bulk_hasher: *const fn ([]u8, []u32) u32,
257
258	sync: bool, // requesting flush
259	best_speed_enc: *fast.DeflateFast, // Encoder for best_speed
260
261	// Input hash chains
262	// hash_head[hashValue] contains the largest inputIndex with the specified hash value
263	// If hash_head[hashValue] is within the current window, then
264	// hash_prev[hash_head[hashValue] & window_mask] contains the previous index
265	// with the same hash value.
266	chain_head: u32,
267	hash_head: []u32, // [hash_size]u32,
268	hash_prev: []u32, // [window_size]u32,
269	hash_offset: u32,
270
271	// input window: unprocessed data is window[index..window_end]
272	index: u32,
273	window: []u8,
274	window_end: usize,
275	block_start: usize, // window index where current tokens start
276	byte_available: bool, // if true, still need to process window[index-1].
277
278	// queued output tokens
279	tokens: []token.Token,
280	tokens_count: u16,
281
282	// deflate state
283	length: u32,
284	offset: u32,
285	hash: u32,
286	max_insert_index: usize,
287	err: bool,
288
289	// hash_match must be able to contain hashes for the maximum match length.
290	hash_match: []u32, // [max_match_length - 1]u32,
291
292	// dictionary
293	dictionary: ?[]const u8,
294
295	fn fillDeflate(self: *Self, b: []const u8) u32 {
296	if (self.index >= 2 * window_size - (min_match_length + max_match_length)) {
297	// shift the window by window_size
298	mem.copyForwards(u8, self.window, self.window[window_size .. 2 * window_size]);
299	self.index -= window_size;
300	self.window_end -= window_size;
301	if (self.block_start >= window_size) {
302	self.block_start -= window_size;
303	} else {
304	self.block_start = math.maxInt(u32);
305	}
306	self.hash_offset += window_size;
307	if (self.hash_offset > max_hash_offset) {
308	const delta = self.hash_offset - 1;
309	self.hash_offset -= delta;
310	self.chain_head -\|= delta;
311
312	// Iterate over slices instead of arrays to avoid copying
313	// the entire table onto the stack (https://golang.org/issue/18625).
314	for (self.hash_prev, 0..) \|v, i\| {
315	if (v > delta) {
316	self.hash_prev[i] = @as(u32, @intCast(v - delta));
317	} else {
318	self.hash_prev[i] = 0;
319	}
320	}
321	for (self.hash_head, 0..) \|v, i\| {
322	if (v > delta) {
323	self.hash_head[i] = @as(u32, @intCast(v - delta));
324	} else {
325	self.hash_head[i] = 0;
326	}
327	}
328	}
329	}
330	const n = std.compress.deflate.copy(self.window[self.window_end..], b);
331	self.window_end += n;
332	return @as(u32, @intCast(n));
333	}
334
335	fn writeBlock(self: *Self, tokens: []token.Token, index: usize) !void {
336	if (index > 0) {
337	var window: ?[]u8 = null;
338	if (self.block_start <= index) {
339	window = self.window[self.block_start..index];
340	}
341	self.block_start = index;
342	try self.hm_bw.writeBlock(tokens, false, window);
343	return;
344	}
345	return;
346	}
347
348	// fillWindow will fill the current window with the supplied
349	// dictionary and calculate all hashes.
350	// This is much faster than doing a full encode.
351	// Should only be used after a reset.
352	fn fillWindow(self: *Self, in_b: []const u8) void {
353	var b = in_b;
354	// Do not fill window if we are in store-only mode (look at the fill() function to see
355	// Compressions which use fillStore() instead of fillDeflate()).
356	if (self.compression == .no_compression or
357	self.compression == .huffman_only or
358	self.compression == .best_speed)
359	{
360	return;
361	}
362
363	// fillWindow() must not be called with stale data
364	assert(self.index == 0 and self.window_end == 0);
365
366	// If we are given too much, cut it.
367	if (b.len > window_size) {
368	b = b[b.len - window_size ..];
369	}
370	// Add all to window.
371	@memcpy(self.window[0..b.len], b);
372	const n = b.len;
373
374	// Calculate 256 hashes at the time (more L1 cache hits)
375	const loops = (n + 256 - min_match_length) / 256;
376	var j: usize = 0;
377	while (j < loops) : (j += 1) {
378	const index = j * 256;
379	var end = index + 256 + min_match_length - 1;
380	if (end > n) {
381	end = n;
382	}
383	const to_check = self.window[index..end];
384	const dst_size = to_check.len - min_match_length + 1;
385
386	if (dst_size <= 0) {
387	continue;
388	}
389
390	const dst = self.hash_match[0..dst_size];
391	_ = self.bulk_hasher(to_check, dst);
392	var new_h: u32 = 0;
393	for (dst, 0..) \|val, i\| {
394	const di = i + index;
395	new_h = val;
396	const hh = &self.hash_head[new_h & hash_mask];
397	// Get previous value with the same hash.
398	// Our chain should point to the previous value.
399	self.hash_prev[di & window_mask] = hh.*;
400	// Set the head of the hash chain to us.
401	hh.* = @as(u32, @intCast(di + self.hash_offset));
402	}
403	self.hash = new_h;
404	}
405	// Update window information.
406	self.window_end = n;
407	self.index = @as(u32, @intCast(n));
408	}
409
410	const Match = struct {
411	length: u32,
412	offset: u32,
413	ok: bool,
414	};
415
416	// Try to find a match starting at pos whose length is greater than prev_length.
417	// We only look at self.compression_level.chain possibilities before giving up.
418	fn findMatch(
419	self: *Self,
420	pos: u32,
421	prev_head: u32,
422	prev_length: u32,
423	lookahead: u32,
424	) Match {
425	var length: u32 = 0;
426	var offset: u32 = 0;
427	var ok: bool = false;
428
429	var min_match_look: u32 = max_match_length;
430	if (lookahead < min_match_look) {
431	min_match_look = lookahead;
432	}
433
434	var win = self.window[0 .. pos + min_match_look];
435
436	// We quit when we get a match that's at least nice long
437	var nice = win.len - pos;
438	if (self.compression_level.nice < nice) {
439	nice = self.compression_level.nice;
440	}
441
442	// If we've got a match that's good enough, only look in 1/4 the chain.
443	var tries = self.compression_level.chain;
444	length = prev_length;
445	if (length >= self.compression_level.good) {
446	tries >>= 2;
447	}
448
449	var w_end = win[pos + length];
450	const w_pos = win[pos..];
451	const min_index = pos -\| window_size;
452
453	var i = prev_head;
454	while (tries > 0) : (tries -= 1) {
455	if (w_end == win[i + length]) {
456	const n = matchLen(win[i..], w_pos, min_match_look);
457
458	if (n > length and (n > min_match_length or pos - i <= 4096)) {
459	length = n;
460	offset = pos - i;
461	ok = true;
462	if (n >= nice) {
463	// The match is good enough that we don't try to find a better one.
464	break;
465	}
466	w_end = win[pos + n];
467	}
468	}
469	if (i == min_index) {
470	// hash_prev[i & window_mask] has already been overwritten, so stop now.
471	break;
472	}
473
474	if (@as(u32, @intCast(self.hash_prev[i & window_mask])) < self.hash_offset) {
475	break;
476	}
477
478	i = @as(u32, @intCast(self.hash_prev[i & window_mask])) - self.hash_offset;
479	if (i < min_index) {
480	break;
481	}
482	}
483
484	return Match{ .length = length, .offset = offset, .ok = ok };
485	}
486
487	fn writeStoredBlock(self: *Self, buf: []u8) !void {
488	try self.hm_bw.writeStoredHeader(buf.len, false);
489	try self.hm_bw.writeBytes(buf);
490	}
491
492	// encSpeed will compress and store the currently added data,
493	// if enough has been accumulated or we at the end of the stream.
494	fn encSpeed(self: *Self) !void {
495	// We only compress if we have max_store_block_size.
496	if (self.window_end < max_store_block_size) {
497	if (!self.sync) {
498	return;
499	}
500
501	// Handle small sizes.
502	if (self.window_end < 128) {
503	switch (self.window_end) {
504	0 => return,
505	1...16 => {
506	try self.writeStoredBlock(self.window[0..self.window_end]);
507	},
508	else => {
509	try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
510	self.err = self.hm_bw.err;
511	},
512	}
513	self.window_end = 0;
514	self.best_speed_enc.reset();
515	return;
516	}
517	}
518	// Encode the block.
519	self.tokens_count = 0;
520	self.best_speed_enc.encode(
521	self.tokens,
522	&self.tokens_count,
523	self.window[0..self.window_end],
524	);
525
526	// If we removed less than 1/16th, Huffman compress the block.
527	if (self.tokens_count > self.window_end - (self.window_end >> 4)) {
528	try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
529	} else {
530	try self.hm_bw.writeBlockDynamic(
531	self.tokens[0..self.tokens_count],
532	false,
533	self.window[0..self.window_end],
534	);
535	}
536	self.err = self.hm_bw.err;
537	self.window_end = 0;
538	}
539
540	fn initDeflate(self: *Self) !void {
541	self.window = try self.allocator.alloc(u8, 2 * window_size);
542	self.hash_offset = 1;
543	self.tokens = try self.allocator.alloc(token.Token, max_flate_block_tokens);
544	self.tokens_count = 0;
545	@memset(self.tokens, 0);
546	self.length = min_match_length - 1;
547	self.offset = 0;
548	self.byte_available = false;
549	self.index = 0;
550	self.hash = 0;
551	self.chain_head = 0;
552	self.bulk_hasher = bulkHash4;
553	}
554
555	fn deflate(self: *Self) !void {
556	if (self.window_end - self.index < min_match_length + max_match_length and !self.sync) {
557	return;
558	}
559
560	self.max_insert_index = self.window_end -\| (min_match_length - 1);
561	if (self.index < self.max_insert_index) {
562	self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
563	}
564
565	while (true) {
566	assert(self.index <= self.window_end);
567
568	const lookahead = self.window_end -\| self.index;
569	if (lookahead < min_match_length + max_match_length) {
570	if (!self.sync) {
571	break;
572	}
573	assert(self.index <= self.window_end);
574
575	if (lookahead == 0) {
576	// Flush current output block if any.
577	if (self.byte_available) {
578	// There is still one pending token that needs to be flushed
579	self.tokens[self.tokens_count] = token.literalToken(@as(u32, @intCast(self.window[self.index - 1])));
580	self.tokens_count += 1;
581	self.byte_available = false;
582	}
583	if (self.tokens.len > 0) {
584	try self.writeBlock(self.tokens[0..self.tokens_count], self.index);
585	self.tokens_count = 0;
586	}
587	break;
588	}
589	}
590	if (self.index < self.max_insert_index) {
591	// Update the hash
592	self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
593	const hh = &self.hash_head[self.hash & hash_mask];
594	self.chain_head = @as(u32, @intCast(hh.*));
595	self.hash_prev[self.index & window_mask] = @as(u32, @intCast(self.chain_head));
596	hh.* = @as(u32, @intCast(self.index + self.hash_offset));
597	}
598	const prev_length = self.length;
599	const prev_offset = self.offset;
600	self.length = min_match_length - 1;
601	self.offset = 0;
602	const min_index = self.index -\| window_size;
603
604	if (self.hash_offset <= self.chain_head and
605	self.chain_head - self.hash_offset >= min_index and
606	(self.compression_level.fast_skip_hashshing != skip_never and
607	lookahead > min_match_length - 1 or
608	self.compression_level.fast_skip_hashshing == skip_never and
609	lookahead > prev_length and
610	prev_length < self.compression_level.lazy))
611	{
612	{
613	const fmatch = self.findMatch(
614	self.index,
615	self.chain_head -\| self.hash_offset,
616	min_match_length - 1,
617	@as(u32, @intCast(lookahead)),
618	);
619	if (fmatch.ok) {
620	self.length = fmatch.length;
621	self.offset = fmatch.offset;
622	}
623	}
624	}
625	if (self.compression_level.fast_skip_hashshing != skip_never and
626	self.length >= min_match_length or
627	self.compression_level.fast_skip_hashshing == skip_never and
628	prev_length >= min_match_length and
629	self.length <= prev_length)
630	{
631	// There was a match at the previous step, and the current match is
632	// not better. Output the previous match.
633	if (self.compression_level.fast_skip_hashshing != skip_never) {
634	self.tokens[self.tokens_count] = token.matchToken(@as(u32, @intCast(self.length - base_match_length)), @as(u32, @intCast(self.offset - base_match_offset)));
635	self.tokens_count += 1;
636	} else {
637	self.tokens[self.tokens_count] = token.matchToken(
638	@as(u32, @intCast(prev_length - base_match_length)),
639	@as(u32, @intCast(prev_offset -\| base_match_offset)),
640	);
641	self.tokens_count += 1;
642	}
643	// Insert in the hash table all strings up to the end of the match.
644	// index and index-1 are already inserted. If there is not enough
645	// lookahead, the last two strings are not inserted into the hash
646	// table.
647	if (self.length <= self.compression_level.fast_skip_hashshing) {
648	var newIndex: u32 = 0;
649	if (self.compression_level.fast_skip_hashshing != skip_never) {
650	newIndex = self.index + self.length;
651	} else {
652	newIndex = self.index + prev_length - 1;
653	}
654	var index = self.index;
655	index += 1;
656	while (index < newIndex) : (index += 1) {
657	if (index < self.max_insert_index) {
658	self.hash = hash4(self.window[index .. index + min_match_length]);
659	// Get previous value with the same hash.
660	// Our chain should point to the previous value.
661	const hh = &self.hash_head[self.hash & hash_mask];
662	self.hash_prev[index & window_mask] = hh.*;
663	// Set the head of the hash chain to us.
664	hh.* = @as(u32, @intCast(index + self.hash_offset));
665	}
666	}
667	self.index = index;
668
669	if (self.compression_level.fast_skip_hashshing == skip_never) {
670	self.byte_available = false;
671	self.length = min_match_length - 1;
672	}
673	} else {
674	// For matches this long, we don't bother inserting each individual
675	// item into the table.
676	self.index += self.length;
677	if (self.index < self.max_insert_index) {
678	self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
679	}
680	}
681	if (self.tokens_count == max_flate_block_tokens) {
682	// The block includes the current character
683	try self.writeBlock(self.tokens[0..self.tokens_count], self.index);
684	self.tokens_count = 0;
685	}
686	} else {
687	if (self.compression_level.fast_skip_hashshing != skip_never or self.byte_available) {
688	var i = self.index -\| 1;
689	if (self.compression_level.fast_skip_hashshing != skip_never) {
690	i = self.index;
691	}
692	self.tokens[self.tokens_count] = token.literalToken(@as(u32, @intCast(self.window[i])));
693	self.tokens_count += 1;
694	if (self.tokens_count == max_flate_block_tokens) {
695	try self.writeBlock(self.tokens[0..self.tokens_count], i + 1);
696	self.tokens_count = 0;
697	}
698	}
699	self.index += 1;
700	if (self.compression_level.fast_skip_hashshing == skip_never) {
701	self.byte_available = true;
702	}
703	}
704	}
705	}
706
707	fn fillStore(self: *Self, b: []const u8) u32 {
708	const n = std.compress.deflate.copy(self.window[self.window_end..], b);
709	self.window_end += n;
710	return @as(u32, @intCast(n));
711	}
712
713	fn store(self: *Self) !void {
714	if (self.window_end > 0 and (self.window_end == max_store_block_size or self.sync)) {
715	try self.writeStoredBlock(self.window[0..self.window_end]);
716	self.window_end = 0;
717	}
718	}
719
720	// storeHuff compresses and stores the currently added data
721	// when the self.window is full or we are at the end of the stream.
722	fn storeHuff(self: *Self) !void {
723	if (self.window_end < self.window.len and !self.sync or self.window_end == 0) {
724	return;
725	}
726	try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
727	self.err = self.hm_bw.err;
728	self.window_end = 0;
729	}
730
731	pub fn bytesWritten(self: *Self) usize {
732	return self.hm_bw.bytes_written;
733	}
734
735	/// Writes the compressed form of `input` to the underlying writer.
736	pub fn write(self: *Self, input: []const u8) Error!usize {
737	var buf = input;
738
739	// writes data to hm_bw, which will eventually write the
740	// compressed form of data to its underlying writer.
741	while (buf.len > 0) {
742	try self.step();
743	const filled = self.fill(buf);
744	buf = buf[filled..];
745	}
746
747	return input.len;
748	}
749
750	/// Flushes any pending data to the underlying writer.
751	/// It is useful mainly in compressed network protocols, to ensure that
752	/// a remote reader has enough data to reconstruct a packet.
753	/// Flush does not return until the data has been written.
754	/// Calling `flush()` when there is no pending data still causes the Writer
755	/// to emit a sync marker of at least 4 bytes.
756	/// If the underlying writer returns an error, `flush()` returns that error.
757	///
758	/// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
759	pub fn flush(self: *Self) Error!void {
760	self.sync = true;
761	try self.step();
762	try self.hm_bw.writeStoredHeader(0, false);
763	try self.hm_bw.flush();
764	self.sync = false;
765	return;
766	}
767
768	fn step(self: *Self) !void {
769	switch (self.compression) {
770	.no_compression => return self.store(),
771	.huffman_only => return self.storeHuff(),
772	.best_speed => return self.encSpeed(),
773	.default_compression,
774	.level_2,
775	.level_3,
776	.level_4,
777	.level_5,
778	.level_6,
779	.level_7,
780	.level_8,
781	.best_compression,
782	=> return self.deflate(),
783	}
784	}
785
786	fn fill(self: *Self, b: []const u8) u32 {
787	switch (self.compression) {
788	.no_compression => return self.fillStore(b),
789	.huffman_only => return self.fillStore(b),
790	.best_speed => return self.fillStore(b),
791	.default_compression,
792	.level_2,
793	.level_3,
794	.level_4,
795	.level_5,
796	.level_6,
797	.level_7,
798	.level_8,
799	.best_compression,
800	=> return self.fillDeflate(b),
801	}
802	}
803
804	fn init(
805	allocator: Allocator,
806	in_writer: WriterType,
807	options: CompressorOptions,
808	) !Self {
809	var s = Self{
810	.allocator = undefined,
811	.compression = undefined,
812	.compression_level = undefined,
813	.hm_bw = undefined, // HuffmanBitWriter
814	.bulk_hasher = undefined,
815	.sync = false,
816	.best_speed_enc = undefined, // Best speed encoder
817	.chain_head = 0,
818	.hash_head = undefined,
819	.hash_prev = undefined, // previous hash
820	.hash_offset = 0,
821	.index = 0,
822	.window = undefined,
823	.window_end = 0,
824	.block_start = 0,
825	.byte_available = false,
826	.tokens = undefined,
827	.tokens_count = 0,
828	.length = 0,
829	.offset = 0,
830	.hash = 0,
831	.max_insert_index = 0,
832	.err = false, // Error
833	.hash_match = undefined,
834	.dictionary = options.dictionary,
835	};
836
837	s.hm_bw = try hm_bw.huffmanBitWriter(allocator, in_writer);
838	s.allocator = allocator;
839
840	s.hash_head = try allocator.alloc(u32, hash_size);
841	s.hash_prev = try allocator.alloc(u32, window_size);
842	s.hash_match = try allocator.alloc(u32, max_match_length - 1);
843	@memset(s.hash_head, 0);
844	@memset(s.hash_prev, 0);
845	@memset(s.hash_match, 0);
846
847	switch (options.level) {
848	.no_compression => {
849	s.compression = options.level;
850	s.compression_level = levels(options.level);
851	s.window = try allocator.alloc(u8, max_store_block_size);
852	s.tokens = try allocator.alloc(token.Token, 0);
853	},
854	.huffman_only => {
855	s.compression = options.level;
856	s.compression_level = levels(options.level);
857	s.window = try allocator.alloc(u8, max_store_block_size);
858	s.tokens = try allocator.alloc(token.Token, 0);
859	},
860	.best_speed => {
861	s.compression = options.level;
862	s.compression_level = levels(options.level);
863	s.window = try allocator.alloc(u8, max_store_block_size);
864	s.tokens = try allocator.alloc(token.Token, max_store_block_size);
865	s.best_speed_enc = try allocator.create(fast.DeflateFast);
866	s.best_speed_enc.* = fast.deflateFast();
867	try s.best_speed_enc.init(allocator);
868	},
869	.default_compression => {
870	s.compression = .level_6;
871	s.compression_level = levels(.level_6);
872	try s.initDeflate();
873	if (options.dictionary != null) {
874	s.fillWindow(options.dictionary.?);
875	}
876	},
877	.level_2,
878	.level_3,
879	.level_4,
880	.level_5,
881	.level_6,
882	.level_7,
883	.level_8,
884	.best_compression,
885	=> {
886	s.compression = options.level;
887	s.compression_level = levels(options.level);
888	try s.initDeflate();
889	if (options.dictionary != null) {
890	s.fillWindow(options.dictionary.?);
891	}
892	},
893	}
894	return s;
895	}
896
897	/// Release all allocated memory.
898	pub fn deinit(self: *Self) void {
899	self.hm_bw.deinit();
900	self.allocator.free(self.window);
901	self.allocator.free(self.tokens);
902	self.allocator.free(self.hash_head);
903	self.allocator.free(self.hash_prev);
904	self.allocator.free(self.hash_match);
905	if (self.compression == .best_speed) {
906	self.best_speed_enc.deinit();
907	self.allocator.destroy(self.best_speed_enc);
908	}
909	}
910
911	/// Reset discards the inner writer's state and replace the inner writer with new_writer.
912	/// new_writer must be of the same type as the previous writer.
913	pub fn reset(self: *Self, new_writer: WriterType) void {
914	self.hm_bw.reset(new_writer);
915	self.sync = false;
916	switch (self.compression) {
917	// Reset window
918	.no_compression => self.window_end = 0,
919	// Reset window, tokens, and encoder
920	.best_speed => {
921	self.window_end = 0;
922	self.tokens_count = 0;
923	self.best_speed_enc.reset();
924	},
925	// Reset everything and reinclude the dictionary if there is one
926	.huffman_only,
927	.default_compression,
928	.level_2,
929	.level_3,
930	.level_4,
931	.level_5,
932	.level_6,
933	.level_7,
934	.level_8,
935	.best_compression,
936	=> {
937	self.chain_head = 0;
938	@memset(self.hash_head, 0);
939	@memset(self.hash_prev, 0);
940	self.hash_offset = 1;
941	self.index = 0;
942	self.window_end = 0;
943	self.block_start = 0;
944	self.byte_available = false;
945	self.tokens_count = 0;
946	self.length = min_match_length - 1;
947	self.offset = 0;
948	self.hash = 0;
949	self.max_insert_index = 0;
950
951	if (self.dictionary != null) {
952	self.fillWindow(self.dictionary.?);
953	}
954	},
955	}
956	}
957
958	/// Writes any pending data to the underlying writer.
959	pub fn close(self: *Self) Error!void {
960	self.sync = true;
961	try self.step();
962	try self.hm_bw.writeStoredHeader(0, true);
963	try self.hm_bw.flush();
964	return;
965	}
966	};
967	}
968
969	// tests
970
971	const expect = std.testing.expect;
972	const testing = std.testing;
973
974	const ArrayList = std.ArrayList;
975
976	const DeflateTest = struct {
977	in: []const u8,
978	level: Compression,
979	out: []const u8,
980	};
981
982	var deflate_tests = [_]DeflateTest{
983	// Level 0
984	.{
985	.in = &[_]u8{},
986	.level = .no_compression,
987	.out = &[_]u8{ 1, 0, 0, 255, 255 },
988	},
989
990	// Level -1
991	.{
992	.in = &[_]u8{0x11},
993	.level = .default_compression,
994	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
995	},
996	.{
997	.in = &[_]u8{0x11},
998	.level = .level_6,
999	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1000	},
1001
1002	// Level 4
1003	.{
1004	.in = &[_]u8{0x11},
1005	.level = .level_4,
1006	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1007	},
1008
1009	// Level 0
1010	.{
1011	.in = &[_]u8{0x11},
1012	.level = .no_compression,
1013	.out = &[_]u8{ 0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255 },
1014	},
1015	.{
1016	.in = &[_]u8{ 0x11, 0x12 },
1017	.level = .no_compression,
1018	.out = &[_]u8{ 0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255 },
1019	},
1020	.{
1021	.in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1022	.level = .no_compression,
1023	.out = &[_]u8{ 0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255 },
1024	},
1025
1026	// Level 2
1027	.{
1028	.in = &[_]u8{},
1029	.level = .level_2,
1030	.out = &[_]u8{ 1, 0, 0, 255, 255 },
1031	},
1032	.{
1033	.in = &[_]u8{0x11},
1034	.level = .level_2,
1035	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1036	},
1037	.{
1038	.in = &[_]u8{ 0x11, 0x12 },
1039	.level = .level_2,
1040	.out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 },
1041	},
1042	.{
1043	.in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1044	.level = .level_2,
1045	.out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 },
1046	},
1047
1048	// Level 9
1049	.{
1050	.in = &[_]u8{},
1051	.level = .best_compression,
1052	.out = &[_]u8{ 1, 0, 0, 255, 255 },
1053	},
1054	.{
1055	.in = &[_]u8{0x11},
1056	.level = .best_compression,
1057	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1058	},
1059	.{
1060	.in = &[_]u8{ 0x11, 0x12 },
1061	.level = .best_compression,
1062	.out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 },
1063	},
1064	.{
1065	.in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1066	.level = .best_compression,
1067	.out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 },
1068	},
1069	};
1070
1071	test "deflate" {
1072	for (deflate_tests) \|dt\| {
1073	var output = ArrayList(u8).init(testing.allocator);
1074	defer output.deinit();
1075
1076	var comp = try compressor(testing.allocator, output.writer(), .{ .level = dt.level });
1077	_ = try comp.write(dt.in);
1078	try comp.close();
1079	comp.deinit();
1080
1081	try testing.expectEqualSlices(u8, dt.out, output.items);
1082	}
1083	}
1084
1085	test "bulkHash4" {
1086	for (deflate_tests) \|x\| {
1087	if (x.out.len < min_match_length) {
1088	continue;
1089	}
1090	// double the test data
1091	var out = try testing.allocator.alloc(u8, x.out.len * 2);
1092	defer testing.allocator.free(out);
1093	@memcpy(out[0..x.out.len], x.out);
1094	@memcpy(out[x.out.len..], x.out);
1095
1096	var j: usize = 4;
1097	while (j < out.len) : (j += 1) {
1098	var y = out[0..j];
1099
1100	const dst = try testing.allocator.alloc(u32, y.len - min_match_length + 1);
1101	defer testing.allocator.free(dst);
1102
1103	_ = bulkHash4(y, dst);
1104	for (dst, 0..) \|got, i\| {
1105	const want = hash4(y[i..]);
1106	try testing.expectEqual(want, got);
1107	}
1108	}
1109	}
1110	}

lib/std/compress/deflate/compressor_test.zig deleted-531

...	@@ -1,531 +0,0 @@
1	const std = @import("std");
2	const expect = std.testing.expect;
3	const fifo = std.fifo;
4	const io = std.io;
5	const math = std.math;
6	const mem = std.mem;
7	const testing = std.testing;
8
9	const ArrayList = std.ArrayList;
10
11	const deflate = @import("compressor.zig");
12	const inflate = @import("decompressor.zig");
13
14	const compressor = deflate.compressor;
15	const decompressor = inflate.decompressor;
16	const huffman_only = deflate.huffman_only;
17
18	fn testSync(level: deflate.Compression, input: []const u8) !void {
19	if (input.len == 0) {
20	return;
21	}
22
23	var divided_buf = fifo
24	.LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
25	.init(testing.allocator);
26	defer divided_buf.deinit();
27	var whole_buf = std.ArrayList(u8).init(testing.allocator);
28	defer whole_buf.deinit();
29
30	const multi_writer = io.multiWriter(.{
31	divided_buf.writer(),
32	whole_buf.writer(),
33	}).writer();
34
35	var comp = try compressor(
36	testing.allocator,
37	multi_writer,
38	.{ .level = level },
39	);
40	defer comp.deinit();
41
42	{
43	var decomp = try decompressor(
44	testing.allocator,
45	divided_buf.reader(),
46	null,
47	);
48	defer decomp.deinit();
49
50	// Write first half of the input and flush()
51	const half: usize = (input.len + 1) / 2;
52	var half_len: usize = half - 0;
53	{
54	_ = try comp.writer().writeAll(input[0..half]);
55
56	// Flush
57	try comp.flush();
58
59	// Read back
60	const decompressed = try testing.allocator.alloc(u8, half_len);
61	defer testing.allocator.free(decompressed);
62
63	const read = try decomp.reader().readAll(decompressed); // read at least half
64	try testing.expectEqual(half_len, read);
65	try testing.expectEqualSlices(u8, input[0..half], decompressed);
66	}
67
68	// Write last half of the input and close()
69	half_len = input.len - half;
70	{
71	_ = try comp.writer().writeAll(input[half..]);
72
73	// Close
74	try comp.close();
75
76	// Read back
77	const decompressed = try testing.allocator.alloc(u8, half_len);
78	defer testing.allocator.free(decompressed);
79
80	var read = try decomp.reader().readAll(decompressed);
81	try testing.expectEqual(half_len, read);
82	try testing.expectEqualSlices(u8, input[half..], decompressed);
83
84	// Extra read
85	var final: [10]u8 = undefined;
86	read = try decomp.reader().readAll(&final);
87	try testing.expectEqual(@as(usize, 0), read); // expect ended stream to return 0 bytes
88
89	try decomp.close();
90	}
91	}
92
93	_ = try comp.writer().writeAll(input);
94	try comp.close();
95
96	// stream should work for ordinary reader too (reading whole_buf in one go)
97	const whole_buf_reader = io.fixedBufferStream(whole_buf.items).reader();
98	var decomp = try decompressor(testing.allocator, whole_buf_reader, null);
99	defer decomp.deinit();
100
101	const decompressed = try testing.allocator.alloc(u8, input.len);
102	defer testing.allocator.free(decompressed);
103
104	_ = try decomp.reader().readAll(decompressed);
105	try decomp.close();
106
107	try testing.expectEqualSlices(u8, input, decompressed);
108	}
109
110	fn testToFromWithLevelAndLimit(level: deflate.Compression, input: []const u8, limit: u32) !void {
111	var compressed = std.ArrayList(u8).init(testing.allocator);
112	defer compressed.deinit();
113
114	var comp = try compressor(testing.allocator, compressed.writer(), .{ .level = level });
115	defer comp.deinit();
116
117	try comp.writer().writeAll(input);
118	try comp.close();
119
120	if (limit > 0) {
121	try expect(compressed.items.len <= limit);
122	}
123
124	var fib = io.fixedBufferStream(compressed.items);
125	var decomp = try decompressor(testing.allocator, fib.reader(), null);
126	defer decomp.deinit();
127
128	const decompressed = try testing.allocator.alloc(u8, input.len);
129	defer testing.allocator.free(decompressed);
130
131	const read: usize = try decomp.reader().readAll(decompressed);
132	try testing.expectEqual(input.len, read);
133	try testing.expectEqualSlices(u8, input, decompressed);
134
135	if (false) {
136	// TODO: this test has regressed
137	try testSync(level, input);
138	}
139	}
140
141	fn testToFromWithLimit(input: []const u8, limit: [11]u32) !void {
142	try testToFromWithLevelAndLimit(.no_compression, input, limit[0]);
143	try testToFromWithLevelAndLimit(.best_speed, input, limit[1]);
144	try testToFromWithLevelAndLimit(.level_2, input, limit[2]);
145	try testToFromWithLevelAndLimit(.level_3, input, limit[3]);
146	try testToFromWithLevelAndLimit(.level_4, input, limit[4]);
147	try testToFromWithLevelAndLimit(.level_5, input, limit[5]);
148	try testToFromWithLevelAndLimit(.level_6, input, limit[6]);
149	try testToFromWithLevelAndLimit(.level_7, input, limit[7]);
150	try testToFromWithLevelAndLimit(.level_8, input, limit[8]);
151	try testToFromWithLevelAndLimit(.best_compression, input, limit[9]);
152	try testToFromWithLevelAndLimit(.huffman_only, input, limit[10]);
153	}
154
155	test "deflate/inflate" {
156	const limits = [_]u32{0} ** 11;
157
158	var test0 = [_]u8{};
159	var test1 = [_]u8{0x11};
160	var test2 = [_]u8{ 0x11, 0x12 };
161	var test3 = [_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 };
162	var test4 = [_]u8{ 0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13 };
163
164	try testToFromWithLimit(&test0, limits);
165	try testToFromWithLimit(&test1, limits);
166	try testToFromWithLimit(&test2, limits);
167	try testToFromWithLimit(&test3, limits);
168	try testToFromWithLimit(&test4, limits);
169
170	var large_data_chunk = try testing.allocator.alloc(u8, 100_000);
171	defer testing.allocator.free(large_data_chunk);
172	// fill with random data
173	for (large_data_chunk, 0..) \|_, i\| {
174	large_data_chunk[i] = @as(u8, @truncate(i)) *% @as(u8, @truncate(i));
175	}
176	try testToFromWithLimit(large_data_chunk, limits);
177	}
178
179	test "very long sparse chunk" {
180	// A SparseReader returns a stream consisting of 0s ending with 65,536 (1<<16) 1s.
181	// This tests missing hash references in a very large input.
182	const SparseReader = struct {
183	l: usize, // length
184	cur: usize, // current position
185
186	const Self = @This();
187	const Error = error{};
188
189	pub const Reader = io.Reader(*Self, Error, read);
190
191	pub fn reader(self: *Self) Reader {
192	return .{ .context = self };
193	}
194
195	fn read(s: *Self, b: []u8) Error!usize {
196	var n: usize = 0; // amount read
197
198	if (s.cur >= s.l) {
199	return 0;
200	}
201	n = b.len;
202	var cur = s.cur + n;
203	if (cur > s.l) {
204	n -= cur - s.l;
205	cur = s.l;
206	}
207	for (b[0..n], 0..) \|_, i\| {
208	if (s.cur + i >= s.l -\| (1 << 16)) {
209	b[i] = 1;
210	} else {
211	b[i] = 0;
212	}
213	}
214	s.cur = cur;
215	return n;
216	}
217	};
218
219	var comp = try compressor(
220	testing.allocator,
221	io.null_writer,
222	.{ .level = .best_speed },
223	);
224	defer comp.deinit();
225	var writer = comp.writer();
226
227	var sparse = SparseReader{ .l = 0x23e8, .cur = 0 };
228	var reader = sparse.reader();
229
230	var read: usize = 1;
231	var written: usize = 0;
232	while (read > 0) {
233	var buf: [1 << 15]u8 = undefined; // 32,768 bytes buffer
234	read = try reader.read(&buf);
235	written += try writer.write(buf[0..read]);
236	}
237	try testing.expectEqual(@as(usize, 0x23e8), written);
238	}
239
240	test "compressor reset" {
241	for (std.enums.values(deflate.Compression)) \|c\| {
242	try testWriterReset(c, null);
243	try testWriterReset(c, "dict");
244	try testWriterReset(c, "hello");
245	}
246	}
247
248	fn testWriterReset(level: deflate.Compression, dict: ?[]const u8) !void {
249	const filler = struct {
250	fn writeData(c: anytype) !void {
251	const msg = "all your base are belong to us";
252	try c.writer().writeAll(msg);
253	try c.flush();
254
255	const hello = "hello world";
256	var i: usize = 0;
257	while (i < 1024) : (i += 1) {
258	try c.writer().writeAll(hello);
259	}
260
261	i = 0;
262	while (i < 65000) : (i += 1) {
263	try c.writer().writeAll("x");
264	}
265	}
266	};
267
268	var buf1 = ArrayList(u8).init(testing.allocator);
269	defer buf1.deinit();
270	var buf2 = ArrayList(u8).init(testing.allocator);
271	defer buf2.deinit();
272
273	var comp = try compressor(
274	testing.allocator,
275	buf1.writer(),
276	.{ .level = level, .dictionary = dict },
277	);
278	defer comp.deinit();
279
280	try filler.writeData(&comp);
281	try comp.close();
282
283	comp.reset(buf2.writer());
284	try filler.writeData(&comp);
285	try comp.close();
286
287	try testing.expectEqualSlices(u8, buf1.items, buf2.items);
288	}
289
290	test "decompressor dictionary" {
291	const dict = "hello world"; // dictionary
292	const text = "hello again world";
293
294	var compressed = fifo
295	.LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
296	.init(testing.allocator);
297	defer compressed.deinit();
298
299	var comp = try compressor(
300	testing.allocator,
301	compressed.writer(),
302	.{
303	.level = .level_5,
304	.dictionary = null, // no dictionary
305	},
306	);
307	defer comp.deinit();
308
309	// imitate a compressor with a dictionary
310	try comp.writer().writeAll(dict);
311	try comp.flush();
312	compressed.discard(compressed.readableLength()); // empty the output
313	try comp.writer().writeAll(text);
314	try comp.close();
315
316	const decompressed = try testing.allocator.alloc(u8, text.len);
317	defer testing.allocator.free(decompressed);
318
319	var decomp = try decompressor(
320	testing.allocator,
321	compressed.reader(),
322	dict,
323	);
324	defer decomp.deinit();
325
326	_ = try decomp.reader().readAll(decompressed);
327	try testing.expectEqualSlices(u8, "hello again world", decompressed);
328	}
329
330	test "compressor dictionary" {
331	const dict = "hello world";
332	const text = "hello again world";
333
334	var compressed_nd = fifo
335	.LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
336	.init(testing.allocator); // compressed with no dictionary
337	defer compressed_nd.deinit();
338
339	var compressed_d = ArrayList(u8).init(testing.allocator); // compressed with a dictionary
340	defer compressed_d.deinit();
341
342	// imitate a compressor with a dictionary
343	var comp_nd = try compressor(
344	testing.allocator,
345	compressed_nd.writer(),
346	.{
347	.level = .level_5,
348	.dictionary = null, // no dictionary
349	},
350	);
351	defer comp_nd.deinit();
352	try comp_nd.writer().writeAll(dict);
353	try comp_nd.flush();
354	compressed_nd.discard(compressed_nd.readableLength()); // empty the output
355	try comp_nd.writer().writeAll(text);
356	try comp_nd.close();
357
358	// use a compressor with a dictionary
359	var comp_d = try compressor(
360	testing.allocator,
361	compressed_d.writer(),
362	.{
363	.level = .level_5,
364	.dictionary = dict, // with a dictionary
365	},
366	);
367	defer comp_d.deinit();
368	try comp_d.writer().writeAll(text);
369	try comp_d.close();
370
371	try testing.expectEqualSlices(u8, compressed_d.items, compressed_nd.readableSlice(0));
372	}
373
374	// Update the hash for best_speed only if d.index < d.maxInsertIndex
375	// See https://golang.org/issue/2508
376	test "Go non-regression test for 2508" {
377	var comp = try compressor(
378	testing.allocator,
379	io.null_writer,
380	.{ .level = .best_speed },
381	);
382	defer comp.deinit();
383
384	var buf = [_]u8{0} ** 1024;
385
386	var i: usize = 0;
387	while (i < 131_072) : (i += 1) {
388	try comp.writer().writeAll(&buf);
389	try comp.close();
390	}
391	}
392
393	test "deflate/inflate string" {
394	const StringTest = struct {
395	filename: []const u8,
396	limit: [11]u32,
397	};
398
399	const deflate_inflate_string_tests = [_]StringTest{
400	.{
401	.filename = "compress-e.txt",
402	.limit = [11]u32{
403	100_018, // no_compression
404	50_650, // best_speed
405	50_960, // 2
406	51_150, // 3
407	50_930, // 4
408	50_790, // 5
409	50_790, // 6
410	50_790, // 7
411	50_790, // 8
412	50_790, // best_compression
413	43_683, // huffman_only
414	},
415	},
416	.{
417	.filename = "rfc1951.txt",
418	.limit = [11]u32{
419	36_954, // no_compression
420	12_952, // best_speed
421	12_228, // 2
422	12_016, // 3
423	11_466, // 4
424	11_191, // 5
425	11_129, // 6
426	11_120, // 7
427	11_112, // 8
428	11_109, // best_compression
429	20_273, // huffman_only
430	},
431	},
432	};
433
434	inline for (deflate_inflate_string_tests) \|t\| {
435	const golden = @embedFile("testdata/" ++ t.filename);
436	try testToFromWithLimit(golden, t.limit);
437	}
438	}
439
440	test "inflate reset" {
441	const strings = [_][]const u8{
442	"lorem ipsum izzle fo rizzle",
443	"the quick brown fox jumped over",
444	};
445
446	var compressed_strings = [_]ArrayList(u8){
447	ArrayList(u8).init(testing.allocator),
448	ArrayList(u8).init(testing.allocator),
449	};
450	defer compressed_strings[0].deinit();
451	defer compressed_strings[1].deinit();
452
453	for (strings, 0..) \|s, i\| {
454	var comp = try compressor(
455	testing.allocator,
456	compressed_strings[i].writer(),
457	.{ .level = .level_6 },
458	);
459	defer comp.deinit();
460
461	try comp.writer().writeAll(s);
462	try comp.close();
463	}
464
465	var fib = io.fixedBufferStream(compressed_strings[0].items);
466	var decomp = try decompressor(testing.allocator, fib.reader(), null);
467	defer decomp.deinit();
468
469	const decompressed_0: []u8 = try decomp.reader()
470	.readAllAlloc(testing.allocator, math.maxInt(usize));
471	defer testing.allocator.free(decompressed_0);
472
473	fib = io.fixedBufferStream(compressed_strings[1].items);
474	try decomp.reset(fib.reader(), null);
475
476	const decompressed_1: []u8 = try decomp.reader()
477	.readAllAlloc(testing.allocator, math.maxInt(usize));
478	defer testing.allocator.free(decompressed_1);
479
480	try decomp.close();
481
482	try testing.expectEqualSlices(u8, strings[0], decompressed_0);
483	try testing.expectEqualSlices(u8, strings[1], decompressed_1);
484	}
485
486	test "inflate reset dictionary" {
487	const dict = "the lorem fox";
488	const strings = [_][]const u8{
489	"lorem ipsum izzle fo rizzle",
490	"the quick brown fox jumped over",
491	};
492
493	var compressed_strings = [_]ArrayList(u8){
494	ArrayList(u8).init(testing.allocator),
495	ArrayList(u8).init(testing.allocator),
496	};
497	defer compressed_strings[0].deinit();
498	defer compressed_strings[1].deinit();
499
500	for (strings, 0..) \|s, i\| {
501	var comp = try compressor(
502	testing.allocator,
503	compressed_strings[i].writer(),
504	.{ .level = .level_6 },
505	);
506	defer comp.deinit();
507
508	try comp.writer().writeAll(s);
509	try comp.close();
510	}
511
512	var fib = io.fixedBufferStream(compressed_strings[0].items);
513	var decomp = try decompressor(testing.allocator, fib.reader(), dict);
514	defer decomp.deinit();
515
516	const decompressed_0: []u8 = try decomp.reader()
517	.readAllAlloc(testing.allocator, math.maxInt(usize));
518	defer testing.allocator.free(decompressed_0);
519
520	fib = io.fixedBufferStream(compressed_strings[1].items);
521	try decomp.reset(fib.reader(), dict);
522
523	const decompressed_1: []u8 = try decomp.reader()
524	.readAllAlloc(testing.allocator, math.maxInt(usize));
525	defer testing.allocator.free(decompressed_1);
526
527	try decomp.close();
528
529	try testing.expectEqualSlices(u8, strings[0], decompressed_0);
530	try testing.expectEqualSlices(u8, strings[1], decompressed_1);
531	}

lib/std/compress/deflate/decompressor.zig deleted-1119

...	@@ -1,1119 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5
6	const Allocator = std.mem.Allocator;
7	const ArrayList = std.ArrayList;
8
9	const bu = @import("bits_utils.zig");
10	const ddec = @import("dict_decoder.zig");
11	const deflate_const = @import("deflate_const.zig");
12
13	const max_match_offset = deflate_const.max_match_offset;
14	const end_block_marker = deflate_const.end_block_marker;
15
16	const max_code_len = 16; // max length of Huffman code
17	// The next three numbers come from the RFC section 3.2.7, with the
18	// additional proviso in section 3.2.5 which implies that distance codes
19	// 30 and 31 should never occur in compressed data.
20	const max_num_lit = 286;
21	const max_num_dist = 30;
22	const num_codes = 19; // number of codes in Huffman meta-code
23
24	var corrupt_input_error_offset: u64 = undefined;
25
26	const InflateError = error{
27	CorruptInput, // A CorruptInput error reports the presence of corrupt input at a given offset.
28	BadInternalState, // An BadInternalState reports an error in the flate code itself.
29	BadReaderState, // An error was encountered while accessing the inner reader
30	UnexpectedEndOfStream,
31	EndOfStreamWithNoError,
32	};
33
34	// The data structure for decoding Huffman tables is based on that of
35	// zlib. There is a lookup table of a fixed bit width (huffman_chunk_bits),
36	// For codes smaller than the table width, there are multiple entries
37	// (each combination of trailing bits has the same value). For codes
38	// larger than the table width, the table contains a link to an overflow
39	// table. The width of each entry in the link table is the maximum code
40	// size minus the chunk width.
41	//
42	// Note that you can do a lookup in the table even without all bits
43	// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
44	// have the property that shorter codes come before longer ones, the
45	// bit length estimate in the result is a lower bound on the actual
46	// number of bits.
47	//
48	// See the following:
49	// https://github.com/madler/zlib/raw/master/doc/algorithm.txt
50
51	// chunk & 15 is number of bits
52	// chunk >> 4 is value, including table link
53
54	const huffman_chunk_bits = 9;
55	const huffman_num_chunks = 1 << huffman_chunk_bits; // 512
56	const huffman_count_mask = 15; // 0b1111
57	const huffman_value_shift = 4;
58
59	const HuffmanDecoder = struct {
60	const Self = @This();
61
62	allocator: Allocator = undefined,
63
64	min: u32 = 0, // the minimum code length
65	chunks: [huffman_num_chunks]u16 = [1]u16{0} ** huffman_num_chunks, // chunks as described above
66	links: [][]u16 = undefined, // overflow links
67	link_mask: u32 = 0, // mask the width of the link table
68	initialized: bool = false,
69	sub_chunks: ArrayList(u32) = undefined,
70
71	// Initialize Huffman decoding tables from array of code lengths.
72	// Following this function, self is guaranteed to be initialized into a complete
73	// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
74	// degenerate case where the tree has only a single symbol with length 1. Empty
75	// trees are permitted.
76	fn init(self: *Self, allocator: Allocator, lengths: []u32) !bool {
77
78	// Sanity enables additional runtime tests during Huffman
79	// table construction. It's intended to be used during
80	// development and debugging
81	const sanity = false;
82
83	if (self.min != 0) {
84	self.* = HuffmanDecoder{};
85	}
86
87	self.allocator = allocator;
88
89	// Count number of codes of each length,
90	// compute min and max length.
91	var count: [max_code_len]u32 = [1]u32{0} ** max_code_len;
92	var min: u32 = 0;
93	var max: u32 = 0;
94	for (lengths) \|n\| {
95	if (n == 0) {
96	continue;
97	}
98	if (min == 0) {
99	min = n;
100	}
101	min = @min(n, min);
102	max = @max(n, max);
103	count[n] += 1;
104	}
105
106	// Empty tree. The decompressor.huffSym function will fail later if the tree
107	// is used. Technically, an empty tree is only valid for the HDIST tree and
108	// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
109	// is guaranteed to fail since it will attempt to use the tree to decode the
110	// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
111	// guaranteed to fail later since the compressed data section must be
112	// composed of at least one symbol (the end-of-block marker).
113	if (max == 0) {
114	return true;
115	}
116
117	var next_code: [max_code_len]u32 = [1]u32{0} ** max_code_len;
118	var code: u32 = 0;
119	{
120	var i = min;
121	while (i <= max) : (i += 1) {
122	code <<= 1;
123	next_code[i] = code;
124	code += count[i];
125	}
126	}
127
128	// Check that the coding is complete (i.e., that we've
129	// assigned all 2-to-the-max possible bit sequences).
130	// Exception: To be compatible with zlib, we also need to
131	// accept degenerate single-code codings. See also
132	// TestDegenerateHuffmanCoding.
133	if (code != @as(u32, 1) << @as(u5, @intCast(max)) and !(code == 1 and max == 1)) {
134	return false;
135	}
136
137	self.min = min;
138	if (max > huffman_chunk_bits) {
139	const num_links = @as(u32, 1) << @as(u5, @intCast(max - huffman_chunk_bits));
140	self.link_mask = @as(u32, @intCast(num_links - 1));
141
142	// create link tables
143	const link = next_code[huffman_chunk_bits + 1] >> 1;
144	self.links = try self.allocator.alloc([]u16, huffman_num_chunks - link);
145	self.sub_chunks = ArrayList(u32).init(self.allocator);
146	self.initialized = true;
147	var j = @as(u32, @intCast(link));
148	while (j < huffman_num_chunks) : (j += 1) {
149	var reverse = @as(u32, @intCast(bu.bitReverse(u16, @as(u16, @intCast(j)), 16)));
150	reverse >>= @as(u32, @intCast(16 - huffman_chunk_bits));
151	const off = j - @as(u32, @intCast(link));
152	if (sanity) {
153	// check we are not overwriting an existing chunk
154	assert(self.chunks[reverse] == 0);
155	}
156	self.chunks[reverse] = @as(u16, @intCast(off << huffman_value_shift \| (huffman_chunk_bits + 1)));
157	self.links[off] = try self.allocator.alloc(u16, num_links);
158	if (sanity) {
159	// initialize to a known invalid chunk code (0) to see if we overwrite
160	// this value later on
161	@memset(self.links[off], 0);
162	}
163	try self.sub_chunks.append(off);
164	}
165	}
166
167	for (lengths, 0..) \|n, li\| {
168	if (n == 0) {
169	continue;
170	}
171	const ncode = next_code[n];
172	next_code[n] += 1;
173	const chunk = @as(u16, @intCast((li << huffman_value_shift) \| n));
174	var reverse = @as(u16, @intCast(bu.bitReverse(u16, @as(u16, @intCast(ncode)), 16)));
175	reverse >>= @as(u4, @intCast(16 - n));
176	if (n <= huffman_chunk_bits) {
177	var off = reverse;
178	while (off < self.chunks.len) : (off += @as(u16, 1) << @as(u4, @intCast(n))) {
179	// We should never need to overwrite
180	// an existing chunk. Also, 0 is
181	// never a valid chunk, because the
182	// lower 4 "count" bits should be
183	// between 1 and 15.
184	if (sanity) {
185	assert(self.chunks[off] == 0);
186	}
187	self.chunks[off] = chunk;
188	}
189	} else {
190	const j = reverse & (huffman_num_chunks - 1);
191	if (sanity) {
192	// Expect an indirect chunk
193	assert(self.chunks[j] & huffman_count_mask == huffman_chunk_bits + 1);
194	// Longer codes should have been
195	// associated with a link table above.
196	}
197	const value = self.chunks[j] >> huffman_value_shift;
198	var link_tab = self.links[value];
199	reverse >>= huffman_chunk_bits;
200	var off = reverse;
201	while (off < link_tab.len) : (off += @as(u16, 1) << @as(u4, @intCast(n - huffman_chunk_bits))) {
202	if (sanity) {
203	// check we are not overwriting an existing chunk
204	assert(link_tab[off] == 0);
205	}
206	link_tab[off] = @as(u16, @intCast(chunk));
207	}
208	}
209	}
210
211	if (sanity) {
212	// Above we've sanity checked that we never overwrote
213	// an existing entry. Here we additionally check that
214	// we filled the tables completely.
215	for (self.chunks, 0..) \|chunk, i\| {
216	// As an exception, in the degenerate
217	// single-code case, we allow odd
218	// chunks to be missing.
219	if (code == 1 and i % 2 == 1) {
220	continue;
221	}
222
223	// Assert we are not missing a chunk.
224	// All chunks should have been written once
225	// thus losing their initial value of 0
226	assert(chunk != 0);
227	}
228
229	if (self.initialized) {
230	for (self.links) \|link_tab\| {
231	for (link_tab) \|chunk\| {
232	// Assert we are not missing a chunk.
233	assert(chunk != 0);
234	}
235	}
236	}
237	}
238
239	return true;
240	}
241
242	/// Release all allocated memory.
243	pub fn deinit(self: *Self) void {
244	if (self.initialized and self.links.len > 0) {
245	for (self.sub_chunks.items) \|off\| {
246	self.allocator.free(self.links[off]);
247	}
248	self.allocator.free(self.links);
249	self.sub_chunks.deinit();
250	self.initialized = false;
251	}
252	}
253	};
254
255	var fixed_huffman_decoder: ?HuffmanDecoder = null;
256
257	fn fixedHuffmanDecoderInit(allocator: Allocator) !HuffmanDecoder {
258	if (fixed_huffman_decoder != null) {
259	return fixed_huffman_decoder.?;
260	}
261
262	// These come from the RFC section 3.2.6.
263	var bits: [288]u32 = undefined;
264	var i: u32 = 0;
265	while (i < 144) : (i += 1) {
266	bits[i] = 8;
267	}
268	while (i < 256) : (i += 1) {
269	bits[i] = 9;
270	}
271	while (i < 280) : (i += 1) {
272	bits[i] = 7;
273	}
274	while (i < 288) : (i += 1) {
275	bits[i] = 8;
276	}
277
278	fixed_huffman_decoder = HuffmanDecoder{};
279	_ = try fixed_huffman_decoder.?.init(allocator, &bits);
280	return fixed_huffman_decoder.?;
281	}
282
283	const DecompressorState = enum {
284	init,
285	dict,
286	};
287
288	/// Returns a new Decompressor that can be used to read the uncompressed version of `reader`.
289	/// `dictionary` is optional and initializes the Decompressor with a preset dictionary.
290	/// The returned Decompressor behaves as if the uncompressed data stream started with the given
291	/// dictionary, which has already been read. Use the same `dictionary` as the compressor used to
292	/// compress the data.
293	/// This decompressor may use at most 300 KiB of heap memory from the provided allocator.
294	/// The uncompressed data will be written into the provided buffer, see `reader()` and `read()`.
295	pub fn decompressor(allocator: Allocator, reader: anytype, dictionary: ?[]const u8) !Decompressor(@TypeOf(reader)) {
296	return Decompressor(@TypeOf(reader)).init(allocator, reader, dictionary);
297	}
298
299	pub fn Decompressor(comptime ReaderType: type) type {
300	return struct {
301	const Self = @This();
302
303	pub const Error =
304	ReaderType.Error \|\|
305	error{EndOfStream} \|\|
306	InflateError \|\|
307	Allocator.Error;
308	pub const Reader = io.Reader(*Self, Error, read);
309
310	allocator: Allocator,
311
312	// Input source.
313	inner_reader: ReaderType,
314	roffset: u64,
315
316	// Input bits, in top of b.
317	b: u32,
318	nb: u32,
319
320	// Huffman decoders for literal/length, distance.
321	hd1: HuffmanDecoder,
322	hd2: HuffmanDecoder,
323
324	// Length arrays used to define Huffman codes.
325	bits: *[max_num_lit + max_num_dist]u32,
326	codebits: *[num_codes]u32,
327
328	// Output history, buffer.
329	dict: ddec.DictDecoder,
330
331	// Temporary buffer (avoids repeated allocation).
332	buf: [4]u8,
333
334	// Next step in the decompression,
335	// and decompression state.
336	step: const fn (Self) Error!void,
337	step_state: DecompressorState,
338	final: bool,
339	err: ?Error,
340	to_read: []u8,
341	// Huffman states for the lit/length values
342	hl: ?*HuffmanDecoder,
343	// Huffman states for the distance values.
344	hd: ?*HuffmanDecoder,
345	copy_len: u32,
346	copy_dist: u32,
347
348	/// Returns a Reader that reads compressed data from an underlying reader and outputs
349	/// uncompressed data.
350	pub fn reader(self: *Self) Reader {
351	return .{ .context = self };
352	}
353
354	fn init(allocator: Allocator, in_reader: ReaderType, dict: ?[]const u8) !Self {
355	fixed_huffman_decoder = try fixedHuffmanDecoderInit(allocator);
356
357	const bits = try allocator.create([max_num_lit + max_num_dist]u32);
358	const codebits = try allocator.create([num_codes]u32);
359
360	var dd = ddec.DictDecoder{};
361	try dd.init(allocator, max_match_offset, dict);
362
363	return Self{
364	.allocator = allocator,
365
366	// Input source.
367	.inner_reader = in_reader,
368	.roffset = 0,
369
370	// Input bits, in top of b.
371	.b = 0,
372	.nb = 0,
373
374	// Huffman decoders for literal/length, distance.
375	.hd1 = HuffmanDecoder{},
376	.hd2 = HuffmanDecoder{},
377
378	// Length arrays used to define Huffman codes.
379	.bits = bits,
380	.codebits = codebits,
381
382	// Output history, buffer.
383	.dict = dd,
384
385	// Temporary buffer (avoids repeated allocation).
386	.buf = [_]u8{0} ** 4,
387
388	// Next step in the decompression and decompression state.
389	.step = nextBlock,
390	.step_state = .init,
391	.final = false,
392	.err = null,
393	.to_read = &[0]u8{},
394	.hl = null,
395	.hd = null,
396	.copy_len = 0,
397	.copy_dist = 0,
398	};
399	}
400
401	/// Release all allocated memory.
402	pub fn deinit(self: *Self) void {
403	self.hd2.deinit();
404	self.hd1.deinit();
405	self.dict.deinit();
406	self.allocator.destroy(self.codebits);
407	self.allocator.destroy(self.bits);
408	}
409
410	fn nextBlock(self: *Self) Error!void {
411	while (self.nb < 1 + 2) {
412	self.moreBits() catch \|e\| {
413	self.err = e;
414	return e;
415	};
416	}
417	self.final = self.b & 1 == 1;
418	self.b >>= 1;
419	const typ = self.b & 3;
420	self.b >>= 2;
421	self.nb -= 1 + 2;
422	switch (typ) {
423	0 => try self.dataBlock(),
424	1 => {
425	// compressed, fixed Huffman tables
426	self.hl = &fixed_huffman_decoder.?;
427	self.hd = null;
428	try self.huffmanBlock();
429	},
430	2 => {
431	// compressed, dynamic Huffman tables
432	self.hd2.deinit();
433	self.hd1.deinit();
434	try self.readHuffman();
435	self.hl = &self.hd1;
436	self.hd = &self.hd2;
437	try self.huffmanBlock();
438	},
439	else => {
440	// 3 is reserved.
441	corrupt_input_error_offset = self.roffset;
442	self.err = InflateError.CorruptInput;
443	return InflateError.CorruptInput;
444	},
445	}
446	}
447
448	/// Reads compressed data from the underlying reader and outputs uncompressed data into
449	/// `output`.
450	pub fn read(self: *Self, output: []u8) Error!usize {
451	while (true) {
452	if (self.to_read.len > 0) {
453	const n = std.compress.deflate.copy(output, self.to_read);
454	self.to_read = self.to_read[n..];
455	if (self.to_read.len == 0 and
456	self.err != null)
457	{
458	if (self.err.? == InflateError.EndOfStreamWithNoError) {
459	return n;
460	}
461	return self.err.?;
462	}
463	return n;
464	}
465	if (self.err != null) {
466	if (self.err.? == InflateError.EndOfStreamWithNoError) {
467	return 0;
468	}
469	return self.err.?;
470	}
471	self.step(self) catch \|e\| {
472	self.err = e;
473	if (self.to_read.len == 0) {
474	self.to_read = self.dict.readFlush(); // Flush what's left in case of error
475	}
476	};
477	}
478	}
479
480	pub fn close(self: *Self) Error!void {
481	if (self.err) \|err\| {
482	if (err != error.EndOfStreamWithNoError) return err;
483	}
484	}
485
486	// RFC 1951 section 3.2.7.
487	// Compression with dynamic Huffman codes
488
489	const code_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
490
491	fn readHuffman(self: *Self) Error!void {
492	// HLIT[5], HDIST[5], HCLEN[4].
493	while (self.nb < 5 + 5 + 4) {
494	try self.moreBits();
495	}
496	const nlit = @as(u32, @intCast(self.b & 0x1F)) + 257;
497	if (nlit > max_num_lit) {
498	corrupt_input_error_offset = self.roffset;
499	self.err = InflateError.CorruptInput;
500	return InflateError.CorruptInput;
501	}
502	self.b >>= 5;
503	const ndist = @as(u32, @intCast(self.b & 0x1F)) + 1;
504	if (ndist > max_num_dist) {
505	corrupt_input_error_offset = self.roffset;
506	self.err = InflateError.CorruptInput;
507	return InflateError.CorruptInput;
508	}
509	self.b >>= 5;
510	const nclen = @as(u32, @intCast(self.b & 0xF)) + 4;
511	// num_codes is 19, so nclen is always valid.
512	self.b >>= 4;
513	self.nb -= 5 + 5 + 4;
514
515	// (HCLEN+4)*3 bits: code lengths in the magic code_order order.
516	var i: u32 = 0;
517	while (i < nclen) : (i += 1) {
518	while (self.nb < 3) {
519	try self.moreBits();
520	}
521	self.codebits[code_order[i]] = @as(u32, @intCast(self.b & 0x7));
522	self.b >>= 3;
523	self.nb -= 3;
524	}
525	i = nclen;
526	while (i < code_order.len) : (i += 1) {
527	self.codebits[code_order[i]] = 0;
528	}
529	if (!try self.hd1.init(self.allocator, self.codebits[0..])) {
530	corrupt_input_error_offset = self.roffset;
531	self.err = InflateError.CorruptInput;
532	return InflateError.CorruptInput;
533	}
534
535	// HLIT + 257 code lengths, HDIST + 1 code lengths,
536	// using the code length Huffman code.
537	i = 0;
538	const n = nlit + ndist;
539	while (i < n) {
540	const x = try self.huffSym(&self.hd1);
541	if (x < 16) {
542	// Actual length.
543	self.bits[i] = x;
544	i += 1;
545	continue;
546	}
547	// Repeat previous length or zero.
548	var rep: u32 = 0;
549	var nb: u32 = 0;
550	var b: u32 = 0;
551	switch (x) {
552	16 => {
553	rep = 3;
554	nb = 2;
555	if (i == 0) {
556	corrupt_input_error_offset = self.roffset;
557	self.err = InflateError.CorruptInput;
558	return InflateError.CorruptInput;
559	}
560	b = self.bits[i - 1];
561	},
562	17 => {
563	rep = 3;
564	nb = 3;
565	b = 0;
566	},
567	18 => {
568	rep = 11;
569	nb = 7;
570	b = 0;
571	},
572	else => return error.BadInternalState, // unexpected length code
573	}
574	while (self.nb < nb) {
575	try self.moreBits();
576	}
577	rep += @as(u32, @intCast(self.b & (@as(u32, 1) << @as(u5, @intCast(nb))) - 1));
578	self.b >>= @as(u5, @intCast(nb));
579	self.nb -= nb;
580	if (i + rep > n) {
581	corrupt_input_error_offset = self.roffset;
582	self.err = InflateError.CorruptInput;
583	return InflateError.CorruptInput;
584	}
585	var j: u32 = 0;
586	while (j < rep) : (j += 1) {
587	self.bits[i] = b;
588	i += 1;
589	}
590	}
591
592	if (!try self.hd1.init(self.allocator, self.bits[0..nlit]) or
593	!try self.hd2.init(self.allocator, self.bits[nlit..][0..ndist]))
594	{
595	corrupt_input_error_offset = self.roffset;
596	self.err = InflateError.CorruptInput;
597	return InflateError.CorruptInput;
598	}
599
600	// As an optimization, we can initialize the min bits to read at a time
601	// for the HLIT tree to the length of the EOB marker since we know that
602	// every block must terminate with one. This preserves the property that
603	// we never read any extra bytes after the end of the DEFLATE stream.
604	if (self.hd1.min < self.bits[end_block_marker]) {
605	self.hd1.min = self.bits[end_block_marker];
606	}
607
608	return;
609	}
610
611	// Decode a single Huffman block.
612	// hl and hd are the Huffman states for the lit/length values
613	// and the distance values, respectively. If hd == null, using the
614	// fixed distance encoding associated with fixed Huffman blocks.
615	fn huffmanBlock(self: *Self) Error!void {
616	while (true) {
617	switch (self.step_state) {
618	.init => {
619	// Read literal and/or (length, distance) according to RFC section 3.2.3.
620	const v = try self.huffSym(self.hl.?);
621	var n: u32 = 0; // number of bits extra
622	var length: u32 = 0;
623	switch (v) {
624	0...255 => {
625	self.dict.writeByte(@as(u8, @intCast(v)));
626	if (self.dict.availWrite() == 0) {
627	self.to_read = self.dict.readFlush();
628	self.step = huffmanBlock;
629	self.step_state = .init;
630	return;
631	}
632	self.step_state = .init;
633	continue;
634	},
635	256 => {
636	self.finishBlock();
637	return;
638	},
639	// otherwise, reference to older data
640	257...264 => {
641	length = v - (257 - 3);
642	n = 0;
643	},
644	265...268 => {
645	length = v * 2 - (265 * 2 - 11);
646	n = 1;
647	},
648	269...272 => {
649	length = v * 4 - (269 * 4 - 19);
650	n = 2;
651	},
652	273...276 => {
653	length = v * 8 - (273 * 8 - 35);
654	n = 3;
655	},
656	277...280 => {
657	length = v * 16 - (277 * 16 - 67);
658	n = 4;
659	},
660	281...284 => {
661	length = v * 32 - (281 * 32 - 131);
662	n = 5;
663	},
664	max_num_lit - 1 => { // 285
665	length = 258;
666	n = 0;
667	},
668	else => {
669	corrupt_input_error_offset = self.roffset;
670	self.err = InflateError.CorruptInput;
671	return InflateError.CorruptInput;
672	},
673	}
674	if (n > 0) {
675	while (self.nb < n) {
676	try self.moreBits();
677	}
678	length += @as(u32, @intCast(self.b)) & ((@as(u32, 1) << @as(u5, @intCast(n))) - 1);
679	self.b >>= @as(u5, @intCast(n));
680	self.nb -= n;
681	}
682
683	var dist: u32 = 0;
684	if (self.hd == null) {
685	while (self.nb < 5) {
686	try self.moreBits();
687	}
688	dist = @as(
689	u32,
690	@intCast(bu.bitReverse(u8, @as(u8, @intCast((self.b & 0x1F) << 3)), 8)),
691	);
692	self.b >>= 5;
693	self.nb -= 5;
694	} else {
695	dist = try self.huffSym(self.hd.?);
696	}
697
698	switch (dist) {
699	0...3 => dist += 1,
700	4...max_num_dist - 1 => { // 4...29
701	const nb = @as(u32, @intCast(dist - 2)) >> 1;
702	// have 1 bit in bottom of dist, need nb more.
703	var extra = (dist & 1) << @as(u5, @intCast(nb));
704	while (self.nb < nb) {
705	try self.moreBits();
706	}
707	extra \|= @as(u32, @intCast(self.b & (@as(u32, 1) << @as(u5, @intCast(nb))) - 1));
708	self.b >>= @as(u5, @intCast(nb));
709	self.nb -= nb;
710	dist = (@as(u32, 1) << @as(u5, @intCast(nb + 1))) + 1 + extra;
711	},
712	else => {
713	corrupt_input_error_offset = self.roffset;
714	self.err = InflateError.CorruptInput;
715	return InflateError.CorruptInput;
716	},
717	}
718
719	// No check on length; encoding can be prescient.
720	if (dist > self.dict.histSize()) {
721	corrupt_input_error_offset = self.roffset;
722	self.err = InflateError.CorruptInput;
723	return InflateError.CorruptInput;
724	}
725
726	self.copy_len = length;
727	self.copy_dist = dist;
728	self.step_state = .dict;
729	},
730
731	.dict => {
732	// Perform a backwards copy according to RFC section 3.2.3.
733	var cnt = self.dict.tryWriteCopy(self.copy_dist, self.copy_len);
734	if (cnt == 0) {
735	cnt = self.dict.writeCopy(self.copy_dist, self.copy_len);
736	}
737	self.copy_len -= cnt;
738
739	if (self.dict.availWrite() == 0 or self.copy_len > 0) {
740	self.to_read = self.dict.readFlush();
741	self.step = huffmanBlock; // We need to continue this work
742	self.step_state = .dict;
743	return;
744	}
745	self.step_state = .init;
746	},
747	}
748	}
749	}
750
751	// Copy a single uncompressed data block from input to output.
752	fn dataBlock(self: *Self) Error!void {
753	// Uncompressed.
754	// Discard current half-byte.
755	self.nb = 0;
756	self.b = 0;
757
758	// Length then ones-complement of length.
759	const nr: u32 = 4;
760	self.inner_reader.readNoEof(self.buf[0..nr]) catch {
761	self.err = InflateError.UnexpectedEndOfStream;
762	return InflateError.UnexpectedEndOfStream;
763	};
764	self.roffset += @as(u64, @intCast(nr));
765	const n = @as(u32, @intCast(self.buf[0])) \| @as(u32, @intCast(self.buf[1])) << 8;
766	const nn = @as(u32, @intCast(self.buf[2])) \| @as(u32, @intCast(self.buf[3])) << 8;
767	if (@as(u16, @intCast(nn)) != @as(u16, @truncate(~n))) {
768	corrupt_input_error_offset = self.roffset;
769	self.err = InflateError.CorruptInput;
770	return InflateError.CorruptInput;
771	}
772
773	if (n == 0) {
774	self.to_read = self.dict.readFlush();
775	self.finishBlock();
776	return;
777	}
778
779	self.copy_len = n;
780	try self.copyData();
781	}
782
783	// copyData copies self.copy_len bytes from the underlying reader into self.hist.
784	// It pauses for reads when self.hist is full.
785	fn copyData(self: *Self) Error!void {
786	var buf = self.dict.writeSlice();
787	if (buf.len > self.copy_len) {
788	buf = buf[0..self.copy_len];
789	}
790
791	const cnt = try self.inner_reader.read(buf);
792	if (cnt < buf.len) {
793	self.err = InflateError.UnexpectedEndOfStream;
794	}
795	self.roffset += @as(u64, @intCast(cnt));
796	self.copy_len -= @as(u32, @intCast(cnt));
797	self.dict.writeMark(@as(u32, @intCast(cnt)));
798	if (self.err != null) {
799	return InflateError.UnexpectedEndOfStream;
800	}
801
802	if (self.dict.availWrite() == 0 or self.copy_len > 0) {
803	self.to_read = self.dict.readFlush();
804	self.step = copyData;
805	return;
806	}
807	self.finishBlock();
808	}
809
810	fn finishBlock(self: *Self) void {
811	if (self.final) {
812	if (self.dict.availRead() > 0) {
813	self.to_read = self.dict.readFlush();
814	}
815	self.err = InflateError.EndOfStreamWithNoError;
816	}
817	self.step = nextBlock;
818	}
819
820	fn moreBits(self: *Self) InflateError!void {
821	const c = self.inner_reader.readByte() catch \|e\| {
822	if (e == error.EndOfStream) {
823	return InflateError.UnexpectedEndOfStream;
824	}
825	return InflateError.BadReaderState;
826	};
827	self.roffset += 1;
828	self.b \|= @as(u32, c) << @as(u5, @intCast(self.nb));
829	self.nb += 8;
830	return;
831	}
832
833	// Read the next Huffman-encoded symbol according to h.
834	fn huffSym(self: Self, h: HuffmanDecoder) InflateError!u32 {
835	// Since a HuffmanDecoder can be empty or be composed of a degenerate tree
836	// with single element, huffSym must error on these two edge cases. In both
837	// cases, the chunks slice will be 0 for the invalid sequence, leading it
838	// satisfy the n == 0 check below.
839	var n: u32 = h.min;
840	// Optimization. Go compiler isn't smart enough to keep self.b, self.nb in registers,
841	// but is smart enough to keep local variables in registers, so use nb and b,
842	// inline call to moreBits and reassign b, nb back to self on return.
843	var nb = self.nb;
844	var b = self.b;
845	while (true) {
846	while (nb < n) {
847	const c = self.inner_reader.readByte() catch \|e\| {
848	self.b = b;
849	self.nb = nb;
850	if (e == error.EndOfStream) {
851	return error.UnexpectedEndOfStream;
852	}
853	return InflateError.BadReaderState;
854	};
855	self.roffset += 1;
856	b \|= @as(u32, @intCast(c)) << @as(u5, @intCast(nb & 31));
857	nb += 8;
858	}
859	var chunk = h.chunks[b & (huffman_num_chunks - 1)];
860	n = @as(u32, @intCast(chunk & huffman_count_mask));
861	if (n > huffman_chunk_bits) {
862	chunk = h.links[chunk >> huffman_value_shift][(b >> huffman_chunk_bits) & h.link_mask];
863	n = @as(u32, @intCast(chunk & huffman_count_mask));
864	}
865	if (n <= nb) {
866	if (n == 0) {
867	self.b = b;
868	self.nb = nb;
869	corrupt_input_error_offset = self.roffset;
870	self.err = InflateError.CorruptInput;
871	return InflateError.CorruptInput;
872	}
873	self.b = b >> @as(u5, @intCast(n & 31));
874	self.nb = nb - n;
875	return @as(u32, @intCast(chunk >> huffman_value_shift));
876	}
877	}
878	}
879
880	/// Replaces the inner reader and dictionary with new_reader and new_dict.
881	/// new_reader must be of the same type as the reader being replaced.
882	pub fn reset(s: *Self, new_reader: ReaderType, new_dict: ?[]const u8) Error!void {
883	s.inner_reader = new_reader;
884	s.step = nextBlock;
885	s.err = null;
886	s.nb = 0;
887
888	s.dict.deinit();
889	try s.dict.init(s.allocator, max_match_offset, new_dict);
890
891	return;
892	}
893	};
894	}
895
896	// tests
897	const expectError = std.testing.expectError;
898	const io = std.io;
899	const testing = std.testing;
900
901	test "confirm decompressor resets" {
902	var compressed = std.ArrayList(u8).init(std.testing.allocator);
903	defer compressed.deinit();
904
905	inline for (.{
906	&[_]u8{ 0x5d, 0xc0, 0x21, 0x01, 0x00, 0x00, 0x00, 0x80, 0x20, 0xff, 0xaf, 0xa6, 0x4b, 0x03 },
907	&[_]u8{ 0x55, 0xc1, 0x41, 0x0d, 0x00, 0x00, 0x00, 0x02, 0xa1, 0x94, 0x96, 0x34, 0x25, 0xef, 0x1b, 0x5f, 0x01 },
908	}) \|data\| {
909	try compressed.writer().writeAll(data);
910	}
911
912	var stream = std.io.fixedBufferStream(compressed.items);
913	var decomp = try decompressor(std.testing.allocator, stream.reader(), null);
914	defer decomp.deinit();
915
916	while (true) {
917	if (try stream.getPos() == try stream.getEndPos()) break;
918
919	const buf = try decomp.reader().readAllAlloc(std.testing.allocator, 1024 * 100);
920	defer std.testing.allocator.free(buf);
921
922	try decomp.close();
923
924	try decomp.reset(stream.reader(), null);
925	}
926	}
927
928	test "truncated input" {
929	const TruncatedTest = struct {
930	input: []const u8,
931	output: []const u8,
932	};
933
934	const tests = [_]TruncatedTest{
935	.{ .input = "\x00", .output = "" },
936	.{ .input = "\x00\x0c", .output = "" },
937	.{ .input = "\x00\x0c\x00", .output = "" },
938	.{ .input = "\x00\x0c\x00\xf3\xff", .output = "" },
939	.{ .input = "\x00\x0c\x00\xf3\xffhello", .output = "hello" },
940	.{ .input = "\x00\x0c\x00\xf3\xffhello, world", .output = "hello, world" },
941	.{ .input = "\x02", .output = "" },
942	.{ .input = "\xf2H\xcd", .output = "He" },
943	.{ .input = "\xf2H͙0a\u{0084}\t", .output = "Hel\x90\x90\x90\x90\x90" },
944	.{ .input = "\xf2H͙0a\u{0084}\t\x00", .output = "Hel\x90\x90\x90\x90\x90" },
945	};
946
947	for (tests) \|t\| {
948	var fib = io.fixedBufferStream(t.input);
949	const r = fib.reader();
950	var z = try decompressor(testing.allocator, r, null);
951	defer z.deinit();
952	var zr = z.reader();
953
954	var output = [1]u8{0} ** 12;
955	try expectError(error.UnexpectedEndOfStream, zr.readAll(&output));
956	try testing.expectEqualSlices(u8, t.output, output[0..t.output.len]);
957	}
958	}
959
960	test "Go non-regression test for 9842" {
961	// See https://golang.org/issue/9842
962
963	const Test = struct {
964	err: ?anyerror,
965	input: []const u8,
966	};
967
968	const tests = [_]Test{
969	.{ .err = error.UnexpectedEndOfStream, .input = ("\x95\x90=o\xc20\x10\x86\xf30") },
970	.{ .err = error.CorruptInput, .input = ("\x950\x00\x0000000") },
971
972	// Huffman.construct errors
973
974	// lencode
975	.{ .err = error.CorruptInput, .input = ("\x950000") },
976	.{ .err = error.CorruptInput, .input = ("\x05000") },
977	// hlen
978	.{ .err = error.CorruptInput, .input = ("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00") },
979	// hdist
980	.{ .err = error.CorruptInput, .input = ("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf.") },
981
982	// like the "empty distance alphabet" test but for ndist instead of nlen
983	.{ .err = error.CorruptInput, .input = ("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce") },
984	.{ .err = null, .input = "\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0" },
985	};
986
987	for (tests) \|t\| {
988	var fib = std.io.fixedBufferStream(t.input);
989	const reader = fib.reader();
990	var decomp = try decompressor(testing.allocator, reader, null);
991	defer decomp.deinit();
992
993	var output: [10]u8 = undefined;
994	if (t.err != null) {
995	try expectError(t.err.?, decomp.reader().read(&output));
996	} else {
997	_ = try decomp.reader().read(&output);
998	}
999	}
1000	}
1001
1002	test "inflate A Tale of Two Cities (1859) intro" {
1003	const compressed = [_]u8{
1004	0x74, 0xeb, 0xcd, 0x0d, 0x80, 0x20, 0x0c, 0x47, 0x71, 0xdc, 0x9d, 0xa2, 0x03, 0xb8, 0x88,
1005	0x63, 0xf0, 0xf1, 0x47, 0x9a, 0x00, 0x35, 0xb4, 0x86, 0xf5, 0x0d, 0x27, 0x63, 0x82, 0xe7,
1006	0xdf, 0x7b, 0x87, 0xd1, 0x70, 0x4a, 0x96, 0x41, 0x1e, 0x6a, 0x24, 0x89, 0x8c, 0x2b, 0x74,
1007	0xdf, 0xf8, 0x95, 0x21, 0xfd, 0x8f, 0xdc, 0x89, 0x09, 0x83, 0x35, 0x4a, 0x5d, 0x49, 0x12,
1008	0x29, 0xac, 0xb9, 0x41, 0xbf, 0x23, 0x2e, 0x09, 0x79, 0x06, 0x1e, 0x85, 0x91, 0xd6, 0xc6,
1009	0x2d, 0x74, 0xc4, 0xfb, 0xa1, 0x7b, 0x0f, 0x52, 0x20, 0x84, 0x61, 0x28, 0x0c, 0x63, 0xdf,
1010	0x53, 0xf4, 0x00, 0x1e, 0xc3, 0xa5, 0x97, 0x88, 0xf4, 0xd9, 0x04, 0xa5, 0x2d, 0x49, 0x54,
1011	0xbc, 0xfd, 0x90, 0xa5, 0x0c, 0xae, 0xbf, 0x3f, 0x84, 0x77, 0x88, 0x3f, 0xaf, 0xc0, 0x40,
1012	0xd6, 0x5b, 0x14, 0x8b, 0x54, 0xf6, 0x0f, 0x9b, 0x49, 0xf7, 0xbf, 0xbf, 0x36, 0x54, 0x5a,
1013	0x0d, 0xe6, 0x3e, 0xf0, 0x9e, 0x29, 0xcd, 0xa1, 0x41, 0x05, 0x36, 0x48, 0x74, 0x4a, 0xe9,
1014	0x46, 0x66, 0x2a, 0x19, 0x17, 0xf4, 0x71, 0x8e, 0xcb, 0x15, 0x5b, 0x57, 0xe4, 0xf3, 0xc7,
1015	0xe7, 0x1e, 0x9d, 0x50, 0x08, 0xc3, 0x50, 0x18, 0xc6, 0x2a, 0x19, 0xa0, 0xdd, 0xc3, 0x35,
1016	0x82, 0x3d, 0x6a, 0xb0, 0x34, 0x92, 0x16, 0x8b, 0xdb, 0x1b, 0xeb, 0x7d, 0xbc, 0xf8, 0x16,
1017	0xf8, 0xc2, 0xe1, 0xaf, 0x81, 0x7e, 0x58, 0xf4, 0x9f, 0x74, 0xf8, 0xcd, 0x39, 0xd3, 0xaa,
1018	0x0f, 0x26, 0x31, 0xcc, 0x8d, 0x9a, 0xd2, 0x04, 0x3e, 0x51, 0xbe, 0x7e, 0xbc, 0xc5, 0x27,
1019	0x3d, 0xa5, 0xf3, 0x15, 0x63, 0x94, 0x42, 0x75, 0x53, 0x6b, 0x61, 0xc8, 0x01, 0x13, 0x4d,
1020	0x23, 0xba, 0x2a, 0x2d, 0x6c, 0x94, 0x65, 0xc7, 0x4b, 0x86, 0x9b, 0x25, 0x3e, 0xba, 0x01,
1021	0x10, 0x84, 0x81, 0x28, 0x80, 0x55, 0x1c, 0xc0, 0xa5, 0xaa, 0x36, 0xa6, 0x09, 0xa8, 0xa1,
1022	0x85, 0xf9, 0x7d, 0x45, 0xbf, 0x80, 0xe4, 0xd1, 0xbb, 0xde, 0xb9, 0x5e, 0xf1, 0x23, 0x89,
1023	0x4b, 0x00, 0xd5, 0x59, 0x84, 0x85, 0xe3, 0xd4, 0xdc, 0xb2, 0x66, 0xe9, 0xc1, 0x44, 0x0b,
1024	0x1e, 0x84, 0xec, 0xe6, 0xa1, 0xc7, 0x42, 0x6a, 0x09, 0x6d, 0x9a, 0x5e, 0x70, 0xa2, 0x36,
1025	0x94, 0x29, 0x2c, 0x85, 0x3f, 0x24, 0x39, 0xf3, 0xae, 0xc3, 0xca, 0xca, 0xaf, 0x2f, 0xce,
1026	0x8e, 0x58, 0x91, 0x00, 0x25, 0xb5, 0xb3, 0xe9, 0xd4, 0xda, 0xef, 0xfa, 0x48, 0x7b, 0x3b,
1027	0xe2, 0x63, 0x12, 0x00, 0x00, 0x20, 0x04, 0x80, 0x70, 0x36, 0x8c, 0xbd, 0x04, 0x71, 0xff,
1028	0xf6, 0x0f, 0x66, 0x38, 0xcf, 0xa1, 0x39, 0x11, 0x0f,
1029	};
1030
1031	const expected =
1032	\\It was the best of times,
1033	\\it was the worst of times,
1034	\\it was the age of wisdom,
1035	\\it was the age of foolishness,
1036	\\it was the epoch of belief,
1037	\\it was the epoch of incredulity,
1038	\\it was the season of Light,
1039	\\it was the season of Darkness,
1040	\\it was the spring of hope,
1041	\\it was the winter of despair,
1042	\\
1043	\\we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way---in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only.
1044	\\
1045	;
1046
1047	var fib = std.io.fixedBufferStream(&compressed);
1048	const reader = fib.reader();
1049	var decomp = try decompressor(testing.allocator, reader, null);
1050	defer decomp.deinit();
1051
1052	var got: [700]u8 = undefined;
1053	const got_len = try decomp.reader().read(&got);
1054	try testing.expectEqual(@as(usize, 616), got_len);
1055	try testing.expectEqualSlices(u8, expected, got[0..expected.len]);
1056	}
1057
1058	test "lengths overflow" {
1059	// malformed final dynamic block, tries to write 321 code lengths (MAXCODES is 316)
1060	// f dy hlit hdist hclen 16 17 18 0 (18) x138 (18) x138 (18) x39 (16) x6
1061	// 1 10 11101 11101 0000 010 010 010 010 (11) 1111111 (11) 1111111 (11) 0011100 (01) 11
1062	const stream = [_]u8{
1063	0b11101101, 0b00011101, 0b00100100, 0b11101001, 0b11111111, 0b11111111, 0b00111001,
1064	0b00001110,
1065	};
1066	try expectError(error.CorruptInput, decompress(stream[0..]));
1067	}
1068
1069	test "empty distance alphabet" {
1070	// dynamic block with empty distance alphabet is valid if only literals and end of data symbol are used
1071	// f dy hlit hdist hclen 16 17 18 0 8 7 9 6 10 5 11 4 12 3 13 2 14 1 15 (18) x128 (18) x128 (1) ( 0) (256)
1072	// 1 10 00000 00000 1111 000 000 010 010 000 000 000 000 000 000 000 000 000 000 000 000 000 001 000 (11) 1110101 (11) 1110101 (0) (10) (0)
1073	const stream = [_]u8{
1074	0b00000101, 0b11100000, 0b00000001, 0b00001001, 0b00000000, 0b00000000,
1075	0b00000000, 0b00000000, 0b00010000, 0b01011100, 0b10111111, 0b00101110,
1076	};
1077	try decompress(stream[0..]);
1078	}
1079
1080	test "distance past beginning of output stream" {
1081	// f fx ('A') ('B') ('C') <len=4, dist=4> (end)
1082	// 1 01 (01110001) (01110010) (01110011) (0000010) (00011) (0000000)
1083	const stream = [_]u8{ 0b01110011, 0b01110100, 0b01110010, 0b00000110, 0b01100001, 0b00000000 };
1084	try std.testing.expectError(error.CorruptInput, decompress(stream[0..]));
1085	}
1086
1087	test "fuzzing" {
1088	const compressed = [_]u8{
1089	0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05, 0xfc, 0x30, 0x0b, 0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05,
1090	} ++ [_]u8{0xe1} 15 ++ [_]u8{0x30} ++ [_]u8{0xe1} 1481;
1091	try expectError(error.UnexpectedEndOfStream, decompress(&compressed));
1092
1093	// see https://github.com/ziglang/zig/issues/9842
1094	try expectError(error.UnexpectedEndOfStream, decompress("\x95\x90=o\xc20\x10\x86\xf30"));
1095	try expectError(error.CorruptInput, decompress("\x950\x00\x0000000"));
1096
1097	// Huffman errors
1098	// lencode
1099	try expectError(error.CorruptInput, decompress("\x950000"));
1100	try expectError(error.CorruptInput, decompress("\x05000"));
1101	// hlen
1102	try expectError(error.CorruptInput, decompress("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00"));
1103	// hdist
1104	try expectError(error.CorruptInput, decompress("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf."));
1105
1106	// like the "empty distance alphabet" test but for ndist instead of nlen
1107	try expectError(error.CorruptInput, decompress("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce"));
1108	try decompress("\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0");
1109	}
1110
1111	fn decompress(input: []const u8) !void {
1112	const allocator = testing.allocator;
1113	var fib = std.io.fixedBufferStream(input);
1114	const reader = fib.reader();
1115	var decomp = try decompressor(allocator, reader, null);
1116	defer decomp.deinit();
1117	const output = try decomp.reader().readAllAlloc(allocator, math.maxInt(usize));
1118	defer std.testing.allocator.free(output);
1119	}

lib/std/compress/deflate/deflate_const.zig deleted-28

...	@@ -1,28 +0,0 @@
1	// Deflate
2
3	// Biggest block size for uncompressed block.
4	pub const max_store_block_size = 65535;
5	// The special code used to mark the end of a block.
6	pub const end_block_marker = 256;
7
8	// LZ77
9
10	// The smallest match length per the RFC section 3.2.5
11	pub const base_match_length = 3;
12	// The smallest match offset.
13	pub const base_match_offset = 1;
14	// The largest match length.
15	pub const max_match_length = 258;
16	// The largest match offset.
17	pub const max_match_offset = 1 << 15;
18
19	// Huffman Codes
20
21	// The largest offset code.
22	pub const offset_code_count = 30;
23	// Max number of frequencies used for a Huffman Code
24	// Possible lengths are codegenCodeCount (19), offset_code_count (30) and max_num_lit (286).
25	// The largest of these is max_num_lit.
26	pub const max_num_frequencies = max_num_lit;
27	// Maximum number of literals.
28	pub const max_num_lit = 286;

lib/std/compress/deflate/deflate_fast.zig deleted-728

...	@@ -1,728 +0,0 @@
1	// This encoding algorithm, which prioritizes speed over output size, is
2	// based on Snappy's LZ77-style encoder: github.com/golang/snappy
3
4	const std = @import("std");
5	const math = std.math;
6	const mem = std.mem;
7
8	const Allocator = std.mem.Allocator;
9
10	const deflate_const = @import("deflate_const.zig");
11	const deflate = @import("compressor.zig");
12	const token = @import("token.zig");
13
14	const base_match_length = deflate_const.base_match_length;
15	const base_match_offset = deflate_const.base_match_offset;
16	const max_match_length = deflate_const.max_match_length;
17	const max_match_offset = deflate_const.max_match_offset;
18	const max_store_block_size = deflate_const.max_store_block_size;
19
20	const table_bits = 14; // Bits used in the table.
21	const table_mask = table_size - 1; // Mask for table indices. Redundant, but can eliminate bounds checks.
22	const table_shift = 32 - table_bits; // Right-shift to get the table_bits most significant bits of a uint32.
23	const table_size = 1 << table_bits; // Size of the table.
24
25	// Reset the buffer offset when reaching this.
26	// Offsets are stored between blocks as i32 values.
27	// Since the offset we are checking against is at the beginning
28	// of the buffer, we need to subtract the current and input
29	// buffer to not risk overflowing the i32.
30	const buffer_reset = math.maxInt(i32) - max_store_block_size * 2;
31
32	fn load32(b: []u8, i: i32) u32 {
33	const s = b[@as(usize, @intCast(i)) .. @as(usize, @intCast(i)) + 4];
34	return @as(u32, @intCast(s[0])) \|
35	@as(u32, @intCast(s[1])) << 8 \|
36	@as(u32, @intCast(s[2])) << 16 \|
37	@as(u32, @intCast(s[3])) << 24;
38	}
39
40	fn load64(b: []u8, i: i32) u64 {
41	const s = b[@as(usize, @intCast(i))..@as(usize, @intCast(i + 8))];
42	return @as(u64, @intCast(s[0])) \|
43	@as(u64, @intCast(s[1])) << 8 \|
44	@as(u64, @intCast(s[2])) << 16 \|
45	@as(u64, @intCast(s[3])) << 24 \|
46	@as(u64, @intCast(s[4])) << 32 \|
47	@as(u64, @intCast(s[5])) << 40 \|
48	@as(u64, @intCast(s[6])) << 48 \|
49	@as(u64, @intCast(s[7])) << 56;
50	}
51
52	fn hash(u: u32) u32 {
53	return (u *% 0x1e35a7bd) >> table_shift;
54	}
55
56	// These constants are defined by the Snappy implementation so that its
57	// assembly implementation can fast-path some 16-bytes-at-a-time copies.
58	// They aren't necessary in the pure Go implementation, and may not be
59	// necessary in Zig, but using the same thresholds doesn't really hurt.
60	const input_margin = 16 - 1;
61	const min_non_literal_block_size = 1 + 1 + input_margin;
62
63	const TableEntry = struct {
64	val: u32, // Value at destination
65	offset: i32,
66	};
67
68	pub fn deflateFast() DeflateFast {
69	return DeflateFast{
70	.table = [_]TableEntry{.{ .val = 0, .offset = 0 }} ** table_size,
71	.prev = undefined,
72	.prev_len = 0,
73	.cur = max_store_block_size,
74	.allocator = undefined,
75	};
76	}
77
78	// DeflateFast maintains the table for matches,
79	// and the previous byte block for cross block matching.
80	pub const DeflateFast = struct {
81	table: [table_size]TableEntry,
82	prev: []u8, // Previous block, zero length if unknown.
83	prev_len: u32, // Previous block length
84	cur: i32, // Current match offset.
85	allocator: Allocator,
86
87	const Self = @This();
88
89	pub fn init(self: *Self, allocator: Allocator) !void {
90	self.allocator = allocator;
91	self.prev = try allocator.alloc(u8, max_store_block_size);
92	self.prev_len = 0;
93	}
94
95	pub fn deinit(self: *Self) void {
96	self.allocator.free(self.prev);
97	self.prev_len = 0;
98	}
99
100	// Encodes a block given in `src` and appends tokens to `dst` and returns the result.
101	pub fn encode(self: Self, dst: []token.Token, tokens_count: u16, src: []u8) void {
102
103	// Ensure that self.cur doesn't wrap.
104	if (self.cur >= buffer_reset) {
105	self.shiftOffsets();
106	}
107
108	// This check isn't in the Snappy implementation, but there, the caller
109	// instead of the callee handles this case.
110	if (src.len < min_non_literal_block_size) {
111	self.cur += max_store_block_size;
112	self.prev_len = 0;
113	emitLiteral(dst, tokens_count, src);
114	return;
115	}
116
117	// s_limit is when to stop looking for offset/length copies. The input_margin
118	// lets us use a fast path for emitLiteral in the main loop, while we are
119	// looking for copies.
120	const s_limit = @as(i32, @intCast(src.len - input_margin));
121
122	// next_emit is where in src the next emitLiteral should start from.
123	var next_emit: i32 = 0;
124	var s: i32 = 0;
125	var cv: u32 = load32(src, s);
126	var next_hash: u32 = hash(cv);
127
128	outer: while (true) {
129	// Copied from the C++ snappy implementation:
130	//
131	// Heuristic match skipping: If 32 bytes are scanned with no matches
132	// found, start looking only at every other byte. If 32 more bytes are
133	// scanned (or skipped), look at every third byte, etc.. When a match
134	// is found, immediately go back to looking at every byte. This is a
135	// small loss (~5% performance, ~0.1% density) for compressible data
136	// due to more bookkeeping, but for non-compressible data (such as
137	// JPEG) it's a huge win since the compressor quickly "realizes" the
138	// data is incompressible and doesn't bother looking for matches
139	// everywhere.
140	//
141	// The "skip" variable keeps track of how many bytes there are since
142	// the last match; dividing it by 32 (ie. right-shifting by five) gives
143	// the number of bytes to move ahead for each iteration.
144	var skip: i32 = 32;
145
146	var next_s: i32 = s;
147	var candidate: TableEntry = undefined;
148	while (true) {
149	s = next_s;
150	const bytes_between_hash_lookups = skip >> 5;
151	next_s = s + bytes_between_hash_lookups;
152	skip += bytes_between_hash_lookups;
153	if (next_s > s_limit) {
154	break :outer;
155	}
156	candidate = self.table[next_hash & table_mask];
157	const now = load32(src, next_s);
158	self.table[next_hash & table_mask] = .{ .offset = s + self.cur, .val = cv };
159	next_hash = hash(now);
160
161	const offset = s - (candidate.offset - self.cur);
162	if (offset > max_match_offset or cv != candidate.val) {
163	// Out of range or not matched.
164	cv = now;
165	continue;
166	}
167	break;
168	}
169
170	// A 4-byte match has been found. We'll later see if more than 4 bytes
171	// match. But, prior to the match, src[next_emit..s] are unmatched. Emit
172	// them as literal bytes.
173	emitLiteral(dst, tokens_count, src[@as(usize, @intCast(next_emit))..@as(usize, @intCast(s))]);
174
175	// Call emitCopy, and then see if another emitCopy could be our next
176	// move. Repeat until we find no match for the input immediately after
177	// what was consumed by the last emitCopy call.
178	//
179	// If we exit this loop normally then we need to call emitLiteral next,
180	// though we don't yet know how big the literal will be. We handle that
181	// by proceeding to the next iteration of the main loop. We also can
182	// exit this loop via goto if we get close to exhausting the input.
183	while (true) {
184	// Invariant: we have a 4-byte match at s, and no need to emit any
185	// literal bytes prior to s.
186
187	// Extend the 4-byte match as long as possible.
188	//
189	s += 4;
190	const t = candidate.offset - self.cur + 4;
191	const l = self.matchLen(s, t, src);
192
193	// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
194	dst[tokens_count.*] = token.matchToken(
195	@as(u32, @intCast(l + 4 - base_match_length)),
196	@as(u32, @intCast(s - t - base_match_offset)),
197	);
198	tokens_count.* += 1;
199	s += l;
200	next_emit = s;
201	if (s >= s_limit) {
202	break :outer;
203	}
204
205	// We could immediately start working at s now, but to improve
206	// compression we first update the hash table at s-1 and at s. If
207	// another emitCopy is not our next move, also calculate next_hash
208	// at s+1. At least on amd64 architecture, these three hash calculations
209	// are faster as one load64 call (with some shifts) instead of
210	// three load32 calls.
211	var x = load64(src, s - 1);
212	const prev_hash = hash(@as(u32, @truncate(x)));
213	self.table[prev_hash & table_mask] = TableEntry{
214	.offset = self.cur + s - 1,
215	.val = @as(u32, @truncate(x)),
216	};
217	x >>= 8;
218	const curr_hash = hash(@as(u32, @truncate(x)));
219	candidate = self.table[curr_hash & table_mask];
220	self.table[curr_hash & table_mask] = TableEntry{
221	.offset = self.cur + s,
222	.val = @as(u32, @truncate(x)),
223	};
224
225	const offset = s - (candidate.offset - self.cur);
226	if (offset > max_match_offset or @as(u32, @truncate(x)) != candidate.val) {
227	cv = @as(u32, @truncate(x >> 8));
228	next_hash = hash(cv);
229	s += 1;
230	break;
231	}
232	}
233	}
234
235	if (@as(u32, @intCast(next_emit)) < src.len) {
236	emitLiteral(dst, tokens_count, src[@as(usize, @intCast(next_emit))..]);
237	}
238	self.cur += @as(i32, @intCast(src.len));
239	self.prev_len = @as(u32, @intCast(src.len));
240	@memcpy(self.prev[0..self.prev_len], src);
241	return;
242	}
243
244	fn emitLiteral(dst: []token.Token, tokens_count: *u16, lit: []u8) void {
245	for (lit) \|v\| {
246	dst[tokens_count.*] = token.literalToken(@as(u32, @intCast(v)));
247	tokens_count.* += 1;
248	}
249	return;
250	}
251
252	// matchLen returns the match length between src[s..] and src[t..].
253	// t can be negative to indicate the match is starting in self.prev.
254	// We assume that src[s-4 .. s] and src[t-4 .. t] already match.
255	fn matchLen(self: *Self, s: i32, t: i32, src: []u8) i32 {
256	var s1 = @as(u32, @intCast(s)) + max_match_length - 4;
257	if (s1 > src.len) {
258	s1 = @as(u32, @intCast(src.len));
259	}
260
261	// If we are inside the current block
262	if (t >= 0) {
263	var b = src[@as(usize, @intCast(t))..];
264	const a = src[@as(usize, @intCast(s))..@as(usize, @intCast(s1))];
265	b = b[0..a.len];
266	// Extend the match to be as long as possible.
267	for (a, 0..) \|_, i\| {
268	if (a[i] != b[i]) {
269	return @as(i32, @intCast(i));
270	}
271	}
272	return @as(i32, @intCast(a.len));
273	}
274
275	// We found a match in the previous block.
276	const tp = @as(i32, @intCast(self.prev_len)) + t;
277	if (tp < 0) {
278	return 0;
279	}
280
281	// Extend the match to be as long as possible.
282	var a = src[@as(usize, @intCast(s))..@as(usize, @intCast(s1))];
283	var b = self.prev[@as(usize, @intCast(tp))..@as(usize, @intCast(self.prev_len))];
284	if (b.len > a.len) {
285	b = b[0..a.len];
286	}
287	a = a[0..b.len];
288	for (b, 0..) \|_, i\| {
289	if (a[i] != b[i]) {
290	return @as(i32, @intCast(i));
291	}
292	}
293
294	// If we reached our limit, we matched everything we are
295	// allowed to in the previous block and we return.
296	const n = @as(i32, @intCast(b.len));
297	if (@as(u32, @intCast(s + n)) == s1) {
298	return n;
299	}
300
301	// Continue looking for more matches in the current block.
302	a = src[@as(usize, @intCast(s + n))..@as(usize, @intCast(s1))];
303	b = src[0..a.len];
304	for (a, 0..) \|_, i\| {
305	if (a[i] != b[i]) {
306	return @as(i32, @intCast(i)) + n;
307	}
308	}
309	return @as(i32, @intCast(a.len)) + n;
310	}
311
312	// Reset resets the encoding history.
313	// This ensures that no matches are made to the previous block.
314	pub fn reset(self: *Self) void {
315	self.prev_len = 0;
316	// Bump the offset, so all matches will fail distance check.
317	// Nothing should be >= self.cur in the table.
318	self.cur += max_match_offset;
319
320	// Protect against self.cur wraparound.
321	if (self.cur >= buffer_reset) {
322	self.shiftOffsets();
323	}
324	}
325
326	// shiftOffsets will shift down all match offset.
327	// This is only called in rare situations to prevent integer overflow.
328	//
329	// See https://golang.org/issue/18636 and https://golang.org/issues/34121.
330	fn shiftOffsets(self: *Self) void {
331	if (self.prev_len == 0) {
332	// We have no history; just clear the table.
333	for (self.table, 0..) \|_, i\| {
334	self.table[i] = TableEntry{ .val = 0, .offset = 0 };
335	}
336	self.cur = max_match_offset + 1;
337	return;
338	}
339
340	// Shift down everything in the table that isn't already too far away.
341	for (self.table, 0..) \|_, i\| {
342	var v = self.table[i].offset - self.cur + max_match_offset + 1;
343	if (v < 0) {
344	// We want to reset self.cur to max_match_offset + 1, so we need to shift
345	// all table entries down by (self.cur - (max_match_offset + 1)).
346	// Because we ignore matches > max_match_offset, we can cap
347	// any negative offsets at 0.
348	v = 0;
349	}
350	self.table[i].offset = v;
351	}
352	self.cur = max_match_offset + 1;
353	}
354	};
355
356	test "best speed match 1/3" {
357	if (@import("builtin").os.tag == .wasi) {
358	// https://github.com/ziglang/zig/issues/18885
359	return error.SkipZigTest;
360	}
361	const expectEqual = std.testing.expectEqual;
362
363	{
364	var previous = [_]u8{ 0, 0, 0, 1, 2 };
365	var e = DeflateFast{
366	.prev = &previous,
367	.prev_len = previous.len,
368	.table = undefined,
369	.allocator = undefined,
370	.cur = 0,
371	};
372	var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 };
373	const got: i32 = e.matchLen(3, -3, &current);
374	try expectEqual(@as(i32, 6), got);
375	}
376	{
377	var previous = [_]u8{ 0, 0, 0, 1, 2 };
378	var e = DeflateFast{
379	.prev = &previous,
380	.prev_len = previous.len,
381	.table = undefined,
382	.allocator = undefined,
383	.cur = 0,
384	};
385	var current = [_]u8{ 2, 4, 5, 0, 1, 2, 3, 4, 5 };
386	const got: i32 = e.matchLen(3, -3, &current);
387	try expectEqual(@as(i32, 3), got);
388	}
389	{
390	var previous = [_]u8{ 0, 0, 0, 1, 1 };
391	var e = DeflateFast{
392	.prev = &previous,
393	.prev_len = previous.len,
394	.table = undefined,
395	.allocator = undefined,
396	.cur = 0,
397	};
398	var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 };
399	const got: i32 = e.matchLen(3, -3, &current);
400	try expectEqual(@as(i32, 2), got);
401	}
402	{
403	var previous = [_]u8{ 0, 0, 0, 1, 2 };
404	var e = DeflateFast{
405	.prev = &previous,
406	.prev_len = previous.len,
407	.table = undefined,
408	.allocator = undefined,
409	.cur = 0,
410	};
411	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
412	const got: i32 = e.matchLen(0, -1, &current);
413	try expectEqual(@as(i32, 4), got);
414	}
415	{
416	var previous = [_]u8{ 0, 0, 0, 1, 2, 3, 4, 5, 2, 2 };
417	var e = DeflateFast{
418	.prev = &previous,
419	.prev_len = previous.len,
420	.table = undefined,
421	.allocator = undefined,
422	.cur = 0,
423	};
424	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
425	const got: i32 = e.matchLen(4, -7, &current);
426	try expectEqual(@as(i32, 5), got);
427	}
428	{
429	var previous = [_]u8{ 9, 9, 9, 9, 9 };
430	var e = DeflateFast{
431	.prev = &previous,
432	.prev_len = previous.len,
433	.table = undefined,
434	.allocator = undefined,
435	.cur = 0,
436	};
437	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
438	const got: i32 = e.matchLen(0, -1, &current);
439	try expectEqual(@as(i32, 0), got);
440	}
441	{
442	var previous = [_]u8{ 9, 9, 9, 9, 9 };
443	var e = DeflateFast{
444	.prev = &previous,
445	.prev_len = previous.len,
446	.table = undefined,
447	.allocator = undefined,
448	.cur = 0,
449	};
450	var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
451	const got: i32 = e.matchLen(1, 0, &current);
452	try expectEqual(@as(i32, 0), got);
453	}
454	}
455
456	test "best speed match 2/3" {
457	if (@import("builtin").os.tag == .wasi) {
458	// https://github.com/ziglang/zig/issues/18885
459	return error.SkipZigTest;
460	}
461	const expectEqual = std.testing.expectEqual;
462
463	{
464	var previous = [_]u8{};
465	var e = DeflateFast{
466	.prev = &previous,
467	.prev_len = previous.len,
468	.table = undefined,
469	.allocator = undefined,
470	.cur = 0,
471	};
472	var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
473	const got: i32 = e.matchLen(1, -5, &current);
474	try expectEqual(@as(i32, 0), got);
475	}
476	{
477	var previous = [_]u8{};
478	var e = DeflateFast{
479	.prev = &previous,
480	.prev_len = previous.len,
481	.table = undefined,
482	.allocator = undefined,
483	.cur = 0,
484	};
485	var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
486	const got: i32 = e.matchLen(1, -1, &current);
487	try expectEqual(@as(i32, 0), got);
488	}
489	{
490	var previous = [_]u8{};
491	var e = DeflateFast{
492	.prev = &previous,
493	.prev_len = previous.len,
494	.table = undefined,
495	.allocator = undefined,
496	.cur = 0,
497	};
498	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
499	const got: i32 = e.matchLen(1, 0, &current);
500	try expectEqual(@as(i32, 3), got);
501	}
502	{
503	var previous = [_]u8{ 3, 4, 5 };
504	var e = DeflateFast{
505	.prev = &previous,
506	.prev_len = previous.len,
507	.table = undefined,
508	.allocator = undefined,
509	.cur = 0,
510	};
511	var current = [_]u8{ 3, 4, 5 };
512	const got: i32 = e.matchLen(0, -3, &current);
513	try expectEqual(@as(i32, 3), got);
514	}
515	}
516
517	test "best speed match 2/2" {
518	const testing = std.testing;
519	const expectEqual = testing.expectEqual;
520
521	const Case = struct {
522	previous: u32,
523	current: u32,
524	s: i32,
525	t: i32,
526	expected: i32,
527	};
528
529	const cases = [_]Case{
530	.{
531	.previous = 1000,
532	.current = 1000,
533	.s = 0,
534	.t = -1000,
535	.expected = max_match_length - 4,
536	},
537	.{
538	.previous = 200,
539	.s = 0,
540	.t = -200,
541	.current = 500,
542	.expected = max_match_length - 4,
543	},
544	.{
545	.previous = 200,
546	.s = 1,
547	.t = 0,
548	.current = 500,
549	.expected = max_match_length - 4,
550	},
551	.{
552	.previous = max_match_length - 4,
553	.s = 0,
554	.t = -(max_match_length - 4),
555	.current = 500,
556	.expected = max_match_length - 4,
557	},
558	.{
559	.previous = 200,
560	.s = 400,
561	.t = -200,
562	.current = 500,
563	.expected = 100,
564	},
565	.{
566	.previous = 10,
567	.s = 400,
568	.t = 200,
569	.current = 500,
570	.expected = 100,
571	},
572	};
573
574	for (cases) \|c\| {
575	const previous = try testing.allocator.alloc(u8, c.previous);
576	defer testing.allocator.free(previous);
577	@memset(previous, 0);
578
579	const current = try testing.allocator.alloc(u8, c.current);
580	defer testing.allocator.free(current);
581	@memset(current, 0);
582
583	var e = DeflateFast{
584	.prev = previous,
585	.prev_len = @as(u32, @intCast(previous.len)),
586	.table = undefined,
587	.allocator = undefined,
588	.cur = 0,
589	};
590	const got: i32 = e.matchLen(c.s, c.t, current);
591	try expectEqual(@as(i32, c.expected), got);
592	}
593	}
594
595	test "best speed shift offsets" {
596	const testing = std.testing;
597	const expect = std.testing.expect;
598
599	// Test if shiftoffsets properly preserves matches and resets out-of-range matches
600	// seen in https://github.com/golang/go/issues/4142
601	var enc = deflateFast();
602	try enc.init(testing.allocator);
603	defer enc.deinit();
604
605	// test_data may not generate internal matches.
606	var test_data = [32]u8{
607	0xf5, 0x25, 0xf2, 0x55, 0xf6, 0xc1, 0x1f, 0x0b, 0x10, 0xa1,
608	0xd0, 0x77, 0x56, 0x38, 0xf1, 0x9c, 0x7f, 0x85, 0xc5, 0xbd,
609	0x16, 0x28, 0xd4, 0xf9, 0x03, 0xd4, 0xc0, 0xa1, 0x1e, 0x58,
610	0x5b, 0xc9,
611	};
612
613	var tokens = [_]token.Token{0} ** 32;
614	var tokens_count: u16 = 0;
615
616	// Encode the testdata with clean state.
617	// Second part should pick up matches from the first block.
618	tokens_count = 0;
619	enc.encode(&tokens, &tokens_count, &test_data);
620	const want_first_tokens = tokens_count;
621	tokens_count = 0;
622	enc.encode(&tokens, &tokens_count, &test_data);
623	const want_second_tokens = tokens_count;
624
625	try expect(want_first_tokens > want_second_tokens);
626
627	// Forward the current indicator to before wraparound.
628	enc.cur = buffer_reset - @as(i32, @intCast(test_data.len));
629
630	// Part 1 before wrap, should match clean state.
631	tokens_count = 0;
632	enc.encode(&tokens, &tokens_count, &test_data);
633	var got = tokens_count;
634	try testing.expectEqual(want_first_tokens, got);
635
636	// Verify we are about to wrap.
637	try testing.expectEqual(@as(i32, buffer_reset), enc.cur);
638
639	// Part 2 should match clean state as well even if wrapped.
640	tokens_count = 0;
641	enc.encode(&tokens, &tokens_count, &test_data);
642	got = tokens_count;
643	try testing.expectEqual(want_second_tokens, got);
644
645	// Verify that we wrapped.
646	try expect(enc.cur < buffer_reset);
647
648	// Forward the current buffer, leaving the matches at the bottom.
649	enc.cur = buffer_reset;
650	enc.shiftOffsets();
651
652	// Ensure that no matches were picked up.
653	tokens_count = 0;
654	enc.encode(&tokens, &tokens_count, &test_data);
655	got = tokens_count;
656	try testing.expectEqual(want_first_tokens, got);
657	}
658
659	test "best speed reset" {
660	// test that encoding is consistent across a warparound of the table offset.
661	// See https://github.com/golang/go/issues/34121
662	const fmt = std.fmt;
663	const testing = std.testing;
664
665	const ArrayList = std.ArrayList;
666
667	const input_size = 65536;
668	const input = try testing.allocator.alloc(u8, input_size);
669	defer testing.allocator.free(input);
670
671	var i: usize = 0;
672	while (i < input_size) : (i += 1) {
673	_ = try fmt.bufPrint(input, "asdfasdfasdfasdf{d}{d}fghfgujyut{d}yutyu\n", .{ i, i, i });
674	}
675	// This is specific to level 1 (best_speed).
676	const level = .best_speed;
677	const offset: usize = 1;
678
679	// We do an encode with a clean buffer to compare.
680	var want = ArrayList(u8).init(testing.allocator);
681	defer want.deinit();
682	var clean_comp = try deflate.compressor(
683	testing.allocator,
684	want.writer(),
685	.{ .level = level },
686	);
687	defer clean_comp.deinit();
688
689	// Write 3 times, close.
690	try clean_comp.writer().writeAll(input);
691	try clean_comp.writer().writeAll(input);
692	try clean_comp.writer().writeAll(input);
693	try clean_comp.close();
694
695	var o = offset;
696	while (o <= 256) : (o *= 2) {
697	var discard = ArrayList(u8).init(testing.allocator);
698	defer discard.deinit();
699
700	var comp = try deflate.compressor(
701	testing.allocator,
702	discard.writer(),
703	.{ .level = level },
704	);
705	defer comp.deinit();
706
707	// Reset until we are right before the wraparound.
708	// Each reset adds max_match_offset to the offset.
709	i = 0;
710	const limit = (buffer_reset - input.len - o - max_match_offset) / max_match_offset;
711	while (i < limit) : (i += 1) {
712	// skip ahead to where we are close to wrap around...
713	comp.reset(discard.writer());
714	}
715	var got = ArrayList(u8).init(testing.allocator);
716	defer got.deinit();
717	comp.reset(got.writer());
718
719	// Write 3 times, close.
720	try comp.writer().writeAll(input);
721	try comp.writer().writeAll(input);
722	try comp.writer().writeAll(input);
723	try comp.close();
724
725	// output must match at wraparound
726	try testing.expectEqualSlices(u8, want.items, got.items);
727	}
728	}

lib/std/compress/deflate/deflate_fast_test.zig deleted-160

...	@@ -1,160 +0,0 @@
1	const std = @import("std");
2	const expect = std.testing.expect;
3	const io = std.io;
4	const mem = std.mem;
5	const testing = std.testing;
6
7	const ArrayList = std.ArrayList;
8
9	const deflate = @import("compressor.zig");
10	const inflate = @import("decompressor.zig");
11	const deflate_const = @import("deflate_const.zig");
12
13	test "best speed" {
14	// Tests that round-tripping through deflate and then inflate recovers the original input.
15	// The Write sizes are near the thresholds in the compressor.encSpeed method (0, 16, 128), as well
16	// as near `deflate_const.max_store_block_size` (65535).
17
18	var abcabc = try testing.allocator.alloc(u8, 131_072);
19	defer testing.allocator.free(abcabc);
20
21	for (abcabc, 0..) \|_, i\| {
22	abcabc[i] = @as(u8, @intCast(i % 128));
23	}
24
25	var tc_01 = [_]u32{ 65536, 0 };
26	var tc_02 = [_]u32{ 65536, 1 };
27	var tc_03 = [_]u32{ 65536, 1, 256 };
28	var tc_04 = [_]u32{ 65536, 1, 65536 };
29	var tc_05 = [_]u32{ 65536, 14 };
30	var tc_06 = [_]u32{ 65536, 15 };
31	var tc_07 = [_]u32{ 65536, 16 };
32	var tc_08 = [_]u32{ 65536, 16, 256 };
33	var tc_09 = [_]u32{ 65536, 16, 65536 };
34	var tc_10 = [_]u32{ 65536, 127 };
35	var tc_11 = [_]u32{ 65536, 127 };
36	var tc_12 = [_]u32{ 65536, 128 };
37	var tc_13 = [_]u32{ 65536, 128, 256 };
38	var tc_14 = [_]u32{ 65536, 128, 65536 };
39	var tc_15 = [_]u32{ 65536, 129 };
40	var tc_16 = [_]u32{ 65536, 65536, 256 };
41	var tc_17 = [_]u32{ 65536, 65536, 65536 };
42	const test_cases = [_][]u32{
43	&tc_01, &tc_02, &tc_03, &tc_04, &tc_05, &tc_06, &tc_07, &tc_08, &tc_09, &tc_10,
44	&tc_11, &tc_12, &tc_13, &tc_14, &tc_15, &tc_16, &tc_17,
45	};
46
47	for (test_cases) \|tc\| {
48	const firsts = [_]u32{ 1, 65534, 65535, 65536, 65537, 131072 };
49
50	for (firsts) \|first_n\| {
51	tc[0] = first_n;
52
53	const to_flush = [_]bool{ false, true };
54	for (to_flush) \|flush\| {
55	var compressed = ArrayList(u8).init(testing.allocator);
56	defer compressed.deinit();
57
58	var want = ArrayList(u8).init(testing.allocator);
59	defer want.deinit();
60
61	var comp = try deflate.compressor(
62	testing.allocator,
63	compressed.writer(),
64	.{ .level = .best_speed },
65	);
66	defer comp.deinit();
67
68	for (tc) \|n\| {
69	try want.appendSlice(abcabc[0..n]);
70	try comp.writer().writeAll(abcabc[0..n]);
71	if (flush) {
72	try comp.flush();
73	}
74	}
75
76	try comp.close();
77
78	const decompressed = try testing.allocator.alloc(u8, want.items.len);
79	defer testing.allocator.free(decompressed);
80
81	var fib = io.fixedBufferStream(compressed.items);
82	var decomp = try inflate.decompressor(testing.allocator, fib.reader(), null);
83	defer decomp.deinit();
84
85	const read = try decomp.reader().readAll(decompressed);
86	try decomp.close();
87
88	try testing.expectEqual(want.items.len, read);
89	try testing.expectEqualSlices(u8, want.items, decompressed);
90	}
91	}
92	}
93	}
94
95	test "best speed max match offset" {
96	const abc = "abcdefgh";
97	const xyz = "stuvwxyz";
98	const input_margin = 16 - 1;
99
100	const match_before = [_]bool{ false, true };
101	for (match_before) \|do_match_before\| {
102	const extras = [_]u32{
103	0,
104	input_margin - 1,
105	input_margin,
106	input_margin + 1,
107	2 * input_margin,
108	};
109	for (extras) \|extra\| {
110	var offset_adj: i32 = -5;
111	while (offset_adj <= 5) : (offset_adj += 1) {
112	const offset = deflate_const.max_match_offset + offset_adj;
113
114	// Make src to be a []u8 of the form
115	// fmt("{s}{s}{s}{s}{s}", .{abc, zeros0, xyzMaybe, abc, zeros1})
116	// where:
117	// zeros0 is approximately max_match_offset zeros.
118	// xyzMaybe is either xyz or the empty string.
119	// zeros1 is between 0 and 30 zeros.
120	// The difference between the two abc's will be offset, which
121	// is max_match_offset plus or minus a small adjustment.
122	const src_len: usize = @as(usize, @intCast(offset + @as(i32, abc.len) + @as(i32, @intCast(extra))));
123	var src = try testing.allocator.alloc(u8, src_len);
124	defer testing.allocator.free(src);
125
126	@memcpy(src[0..abc.len], abc);
127	if (!do_match_before) {
128	const src_offset: usize = @as(usize, @intCast(offset - @as(i32, xyz.len)));
129	@memcpy(src[src_offset..][0..xyz.len], xyz);
130	}
131	const src_offset: usize = @as(usize, @intCast(offset));
132	@memcpy(src[src_offset..][0..abc.len], abc);
133
134	var compressed = ArrayList(u8).init(testing.allocator);
135	defer compressed.deinit();
136
137	var comp = try deflate.compressor(
138	testing.allocator,
139	compressed.writer(),
140	.{ .level = .best_speed },
141	);
142	defer comp.deinit();
143	try comp.writer().writeAll(src);
144	_ = try comp.close();
145
146	const decompressed = try testing.allocator.alloc(u8, src.len);
147	defer testing.allocator.free(decompressed);
148
149	var fib = io.fixedBufferStream(compressed.items);
150	var decomp = try inflate.decompressor(testing.allocator, fib.reader(), null);
151	defer decomp.deinit();
152	const read = try decomp.reader().readAll(decompressed);
153	try decomp.close();
154
155	try testing.expectEqual(src.len, read);
156	try testing.expectEqualSlices(u8, src, decompressed);
157	}
158	}
159	}
160	}

lib/std/compress/deflate/dict_decoder.zig deleted-423

...	@@ -1,423 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const mem = std.mem;
4
5	const Allocator = std.mem.Allocator;
6
7	// Implements the LZ77 sliding dictionary as used in decompression.
8	// LZ77 decompresses data through sequences of two forms of commands:
9	//
10	// * Literal insertions: Runs of one or more symbols are inserted into the data
11	// stream as is. This is accomplished through the writeByte method for a
12	// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
13	// Any valid stream must start with a literal insertion if no preset dictionary
14	// is used.
15	//
16	// * Backward copies: Runs of one or more symbols are copied from previously
17	// emitted data. Backward copies come as the tuple (dist, length) where dist
18	// determines how far back in the stream to copy from and length determines how
19	// many bytes to copy. Note that it is valid for the length to be greater than
20	// the distance. Since LZ77 uses forward copies, that situation is used to
21	// perform a form of run-length encoding on repeated runs of symbols.
22	// The writeCopy and tryWriteCopy are used to implement this command.
23	//
24	// For performance reasons, this implementation performs little to no sanity
25	// checks about the arguments. As such, the invariants documented for each
26	// method call must be respected.
27	pub const DictDecoder = struct {
28	const Self = @This();
29
30	allocator: Allocator = undefined,
31
32	hist: []u8 = undefined, // Sliding window history
33
34	// Invariant: 0 <= rd_pos <= wr_pos <= hist.len
35	wr_pos: u32 = 0, // Current output position in buffer
36	rd_pos: u32 = 0, // Have emitted hist[0..rd_pos] already
37	full: bool = false, // Has a full window length been written yet?
38
39	// init initializes DictDecoder to have a sliding window dictionary of the given
40	// size. If a preset dict is provided, it will initialize the dictionary with
41	// the contents of dict.
42	pub fn init(self: *Self, allocator: Allocator, size: u32, dict: ?[]const u8) !void {
43	self.allocator = allocator;
44
45	self.hist = try allocator.alloc(u8, size);
46
47	self.wr_pos = 0;
48
49	if (dict != null) {
50	const src = dict.?[dict.?.len -\| self.hist.len..];
51	@memcpy(self.hist[0..src.len], src);
52	self.wr_pos = @as(u32, @intCast(dict.?.len));
53	}
54
55	if (self.wr_pos == self.hist.len) {
56	self.wr_pos = 0;
57	self.full = true;
58	}
59	self.rd_pos = self.wr_pos;
60	}
61
62	pub fn deinit(self: *Self) void {
63	self.allocator.free(self.hist);
64	}
65
66	// Reports the total amount of historical data in the dictionary.
67	pub fn histSize(self: *Self) u32 {
68	if (self.full) {
69	return @as(u32, @intCast(self.hist.len));
70	}
71	return self.wr_pos;
72	}
73
74	// Reports the number of bytes that can be flushed by readFlush.
75	pub fn availRead(self: *Self) u32 {
76	return self.wr_pos - self.rd_pos;
77	}
78
79	// Reports the available amount of output buffer space.
80	pub fn availWrite(self: *Self) u32 {
81	return @as(u32, @intCast(self.hist.len - self.wr_pos));
82	}
83
84	// Returns a slice of the available buffer to write data to.
85	//
86	// This invariant will be kept: s.len <= availWrite()
87	pub fn writeSlice(self: *Self) []u8 {
88	return self.hist[self.wr_pos..];
89	}
90
91	// Advances the writer pointer by `count`.
92	//
93	// This invariant must be kept: 0 <= count <= availWrite()
94	pub fn writeMark(self: *Self, count: u32) void {
95	assert(0 <= count and count <= self.availWrite());
96	self.wr_pos += count;
97	}
98
99	// Writes a single byte to the dictionary.
100	//
101	// This invariant must be kept: 0 < availWrite()
102	pub fn writeByte(self: *Self, byte: u8) void {
103	self.hist[self.wr_pos] = byte;
104	self.wr_pos += 1;
105	}
106
107	/// TODO: eliminate this function because the callsites should care about whether
108	/// or not their arguments alias and then they should directly call `@memcpy` or
109	/// `mem.copyForwards`.
110	fn copy(dst: []u8, src: []const u8) u32 {
111	if (src.len > dst.len) {
112	mem.copyForwards(u8, dst, src[0..dst.len]);
113	return @as(u32, @intCast(dst.len));
114	}
115	mem.copyForwards(u8, dst[0..src.len], src);
116	return @as(u32, @intCast(src.len));
117	}
118
119	// Copies a string at a given (dist, length) to the output.
120	// This returns the number of bytes copied and may be less than the requested
121	// length if the available space in the output buffer is too small.
122	//
123	// This invariant must be kept: 0 < dist <= histSize()
124	pub fn writeCopy(self: *Self, dist: u32, length: u32) u32 {
125	assert(0 < dist and dist <= self.histSize());
126	const dst_base = self.wr_pos;
127	var dst_pos = dst_base;
128	var src_pos: i32 = @as(i32, @intCast(dst_pos)) - @as(i32, @intCast(dist));
129	var end_pos = dst_pos + length;
130	if (end_pos > self.hist.len) {
131	end_pos = @as(u32, @intCast(self.hist.len));
132	}
133
134	// Copy non-overlapping section after destination position.
135	//
136	// This section is non-overlapping in that the copy length for this section
137	// is always less than or equal to the backwards distance. This can occur
138	// if a distance refers to data that wraps-around in the buffer.
139	// Thus, a backwards copy is performed here; that is, the exact bytes in
140	// the source prior to the copy is placed in the destination.
141	if (src_pos < 0) {
142	src_pos += @as(i32, @intCast(self.hist.len));
143	dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@as(usize, @intCast(src_pos))..]);
144	src_pos = 0;
145	}
146
147	// Copy possibly overlapping section before destination position.
148	//
149	// This section can overlap if the copy length for this section is larger
150	// than the backwards distance. This is allowed by LZ77 so that repeated
151	// strings can be succinctly represented using (dist, length) pairs.
152	// Thus, a forwards copy is performed here; that is, the bytes copied is
153	// possibly dependent on the resulting bytes in the destination as the copy
154	// progresses along. This is functionally equivalent to the following:
155	//
156	// var i = 0;
157	// while(i < end_pos - dst_pos) : (i+=1) {
158	// self.hist[dst_pos+i] = self.hist[src_pos+i];
159	// }
160	// dst_pos = end_pos;
161	//
162	while (dst_pos < end_pos) {
163	dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@as(usize, @intCast(src_pos))..dst_pos]);
164	}
165
166	self.wr_pos = dst_pos;
167	return dst_pos - dst_base;
168	}
169
170	// Tries to copy a string at a given (distance, length) to the
171	// output. This specialized version is optimized for short distances.
172	//
173	// This method is designed to be inlined for performance reasons.
174	//
175	// This invariant must be kept: 0 < dist <= histSize()
176	pub fn tryWriteCopy(self: *Self, dist: u32, length: u32) u32 {
177	var dst_pos = self.wr_pos;
178	const end_pos = dst_pos + length;
179	if (dst_pos < dist or end_pos > self.hist.len) {
180	return 0;
181	}
182	const dst_base = dst_pos;
183	const src_pos = dst_pos - dist;
184
185	// Copy possibly overlapping section before destination position.
186	while (dst_pos < end_pos) {
187	dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[src_pos..dst_pos]);
188	}
189
190	self.wr_pos = dst_pos;
191	return dst_pos - dst_base;
192	}
193
194	// Returns a slice of the historical buffer that is ready to be
195	// emitted to the user. The data returned by readFlush must be fully consumed
196	// before calling any other DictDecoder methods.
197	pub fn readFlush(self: *Self) []u8 {
198	const to_read = self.hist[self.rd_pos..self.wr_pos];
199	self.rd_pos = self.wr_pos;
200	if (self.wr_pos == self.hist.len) {
201	self.wr_pos = 0;
202	self.rd_pos = 0;
203	self.full = true;
204	}
205	return to_read;
206	}
207	};
208
209	// tests
210
211	test "dictionary decoder" {
212	const ArrayList = std.ArrayList;
213	const testing = std.testing;
214
215	const abc = "ABC\n";
216	const fox = "The quick brown fox jumped over the lazy dog!\n";
217	const poem: []const u8 =
218	\\The Road Not Taken
219	\\Robert Frost
220	\\
221	\\Two roads diverged in a yellow wood,
222	\\And sorry I could not travel both
223	\\And be one traveler, long I stood
224	\\And looked down one as far as I could
225	\\To where it bent in the undergrowth;
226	\\
227	\\Then took the other, as just as fair,
228	\\And having perhaps the better claim,
229	\\Because it was grassy and wanted wear;
230	\\Though as for that the passing there
231	\\Had worn them really about the same,
232	\\
233	\\And both that morning equally lay
234	\\In leaves no step had trodden black.
235	\\Oh, I kept the first for another day!
236	\\Yet knowing how way leads on to way,
237	\\I doubted if I should ever come back.
238	\\
239	\\I shall be telling this with a sigh
240	\\Somewhere ages and ages hence:
241	\\Two roads diverged in a wood, and I-
242	\\I took the one less traveled by,
243	\\And that has made all the difference.
244	\\
245	;
246
247	const uppercase: []const u8 =
248	\\THE ROAD NOT TAKEN
249	\\ROBERT FROST
250	\\
251	\\TWO ROADS DIVERGED IN A YELLOW WOOD,
252	\\AND SORRY I COULD NOT TRAVEL BOTH
253	\\AND BE ONE TRAVELER, LONG I STOOD
254	\\AND LOOKED DOWN ONE AS FAR AS I COULD
255	\\TO WHERE IT BENT IN THE UNDERGROWTH;
256	\\
257	\\THEN TOOK THE OTHER, AS JUST AS FAIR,
258	\\AND HAVING PERHAPS THE BETTER CLAIM,
259	\\BECAUSE IT WAS GRASSY AND WANTED WEAR;
260	\\THOUGH AS FOR THAT THE PASSING THERE
261	\\HAD WORN THEM REALLY ABOUT THE SAME,
262	\\
263	\\AND BOTH THAT MORNING EQUALLY LAY
264	\\IN LEAVES NO STEP HAD TRODDEN BLACK.
265	\\OH, I KEPT THE FIRST FOR ANOTHER DAY!
266	\\YET KNOWING HOW WAY LEADS ON TO WAY,
267	\\I DOUBTED IF I SHOULD EVER COME BACK.
268	\\
269	\\I SHALL BE TELLING THIS WITH A SIGH
270	\\SOMEWHERE AGES AND AGES HENCE:
271	\\TWO ROADS DIVERGED IN A WOOD, AND I-
272	\\I TOOK THE ONE LESS TRAVELED BY,
273	\\AND THAT HAS MADE ALL THE DIFFERENCE.
274	\\
275	;
276
277	const PoemRefs = struct {
278	dist: u32, // Backward distance (0 if this is an insertion)
279	length: u32, // Length of copy or insertion
280	};
281
282	const poem_refs = [_]PoemRefs{
283	.{ .dist = 0, .length = 38 }, .{ .dist = 33, .length = 3 }, .{ .dist = 0, .length = 48 },
284	.{ .dist = 79, .length = 3 }, .{ .dist = 0, .length = 11 }, .{ .dist = 34, .length = 5 },
285	.{ .dist = 0, .length = 6 }, .{ .dist = 23, .length = 7 }, .{ .dist = 0, .length = 8 },
286	.{ .dist = 50, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 69, .length = 3 },
287	.{ .dist = 34, .length = 5 }, .{ .dist = 0, .length = 4 }, .{ .dist = 97, .length = 3 },
288	.{ .dist = 0, .length = 4 }, .{ .dist = 43, .length = 5 }, .{ .dist = 0, .length = 6 },
289	.{ .dist = 7, .length = 4 }, .{ .dist = 88, .length = 7 }, .{ .dist = 0, .length = 12 },
290	.{ .dist = 80, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 141, .length = 4 },
291	.{ .dist = 0, .length = 1 }, .{ .dist = 196, .length = 3 }, .{ .dist = 0, .length = 3 },
292	.{ .dist = 157, .length = 3 }, .{ .dist = 0, .length = 6 }, .{ .dist = 181, .length = 3 },
293	.{ .dist = 0, .length = 2 }, .{ .dist = 23, .length = 3 }, .{ .dist = 77, .length = 3 },
294	.{ .dist = 28, .length = 5 }, .{ .dist = 128, .length = 3 }, .{ .dist = 110, .length = 4 },
295	.{ .dist = 70, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 85, .length = 6 },
296	.{ .dist = 0, .length = 2 }, .{ .dist = 182, .length = 6 }, .{ .dist = 0, .length = 4 },
297	.{ .dist = 133, .length = 3 }, .{ .dist = 0, .length = 7 }, .{ .dist = 47, .length = 5 },
298	.{ .dist = 0, .length = 20 }, .{ .dist = 112, .length = 5 }, .{ .dist = 0, .length = 1 },
299	.{ .dist = 58, .length = 3 }, .{ .dist = 0, .length = 8 }, .{ .dist = 59, .length = 3 },
300	.{ .dist = 0, .length = 4 }, .{ .dist = 173, .length = 3 }, .{ .dist = 0, .length = 5 },
301	.{ .dist = 114, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 92, .length = 5 },
302	.{ .dist = 0, .length = 2 }, .{ .dist = 71, .length = 3 }, .{ .dist = 0, .length = 2 },
303	.{ .dist = 76, .length = 5 }, .{ .dist = 0, .length = 1 }, .{ .dist = 46, .length = 3 },
304	.{ .dist = 96, .length = 4 }, .{ .dist = 130, .length = 4 }, .{ .dist = 0, .length = 3 },
305	.{ .dist = 360, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 178, .length = 5 },
306	.{ .dist = 0, .length = 7 }, .{ .dist = 75, .length = 3 }, .{ .dist = 0, .length = 3 },
307	.{ .dist = 45, .length = 6 }, .{ .dist = 0, .length = 6 }, .{ .dist = 299, .length = 6 },
308	.{ .dist = 180, .length = 3 }, .{ .dist = 70, .length = 6 }, .{ .dist = 0, .length = 1 },
309	.{ .dist = 48, .length = 3 }, .{ .dist = 66, .length = 4 }, .{ .dist = 0, .length = 3 },
310	.{ .dist = 47, .length = 5 }, .{ .dist = 0, .length = 9 }, .{ .dist = 325, .length = 3 },
311	.{ .dist = 0, .length = 1 }, .{ .dist = 359, .length = 3 }, .{ .dist = 318, .length = 3 },
312	.{ .dist = 0, .length = 2 }, .{ .dist = 199, .length = 3 }, .{ .dist = 0, .length = 1 },
313	.{ .dist = 344, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 248, .length = 3 },
314	.{ .dist = 0, .length = 10 }, .{ .dist = 310, .length = 3 }, .{ .dist = 0, .length = 3 },
315	.{ .dist = 93, .length = 6 }, .{ .dist = 0, .length = 3 }, .{ .dist = 252, .length = 3 },
316	.{ .dist = 157, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 273, .length = 5 },
317	.{ .dist = 0, .length = 14 }, .{ .dist = 99, .length = 4 }, .{ .dist = 0, .length = 1 },
318	.{ .dist = 464, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 92, .length = 4 },
319	.{ .dist = 495, .length = 3 }, .{ .dist = 0, .length = 1 }, .{ .dist = 322, .length = 4 },
320	.{ .dist = 16, .length = 4 }, .{ .dist = 0, .length = 3 }, .{ .dist = 402, .length = 3 },
321	.{ .dist = 0, .length = 2 }, .{ .dist = 237, .length = 4 }, .{ .dist = 0, .length = 2 },
322	.{ .dist = 432, .length = 4 }, .{ .dist = 0, .length = 1 }, .{ .dist = 483, .length = 5 },
323	.{ .dist = 0, .length = 2 }, .{ .dist = 294, .length = 4 }, .{ .dist = 0, .length = 2 },
324	.{ .dist = 306, .length = 3 }, .{ .dist = 113, .length = 5 }, .{ .dist = 0, .length = 1 },
325	.{ .dist = 26, .length = 4 }, .{ .dist = 164, .length = 3 }, .{ .dist = 488, .length = 4 },
326	.{ .dist = 0, .length = 1 }, .{ .dist = 542, .length = 3 }, .{ .dist = 248, .length = 6 },
327	.{ .dist = 0, .length = 5 }, .{ .dist = 205, .length = 3 }, .{ .dist = 0, .length = 8 },
328	.{ .dist = 48, .length = 3 }, .{ .dist = 449, .length = 6 }, .{ .dist = 0, .length = 2 },
329	.{ .dist = 192, .length = 3 }, .{ .dist = 328, .length = 4 }, .{ .dist = 9, .length = 5 },
330	.{ .dist = 433, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 622, .length = 25 },
331	.{ .dist = 615, .length = 5 }, .{ .dist = 46, .length = 5 }, .{ .dist = 0, .length = 2 },
332	.{ .dist = 104, .length = 3 }, .{ .dist = 475, .length = 10 }, .{ .dist = 549, .length = 3 },
333	.{ .dist = 0, .length = 4 }, .{ .dist = 597, .length = 8 }, .{ .dist = 314, .length = 3 },
334	.{ .dist = 0, .length = 1 }, .{ .dist = 473, .length = 6 }, .{ .dist = 317, .length = 5 },
335	.{ .dist = 0, .length = 1 }, .{ .dist = 400, .length = 3 }, .{ .dist = 0, .length = 3 },
336	.{ .dist = 109, .length = 3 }, .{ .dist = 151, .length = 3 }, .{ .dist = 48, .length = 4 },
337	.{ .dist = 0, .length = 4 }, .{ .dist = 125, .length = 3 }, .{ .dist = 108, .length = 3 },
338	.{ .dist = 0, .length = 2 },
339	};
340
341	var got_list = ArrayList(u8).init(testing.allocator);
342	defer got_list.deinit();
343	var got = got_list.writer();
344
345	var want_list = ArrayList(u8).init(testing.allocator);
346	defer want_list.deinit();
347	var want = want_list.writer();
348
349	var dd = DictDecoder{};
350	try dd.init(testing.allocator, 1 << 11, null);
351	defer dd.deinit();
352
353	const util = struct {
354	fn writeCopy(dst_dd: *DictDecoder, dst: anytype, dist: u32, length: u32) !void {
355	var len = length;
356	while (len > 0) {
357	var n = dst_dd.tryWriteCopy(dist, len);
358	if (n == 0) {
359	n = dst_dd.writeCopy(dist, len);
360	}
361
362	len -= n;
363	if (dst_dd.availWrite() == 0) {
364	_ = try dst.write(dst_dd.readFlush());
365	}
366	}
367	}
368	fn writeString(dst_dd: *DictDecoder, dst: anytype, str: []const u8) !void {
369	var string = str;
370	while (string.len > 0) {
371	const cnt = DictDecoder.copy(dst_dd.writeSlice(), string);
372	dst_dd.writeMark(cnt);
373	string = string[cnt..];
374	if (dst_dd.availWrite() == 0) {
375	_ = try dst.write(dst_dd.readFlush());
376	}
377	}
378	}
379	};
380
381	try util.writeString(&dd, got, ".");
382	_ = try want.write(".");
383
384	var str = poem;
385	for (poem_refs, 0..) \|ref, i\| {
386	_ = i;
387	if (ref.dist == 0) {
388	try util.writeString(&dd, got, str[0..ref.length]);
389	} else {
390	try util.writeCopy(&dd, got, ref.dist, ref.length);
391	}
392	str = str[ref.length..];
393	}
394	_ = try want.write(poem);
395
396	try util.writeCopy(&dd, got, dd.histSize(), 33);
397	_ = try want.write(want_list.items[0..33]);
398
399	try util.writeString(&dd, got, abc);
400	try util.writeCopy(&dd, got, abc.len, 59 * abc.len);
401	_ = try want.write(abc ** 60);
402
403	try util.writeString(&dd, got, fox);
404	try util.writeCopy(&dd, got, fox.len, 9 * fox.len);
405	_ = try want.write(fox ** 10);
406
407	try util.writeString(&dd, got, ".");
408	try util.writeCopy(&dd, got, 1, 9);
409	_ = try want.write("." ** 10);
410
411	try util.writeString(&dd, got, uppercase);
412	try util.writeCopy(&dd, got, uppercase.len, 7 * uppercase.len);
413	var i: u8 = 0;
414	while (i < 8) : (i += 1) {
415	_ = try want.write(uppercase);
416	}
417
418	try util.writeCopy(&dd, got, dd.histSize(), 10);
419	_ = try want.write(want_list.items[want_list.items.len - dd.histSize() ..][0..10]);
420
421	_ = try got.write(dd.readFlush());
422	try testing.expectEqualSlices(u8, want_list.items, got_list.items);
423	}

lib/std/compress/deflate/huffman_bit_writer.zig deleted-1686

...	@@ -1,1686 +0,0 @@
1	const std = @import("std");
2	const io = std.io;
3
4	const Allocator = std.mem.Allocator;
5
6	const deflate_const = @import("deflate_const.zig");
7	const hm_code = @import("huffman_code.zig");
8	const token = @import("token.zig");
9
10	// The first length code.
11	const length_codes_start = 257;
12
13	// The number of codegen codes.
14	const codegen_code_count = 19;
15	const bad_code = 255;
16
17	// buffer_flush_size indicates the buffer size
18	// after which bytes are flushed to the writer.
19	// Should preferably be a multiple of 6, since
20	// we accumulate 6 bytes between writes to the buffer.
21	const buffer_flush_size = 240;
22
23	// buffer_size is the actual output byte buffer size.
24	// It must have additional headroom for a flush
25	// which can contain up to 8 bytes.
26	const buffer_size = buffer_flush_size + 8;
27
28	// The number of extra bits needed by length code X - LENGTH_CODES_START.
29	var length_extra_bits = [_]u8{
30	0, 0, 0, // 257
31	0, 0, 0, 0, 0, 1, 1, 1, 1, 2, // 260
32	2, 2, 2, 3, 3, 3, 3, 4, 4, 4, // 270
33	4, 5, 5, 5, 5, 0, // 280
34	};
35
36	// The length indicated by length code X - LENGTH_CODES_START.
37	var length_base = [_]u32{
38	0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
39	12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
40	64, 80, 96, 112, 128, 160, 192, 224, 255,
41	};
42
43	// offset code word extra bits.
44	var offset_extra_bits = [_]i8{
45	0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
46	4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
47	9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
48	};
49
50	var offset_base = [_]u32{
51	0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
52	0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
53	0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
54	0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
55	0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
56	0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
57	};
58
59	// The odd order in which the codegen code sizes are written.
60	var codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
61
62	pub fn HuffmanBitWriter(comptime WriterType: type) type {
63	return struct {
64	const Self = @This();
65	pub const Error = WriterType.Error;
66
67	// writer is the underlying writer.
68	// Do not use it directly; use the write method, which ensures
69	// that Write errors are sticky.
70	inner_writer: WriterType,
71	bytes_written: usize,
72
73	// Data waiting to be written is bytes[0 .. nbytes]
74	// and then the low nbits of bits. Data is always written
75	// sequentially into the bytes array.
76	bits: u64,
77	nbits: u32, // number of bits
78	bytes: [buffer_size]u8,
79	codegen_freq: [codegen_code_count]u16,
80	nbytes: u32, // number of bytes
81	literal_freq: []u16,
82	offset_freq: []u16,
83	codegen: []u8,
84	literal_encoding: hm_code.HuffmanEncoder,
85	offset_encoding: hm_code.HuffmanEncoder,
86	codegen_encoding: hm_code.HuffmanEncoder,
87	err: bool = false,
88	fixed_literal_encoding: hm_code.HuffmanEncoder,
89	fixed_offset_encoding: hm_code.HuffmanEncoder,
90	allocator: Allocator,
91	huff_offset: hm_code.HuffmanEncoder,
92
93	pub fn reset(self: *Self, new_writer: WriterType) void {
94	self.inner_writer = new_writer;
95	self.bytes_written = 0;
96	self.bits = 0;
97	self.nbits = 0;
98	self.nbytes = 0;
99	self.err = false;
100	}
101
102	pub fn flush(self: *Self) Error!void {
103	if (self.err) {
104	self.nbits = 0;
105	return;
106	}
107	var n = self.nbytes;
108	while (self.nbits != 0) {
109	self.bytes[n] = @as(u8, @truncate(self.bits));
110	self.bits >>= 8;
111	if (self.nbits > 8) { // Avoid underflow
112	self.nbits -= 8;
113	} else {
114	self.nbits = 0;
115	}
116	n += 1;
117	}
118	self.bits = 0;
119	try self.write(self.bytes[0..n]);
120	self.nbytes = 0;
121	}
122
123	fn write(self: *Self, b: []const u8) Error!void {
124	if (self.err) {
125	return;
126	}
127	try self.inner_writer.writeAll(b);
128	self.bytes_written += b.len;
129	}
130
131	fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
132	if (self.err) {
133	return;
134	}
135	self.bits \|= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
136	self.nbits += nb;
137	if (self.nbits >= 48) {
138	const bits = self.bits;
139	self.bits >>= 48;
140	self.nbits -= 48;
141	var n = self.nbytes;
142	var bytes = self.bytes[n..][0..6];
143	bytes[0] = @as(u8, @truncate(bits));
144	bytes[1] = @as(u8, @truncate(bits >> 8));
145	bytes[2] = @as(u8, @truncate(bits >> 16));
146	bytes[3] = @as(u8, @truncate(bits >> 24));
147	bytes[4] = @as(u8, @truncate(bits >> 32));
148	bytes[5] = @as(u8, @truncate(bits >> 40));
149	n += 6;
150	if (n >= buffer_flush_size) {
151	try self.write(self.bytes[0..n]);
152	n = 0;
153	}
154	self.nbytes = n;
155	}
156	}
157
158	pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
159	if (self.err) {
160	return;
161	}
162	var n = self.nbytes;
163	if (self.nbits & 7 != 0) {
164	self.err = true; // unfinished bits
165	return;
166	}
167	while (self.nbits != 0) {
168	self.bytes[n] = @as(u8, @truncate(self.bits));
169	self.bits >>= 8;
170	self.nbits -= 8;
171	n += 1;
172	}
173	if (n != 0) {
174	try self.write(self.bytes[0..n]);
175	}
176	self.nbytes = 0;
177	try self.write(bytes);
178	}
179
180	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
181	// the literal and offset lengths arrays (which are concatenated into a single
182	// array). This method generates that run-length encoding.
183	//
184	// The result is written into the codegen array, and the frequencies
185	// of each code is written into the codegen_freq array.
186	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
187	// information. Code bad_code is an end marker
188	//
189	// num_literals: The number of literals in literal_encoding
190	// num_offsets: The number of offsets in offset_encoding
191	// lit_enc: The literal encoder to use
192	// off_enc: The offset encoder to use
193	fn generateCodegen(
194	self: *Self,
195	num_literals: u32,
196	num_offsets: u32,
197	lit_enc: *hm_code.HuffmanEncoder,
198	off_enc: *hm_code.HuffmanEncoder,
199	) void {
200	for (self.codegen_freq, 0..) \|_, i\| {
201	self.codegen_freq[i] = 0;
202	}
203
204	// Note that we are using codegen both as a temporary variable for holding
205	// a copy of the frequencies, and as the place where we put the result.
206	// This is fine because the output is always shorter than the input used
207	// so far.
208	var codegen = self.codegen; // cache
209	// Copy the concatenated code sizes to codegen. Put a marker at the end.
210	var cgnl = codegen[0..num_literals];
211	for (cgnl, 0..) \|_, i\| {
212	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
213	}
214
215	cgnl = codegen[num_literals .. num_literals + num_offsets];
216	for (cgnl, 0..) \|_, i\| {
217	cgnl[i] = @as(u8, @intCast(off_enc.codes[i].len));
218	}
219	codegen[num_literals + num_offsets] = bad_code;
220
221	var size = codegen[0];
222	var count: i32 = 1;
223	var out_index: u32 = 0;
224	var in_index: u32 = 1;
225	while (size != bad_code) : (in_index += 1) {
226	// INVARIANT: We have seen "count" copies of size that have not yet
227	// had output generated for them.
228	const next_size = codegen[in_index];
229	if (next_size == size) {
230	count += 1;
231	continue;
232	}
233	// We need to generate codegen indicating "count" of size.
234	if (size != 0) {
235	codegen[out_index] = size;
236	out_index += 1;
237	self.codegen_freq[size] += 1;
238	count -= 1;
239	while (count >= 3) {
240	var n: i32 = 6;
241	if (n > count) {
242	n = count;
243	}
244	codegen[out_index] = 16;
245	out_index += 1;
246	codegen[out_index] = @as(u8, @intCast(n - 3));
247	out_index += 1;
248	self.codegen_freq[16] += 1;
249	count -= n;
250	}
251	} else {
252	while (count >= 11) {
253	var n: i32 = 138;
254	if (n > count) {
255	n = count;
256	}
257	codegen[out_index] = 18;
258	out_index += 1;
259	codegen[out_index] = @as(u8, @intCast(n - 11));
260	out_index += 1;
261	self.codegen_freq[18] += 1;
262	count -= n;
263	}
264	if (count >= 3) {
265	// 3 <= count <= 10
266	codegen[out_index] = 17;
267	out_index += 1;
268	codegen[out_index] = @as(u8, @intCast(count - 3));
269	out_index += 1;
270	self.codegen_freq[17] += 1;
271	count = 0;
272	}
273	}
274	count -= 1;
275	while (count >= 0) : (count -= 1) {
276	codegen[out_index] = size;
277	out_index += 1;
278	self.codegen_freq[size] += 1;
279	}
280	// Set up invariant for next time through the loop.
281	size = next_size;
282	count = 1;
283	}
284	// Marker indicating the end of the codegen.
285	codegen[out_index] = bad_code;
286	}
287
288	// dynamicSize returns the size of dynamically encoded data in bits.
289	fn dynamicSize(
290	self: *Self,
291	lit_enc: *hm_code.HuffmanEncoder, // literal encoder
292	off_enc: *hm_code.HuffmanEncoder, // offset encoder
293	extra_bits: u32,
294	) DynamicSize {
295	var num_codegens = self.codegen_freq.len;
296	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
297	num_codegens -= 1;
298	}
299	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
300	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
301	self.codegen_freq[16] * 2 +
302	self.codegen_freq[17] * 3 +
303	self.codegen_freq[18] * 7;
304	const size = header +
305	lit_enc.bitLength(self.literal_freq) +
306	off_enc.bitLength(self.offset_freq) +
307	extra_bits;
308
309	return DynamicSize{
310	.size = @as(u32, @intCast(size)),
311	.num_codegens = @as(u32, @intCast(num_codegens)),
312	};
313	}
314
315	// fixedSize returns the size of dynamically encoded data in bits.
316	fn fixedSize(self: *Self, extra_bits: u32) u32 {
317	return 3 +
318	self.fixed_literal_encoding.bitLength(self.literal_freq) +
319	self.fixed_offset_encoding.bitLength(self.offset_freq) +
320	extra_bits;
321	}
322
323	// storedSizeFits calculates the stored size, including header.
324	// The function returns the size in bits and whether the block
325	// fits inside a single block.
326	fn storedSizeFits(in: ?[]const u8) StoredSize {
327	if (in == null) {
328	return .{ .size = 0, .storable = false };
329	}
330	if (in.?.len <= deflate_const.max_store_block_size) {
331	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
332	}
333	return .{ .size = 0, .storable = false };
334	}
335
336	fn writeCode(self: *Self, c: hm_code.HuffCode) Error!void {
337	if (self.err) {
338	return;
339	}
340	self.bits \|= @as(u64, @intCast(c.code)) << @as(u6, @intCast(self.nbits));
341	self.nbits += @as(u32, @intCast(c.len));
342	if (self.nbits >= 48) {
343	const bits = self.bits;
344	self.bits >>= 48;
345	self.nbits -= 48;
346	var n = self.nbytes;
347	var bytes = self.bytes[n..][0..6];
348	bytes[0] = @as(u8, @truncate(bits));
349	bytes[1] = @as(u8, @truncate(bits >> 8));
350	bytes[2] = @as(u8, @truncate(bits >> 16));
351	bytes[3] = @as(u8, @truncate(bits >> 24));
352	bytes[4] = @as(u8, @truncate(bits >> 32));
353	bytes[5] = @as(u8, @truncate(bits >> 40));
354	n += 6;
355	if (n >= buffer_flush_size) {
356	try self.write(self.bytes[0..n]);
357	n = 0;
358	}
359	self.nbytes = n;
360	}
361	}
362
363	// Write the header of a dynamic Huffman block to the output stream.
364	//
365	// num_literals: The number of literals specified in codegen
366	// num_offsets: The number of offsets specified in codegen
367	// num_codegens: The number of codegens used in codegen
368	// is_eof: Is it the end-of-file? (end of stream)
369	fn writeDynamicHeader(
370	self: *Self,
371	num_literals: u32,
372	num_offsets: u32,
373	num_codegens: u32,
374	is_eof: bool,
375	) Error!void {
376	if (self.err) {
377	return;
378	}
379	var first_bits: u32 = 4;
380	if (is_eof) {
381	first_bits = 5;
382	}
383	try self.writeBits(first_bits, 3);
384	try self.writeBits(@as(u32, @intCast(num_literals - 257)), 5);
385	try self.writeBits(@as(u32, @intCast(num_offsets - 1)), 5);
386	try self.writeBits(@as(u32, @intCast(num_codegens - 4)), 4);
387
388	var i: u32 = 0;
389	while (i < num_codegens) : (i += 1) {
390	const value = @as(u32, @intCast(self.codegen_encoding.codes[codegen_order[i]].len));
391	try self.writeBits(@as(u32, @intCast(value)), 3);
392	}
393
394	i = 0;
395	while (true) {
396	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
397	i += 1;
398	if (code_word == bad_code) {
399	break;
400	}
401	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
402
403	switch (code_word) {
404	16 => {
405	try self.writeBits(@as(u32, @intCast(self.codegen[i])), 2);
406	i += 1;
407	},
408	17 => {
409	try self.writeBits(@as(u32, @intCast(self.codegen[i])), 3);
410	i += 1;
411	},
412	18 => {
413	try self.writeBits(@as(u32, @intCast(self.codegen[i])), 7);
414	i += 1;
415	},
416	else => {},
417	}
418	}
419	}
420
421	pub fn writeStoredHeader(self: *Self, length: usize, is_eof: bool) Error!void {
422	if (self.err) {
423	return;
424	}
425	var flag: u32 = 0;
426	if (is_eof) {
427	flag = 1;
428	}
429	try self.writeBits(flag, 3);
430	try self.flush();
431	try self.writeBits(@as(u32, @intCast(length)), 16);
432	try self.writeBits(@as(u32, @intCast(~@as(u16, @intCast(length)))), 16);
433	}
434
435	fn writeFixedHeader(self: *Self, is_eof: bool) Error!void {
436	if (self.err) {
437	return;
438	}
439	// Indicate that we are a fixed Huffman block
440	var value: u32 = 2;
441	if (is_eof) {
442	value = 3;
443	}
444	try self.writeBits(value, 3);
445	}
446
447	// Write a block of tokens with the smallest encoding.
448	// The original input can be supplied, and if the huffman encoded data
449	// is larger than the original bytes, the data will be written as a
450	// stored block.
451	// If the input is null, the tokens will always be Huffman encoded.
452	pub fn writeBlock(
453	self: *Self,
454	tokens: []const token.Token,
455	eof: bool,
456	input: ?[]const u8,
457	) Error!void {
458	if (self.err) {
459	return;
460	}
461
462	const lit_and_off = self.indexTokens(tokens);
463	const num_literals = lit_and_off.num_literals;
464	const num_offsets = lit_and_off.num_offsets;
465
466	var extra_bits: u32 = 0;
467	const ret = storedSizeFits(input);
468	const stored_size = ret.size;
469	const storable = ret.storable;
470
471	if (storable) {
472	// We only bother calculating the costs of the extra bits required by
473	// the length of offset fields (which will be the same for both fixed
474	// and dynamic encoding), if we need to compare those two encodings
475	// against stored encoding.
476	var length_code: u32 = length_codes_start + 8;
477	while (length_code < num_literals) : (length_code += 1) {
478	// First eight length codes have extra size = 0.
479	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
480	@as(u32, @intCast(length_extra_bits[length_code - length_codes_start]));
481	}
482	var offset_code: u32 = 4;
483	while (offset_code < num_offsets) : (offset_code += 1) {
484	// First four offset codes have extra size = 0.
485	extra_bits += @as(u32, @intCast(self.offset_freq[offset_code])) *
486	@as(u32, @intCast(offset_extra_bits[offset_code]));
487	}
488	}
489
490	// Figure out smallest code.
491	// Fixed Huffman baseline.
492	var literal_encoding = &self.fixed_literal_encoding;
493	var offset_encoding = &self.fixed_offset_encoding;
494	var size = self.fixedSize(extra_bits);
495
496	// Dynamic Huffman?
497	var num_codegens: u32 = 0;
498
499	// Generate codegen and codegenFrequencies, which indicates how to encode
500	// the literal_encoding and the offset_encoding.
501	self.generateCodegen(
502	num_literals,
503	num_offsets,
504	&self.literal_encoding,
505	&self.offset_encoding,
506	);
507	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
508	const dynamic_size = self.dynamicSize(
509	&self.literal_encoding,
510	&self.offset_encoding,
511	extra_bits,
512	);
513	const dyn_size = dynamic_size.size;
514	num_codegens = dynamic_size.num_codegens;
515
516	if (dyn_size < size) {
517	size = dyn_size;
518	literal_encoding = &self.literal_encoding;
519	offset_encoding = &self.offset_encoding;
520	}
521
522	// Stored bytes?
523	if (storable and stored_size < size) {
524	try self.writeStoredHeader(input.?.len, eof);
525	try self.writeBytes(input.?);
526	return;
527	}
528
529	// Huffman.
530	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
531	try self.writeFixedHeader(eof);
532	} else {
533	try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
534	}
535
536	// Write the tokens.
537	try self.writeTokens(tokens, literal_encoding.codes, offset_encoding.codes);
538	}
539
540	// writeBlockDynamic encodes a block using a dynamic Huffman table.
541	// This should be used if the symbols used have a disproportionate
542	// histogram distribution.
543	// If input is supplied and the compression savings are below 1/16th of the
544	// input size the block is stored.
545	pub fn writeBlockDynamic(
546	self: *Self,
547	tokens: []const token.Token,
548	eof: bool,
549	input: ?[]const u8,
550	) Error!void {
551	if (self.err) {
552	return;
553	}
554
555	const total_tokens = self.indexTokens(tokens);
556	const num_literals = total_tokens.num_literals;
557	const num_offsets = total_tokens.num_offsets;
558
559	// Generate codegen and codegenFrequencies, which indicates how to encode
560	// the literal_encoding and the offset_encoding.
561	self.generateCodegen(
562	num_literals,
563	num_offsets,
564	&self.literal_encoding,
565	&self.offset_encoding,
566	);
567	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
568	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.offset_encoding, 0);
569	const size = dynamic_size.size;
570	const num_codegens = dynamic_size.num_codegens;
571
572	// Store bytes, if we don't get a reasonable improvement.
573
574	const stored_size = storedSizeFits(input);
575	const ssize = stored_size.size;
576	const storable = stored_size.storable;
577	if (storable and ssize < (size + (size >> 4))) {
578	try self.writeStoredHeader(input.?.len, eof);
579	try self.writeBytes(input.?);
580	return;
581	}
582
583	// Write Huffman table.
584	try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
585
586	// Write the tokens.
587	try self.writeTokens(tokens, self.literal_encoding.codes, self.offset_encoding.codes);
588	}
589
590	const TotalIndexedTokens = struct {
591	num_literals: u32,
592	num_offsets: u32,
593	};
594
595	// Indexes a slice of tokens followed by an end_block_marker, and updates
596	// literal_freq and offset_freq, and generates literal_encoding
597	// and offset_encoding.
598	// The number of literal and offset tokens is returned.
599	fn indexTokens(self: *Self, tokens: []const token.Token) TotalIndexedTokens {
600	var num_literals: u32 = 0;
601	var num_offsets: u32 = 0;
602
603	for (self.literal_freq, 0..) \|_, i\| {
604	self.literal_freq[i] = 0;
605	}
606	for (self.offset_freq, 0..) \|_, i\| {
607	self.offset_freq[i] = 0;
608	}
609
610	for (tokens) \|t\| {
611	if (t < token.match_type) {
612	self.literal_freq[token.literal(t)] += 1;
613	continue;
614	}
615	const length = token.length(t);
616	const offset = token.offset(t);
617	self.literal_freq[length_codes_start + token.lengthCode(length)] += 1;
618	self.offset_freq[token.offsetCode(offset)] += 1;
619	}
620	// add end_block_marker token at the end
621	self.literal_freq[token.literal(deflate_const.end_block_marker)] += 1;
622
623	// get the number of literals
624	num_literals = @as(u32, @intCast(self.literal_freq.len));
625	while (self.literal_freq[num_literals - 1] == 0) {
626	num_literals -= 1;
627	}
628	// get the number of offsets
629	num_offsets = @as(u32, @intCast(self.offset_freq.len));
630	while (num_offsets > 0 and self.offset_freq[num_offsets - 1] == 0) {
631	num_offsets -= 1;
632	}
633	if (num_offsets == 0) {
634	// We haven't found a single match. If we want to go with the dynamic encoding,
635	// we should count at least one offset to be sure that the offset huffman tree could be encoded.
636	self.offset_freq[0] = 1;
637	num_offsets = 1;
638	}
639	self.literal_encoding.generate(self.literal_freq, 15);
640	self.offset_encoding.generate(self.offset_freq, 15);
641	return TotalIndexedTokens{
642	.num_literals = num_literals,
643	.num_offsets = num_offsets,
644	};
645	}
646
647	// Writes a slice of tokens to the output followed by and end_block_marker.
648	// codes for literal and offset encoding must be supplied.
649	fn writeTokens(
650	self: *Self,
651	tokens: []const token.Token,
652	le_codes: []hm_code.HuffCode,
653	oe_codes: []hm_code.HuffCode,
654	) Error!void {
655	if (self.err) {
656	return;
657	}
658	for (tokens) \|t\| {
659	if (t < token.match_type) {
660	try self.writeCode(le_codes[token.literal(t)]);
661	continue;
662	}
663	// Write the length
664	const length = token.length(t);
665	const length_code = token.lengthCode(length);
666	try self.writeCode(le_codes[length_code + length_codes_start]);
667	const extra_length_bits = @as(u32, @intCast(length_extra_bits[length_code]));
668	if (extra_length_bits > 0) {
669	const extra_length = @as(u32, @intCast(length - length_base[length_code]));
670	try self.writeBits(extra_length, extra_length_bits);
671	}
672	// Write the offset
673	const offset = token.offset(t);
674	const offset_code = token.offsetCode(offset);
675	try self.writeCode(oe_codes[offset_code]);
676	const extra_offset_bits = @as(u32, @intCast(offset_extra_bits[offset_code]));
677	if (extra_offset_bits > 0) {
678	const extra_offset = @as(u32, @intCast(offset - offset_base[offset_code]));
679	try self.writeBits(extra_offset, extra_offset_bits);
680	}
681	}
682	// add end_block_marker at the end
683	try self.writeCode(le_codes[token.literal(deflate_const.end_block_marker)]);
684	}
685
686	// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
687	// if the results only gains very little from compression.
688	pub fn writeBlockHuff(self: *Self, eof: bool, input: []const u8) Error!void {
689	if (self.err) {
690	return;
691	}
692
693	// Clear histogram
694	for (self.literal_freq, 0..) \|_, i\| {
695	self.literal_freq[i] = 0;
696	}
697
698	// Add everything as literals
699	histogram(input, &self.literal_freq);
700
701	self.literal_freq[deflate_const.end_block_marker] = 1;
702
703	const num_literals = deflate_const.end_block_marker + 1;
704	self.offset_freq[0] = 1;
705	const num_offsets = 1;
706
707	self.literal_encoding.generate(self.literal_freq, 15);
708
709	// Figure out smallest code.
710	// Always use dynamic Huffman or Store
711	var num_codegens: u32 = 0;
712
713	// Generate codegen and codegenFrequencies, which indicates how to encode
714	// the literal_encoding and the offset_encoding.
715	self.generateCodegen(
716	num_literals,
717	num_offsets,
718	&self.literal_encoding,
719	&self.huff_offset,
720	);
721	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
722	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_offset, 0);
723	const size = dynamic_size.size;
724	num_codegens = dynamic_size.num_codegens;
725
726	// Store bytes, if we don't get a reasonable improvement.
727
728	const stored_size_ret = storedSizeFits(input);
729	const ssize = stored_size_ret.size;
730	const storable = stored_size_ret.storable;
731
732	if (storable and ssize < (size + (size >> 4))) {
733	try self.writeStoredHeader(input.len, eof);
734	try self.writeBytes(input);
735	return;
736	}
737
738	// Huffman.
739	try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
740	const encoding = self.literal_encoding.codes[0..257];
741	var n = self.nbytes;
742	for (input) \|t\| {
743	// Bitwriting inlined, ~30% speedup
744	const c = encoding[t];
745	self.bits \|= @as(u64, @intCast(c.code)) << @as(u6, @intCast(self.nbits));
746	self.nbits += @as(u32, @intCast(c.len));
747	if (self.nbits < 48) {
748	continue;
749	}
750	// Store 6 bytes
751	const bits = self.bits;
752	self.bits >>= 48;
753	self.nbits -= 48;
754	var bytes = self.bytes[n..][0..6];
755	bytes[0] = @as(u8, @truncate(bits));
756	bytes[1] = @as(u8, @truncate(bits >> 8));
757	bytes[2] = @as(u8, @truncate(bits >> 16));
758	bytes[3] = @as(u8, @truncate(bits >> 24));
759	bytes[4] = @as(u8, @truncate(bits >> 32));
760	bytes[5] = @as(u8, @truncate(bits >> 40));
761	n += 6;
762	if (n < buffer_flush_size) {
763	continue;
764	}
765	try self.write(self.bytes[0..n]);
766	if (self.err) {
767	return; // Return early in the event of write failures
768	}
769	n = 0;
770	}
771	self.nbytes = n;
772	try self.writeCode(encoding[deflate_const.end_block_marker]);
773	}
774
775	pub fn deinit(self: *Self) void {
776	self.allocator.free(self.literal_freq);
777	self.allocator.free(self.offset_freq);
778	self.allocator.free(self.codegen);
779	self.literal_encoding.deinit();
780	self.codegen_encoding.deinit();
781	self.offset_encoding.deinit();
782	self.fixed_literal_encoding.deinit();
783	self.fixed_offset_encoding.deinit();
784	self.huff_offset.deinit();
785	}
786	};
787	}
788
789	const DynamicSize = struct {
790	size: u32,
791	num_codegens: u32,
792	};
793
794	const StoredSize = struct {
795	size: u32,
796	storable: bool,
797	};
798
799	pub fn huffmanBitWriter(allocator: Allocator, writer: anytype) !HuffmanBitWriter(@TypeOf(writer)) {
800	var offset_freq = [1]u16{0} ** deflate_const.offset_code_count;
801	offset_freq[0] = 1;
802	// huff_offset is a static offset encoder used for huffman only encoding.
803	// It can be reused since we will not be encoding offset values.
804	var huff_offset = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count);
805	huff_offset.generate(offset_freq[0..], 15);
806
807	return HuffmanBitWriter(@TypeOf(writer)){
808	.inner_writer = writer,
809	.bytes_written = 0,
810	.bits = 0,
811	.nbits = 0,
812	.nbytes = 0,
813	.bytes = [1]u8{0} ** buffer_size,
814	.codegen_freq = [1]u16{0} ** codegen_code_count,
815	.literal_freq = try allocator.alloc(u16, deflate_const.max_num_lit),
816	.offset_freq = try allocator.alloc(u16, deflate_const.offset_code_count),
817	.codegen = try allocator.alloc(u8, deflate_const.max_num_lit + deflate_const.offset_code_count + 1),
818	.literal_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.max_num_lit),
819	.codegen_encoding = try hm_code.newHuffmanEncoder(allocator, codegen_code_count),
820	.offset_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count),
821	.allocator = allocator,
822	.fixed_literal_encoding = try hm_code.generateFixedLiteralEncoding(allocator),
823	.fixed_offset_encoding = try hm_code.generateFixedOffsetEncoding(allocator),
824	.huff_offset = huff_offset,
825	};
826	}
827
828	// histogram accumulates a histogram of b in h.
829	//
830	// h.len must be >= 256, and h's elements must be all zeroes.
831	fn histogram(b: []const u8, h: *[]u16) void {
832	var lh = h.*[0..256];
833	for (b) \|t\| {
834	lh[t] += 1;
835	}
836	}
837
838	// tests
839	const expect = std.testing.expect;
840	const fmt = std.fmt;
841	const math = std.math;
842	const mem = std.mem;
843	const testing = std.testing;
844
845	const ArrayList = std.ArrayList;
846
847	test "writeBlockHuff" {
848	// Tests huffman encoding against reference files to detect possible regressions.
849	// If encoding/bit allocation changes you can regenerate these files
850
851	try testBlockHuff(
852	"huffman-null-max.input",
853	"huffman-null-max.golden",
854	);
855	try testBlockHuff(
856	"huffman-pi.input",
857	"huffman-pi.golden",
858	);
859	try testBlockHuff(
860	"huffman-rand-1k.input",
861	"huffman-rand-1k.golden",
862	);
863	try testBlockHuff(
864	"huffman-rand-limit.input",
865	"huffman-rand-limit.golden",
866	);
867	try testBlockHuff(
868	"huffman-rand-max.input",
869	"huffman-rand-max.golden",
870	);
871	try testBlockHuff(
872	"huffman-shifts.input",
873	"huffman-shifts.golden",
874	);
875	try testBlockHuff(
876	"huffman-text.input",
877	"huffman-text.golden",
878	);
879	try testBlockHuff(
880	"huffman-text-shift.input",
881	"huffman-text-shift.golden",
882	);
883	try testBlockHuff(
884	"huffman-zero.input",
885	"huffman-zero.golden",
886	);
887	}
888
889	fn testBlockHuff(comptime in_name: []const u8, comptime want_name: []const u8) !void {
890	const in: []const u8 = @embedFile("testdata/" ++ in_name);
891	const want: []const u8 = @embedFile("testdata/" ++ want_name);
892
893	var buf = ArrayList(u8).init(testing.allocator);
894	defer buf.deinit();
895	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
896	defer bw.deinit();
897	try bw.writeBlockHuff(false, in);
898	try bw.flush();
899
900	try std.testing.expectEqualSlices(u8, want, buf.items);
901
902	// Test if the writer produces the same output after reset.
903	var buf_after_reset = ArrayList(u8).init(testing.allocator);
904	defer buf_after_reset.deinit();
905
906	bw.reset(buf_after_reset.writer());
907
908	try bw.writeBlockHuff(false, in);
909	try bw.flush();
910
911	try std.testing.expectEqualSlices(u8, buf.items, buf_after_reset.items);
912	try std.testing.expectEqualSlices(u8, want, buf_after_reset.items);
913
914	try testWriterEOF(.write_huffman_block, &[0]token.Token{}, in);
915	}
916
917	const HuffTest = struct {
918	tokens: []const token.Token,
919	input: []const u8 = "", // File name of input data matching the tokens.
920	want: []const u8 = "", // File name of data with the expected output with input available.
921	want_no_input: []const u8 = "", // File name of the expected output when no input is available.
922	};
923
924	const ml = 0x7fc00000; // Maximum length token. Used to reduce the size of writeBlockTests
925
926	const writeBlockTests = &[_]HuffTest{
927	HuffTest{
928	.input = "huffman-null-max.input",
929	.want = "huffman-null-max.{s}.expect",
930	.want_no_input = "huffman-null-max.{s}.expect-noinput",
931	.tokens = &[_]token.Token{
932	0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
933	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
934	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
935	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
936	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
937	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
938	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
939	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
940	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
941	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
942	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
943	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
944	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x0, 0x0,
945	},
946	},
947	HuffTest{
948	.input = "huffman-pi.input",
949	.want = "huffman-pi.{s}.expect",
950	.want_no_input = "huffman-pi.{s}.expect-noinput",
951	.tokens = &[_]token.Token{
952	0x33, 0x2e, 0x31, 0x34, 0x31, 0x35, 0x39,
953	0x32, 0x36, 0x35, 0x33, 0x35, 0x38, 0x39,
954	0x37, 0x39, 0x33, 0x32, 0x33, 0x38, 0x34,
955	0x36, 0x32, 0x36, 0x34, 0x33, 0x33, 0x38,
956	0x33, 0x32, 0x37, 0x39, 0x35, 0x30, 0x32,
957	0x38, 0x38, 0x34, 0x31, 0x39, 0x37, 0x31,
958	0x36, 0x39, 0x33, 0x39, 0x39, 0x33, 0x37,
959	0x35, 0x31, 0x30, 0x35, 0x38, 0x32, 0x30,
960	0x39, 0x37, 0x34, 0x39, 0x34, 0x34, 0x35,
961	0x39, 0x32, 0x33, 0x30, 0x37, 0x38, 0x31,
962	0x36, 0x34, 0x30, 0x36, 0x32, 0x38, 0x36,
963	0x32, 0x30, 0x38, 0x39, 0x39, 0x38, 0x36,
964	0x32, 0x38, 0x30, 0x33, 0x34, 0x38, 0x32,
965	0x35, 0x33, 0x34, 0x32, 0x31, 0x31, 0x37,
966	0x30, 0x36, 0x37, 0x39, 0x38, 0x32, 0x31,
967	0x34, 0x38, 0x30, 0x38, 0x36, 0x35, 0x31,
968	0x33, 0x32, 0x38, 0x32, 0x33, 0x30, 0x36,
969	0x36, 0x34, 0x37, 0x30, 0x39, 0x33, 0x38,
970	0x34, 0x34, 0x36, 0x30, 0x39, 0x35, 0x35,
971	0x30, 0x35, 0x38, 0x32, 0x32, 0x33, 0x31,
972	0x37, 0x32, 0x35, 0x33, 0x35, 0x39, 0x34,
973	0x30, 0x38, 0x31, 0x32, 0x38, 0x34, 0x38,
974	0x31, 0x31, 0x31, 0x37, 0x34, 0x4040007e, 0x34,
975	0x31, 0x30, 0x32, 0x37, 0x30, 0x31, 0x39,
976	0x33, 0x38, 0x35, 0x32, 0x31, 0x31, 0x30,
977	0x35, 0x35, 0x35, 0x39, 0x36, 0x34, 0x34,
978	0x36, 0x32, 0x32, 0x39, 0x34, 0x38, 0x39,
979	0x35, 0x34, 0x39, 0x33, 0x30, 0x33, 0x38,
980	0x31, 0x40400012, 0x32, 0x38, 0x38, 0x31, 0x30,
981	0x39, 0x37, 0x35, 0x36, 0x36, 0x35, 0x39,
982	0x33, 0x33, 0x34, 0x34, 0x36, 0x40400047, 0x37,
983	0x35, 0x36, 0x34, 0x38, 0x32, 0x33, 0x33,
984	0x37, 0x38, 0x36, 0x37, 0x38, 0x33, 0x31,
985	0x36, 0x35, 0x32, 0x37, 0x31, 0x32, 0x30,
986	0x31, 0x39, 0x30, 0x39, 0x31, 0x34, 0x4040001a,
987	0x35, 0x36, 0x36, 0x39, 0x32, 0x33, 0x34,
988	0x36, 0x404000b2, 0x36, 0x31, 0x30, 0x34, 0x35,
989	0x34, 0x33, 0x32, 0x36, 0x40400032, 0x31, 0x33,
990	0x33, 0x39, 0x33, 0x36, 0x30, 0x37, 0x32,
991	0x36, 0x30, 0x32, 0x34, 0x39, 0x31, 0x34,
992	0x31, 0x32, 0x37, 0x33, 0x37, 0x32, 0x34,
993	0x35, 0x38, 0x37, 0x30, 0x30, 0x36, 0x36,
994	0x30, 0x36, 0x33, 0x31, 0x35, 0x35, 0x38,
995	0x38, 0x31, 0x37, 0x34, 0x38, 0x38, 0x31,
996	0x35, 0x32, 0x30, 0x39, 0x32, 0x30, 0x39,
997	0x36, 0x32, 0x38, 0x32, 0x39, 0x32, 0x35,
998	0x34, 0x30, 0x39, 0x31, 0x37, 0x31, 0x35,
999	0x33, 0x36, 0x34, 0x33, 0x36, 0x37, 0x38,
1000	0x39, 0x32, 0x35, 0x39, 0x30, 0x33, 0x36,
1001	0x30, 0x30, 0x31, 0x31, 0x33, 0x33, 0x30,
1002	0x35, 0x33, 0x30, 0x35, 0x34, 0x38, 0x38,
1003	0x32, 0x30, 0x34, 0x36, 0x36, 0x35, 0x32,
1004	0x31, 0x33, 0x38, 0x34, 0x31, 0x34, 0x36,
1005	0x39, 0x35, 0x31, 0x39, 0x34, 0x31, 0x35,
1006	0x31, 0x31, 0x36, 0x30, 0x39, 0x34, 0x33,
1007	0x33, 0x30, 0x35, 0x37, 0x32, 0x37, 0x30,
1008	0x33, 0x36, 0x35, 0x37, 0x35, 0x39, 0x35,
1009	0x39, 0x31, 0x39, 0x35, 0x33, 0x30, 0x39,
1010	0x32, 0x31, 0x38, 0x36, 0x31, 0x31, 0x37,
1011	0x404000e9, 0x33, 0x32, 0x40400009, 0x39, 0x33, 0x31,
1012	0x30, 0x35, 0x31, 0x31, 0x38, 0x35, 0x34,
1013	0x38, 0x30, 0x37, 0x4040010e, 0x33, 0x37, 0x39,
1014	0x39, 0x36, 0x32, 0x37, 0x34, 0x39, 0x35,
1015	0x36, 0x37, 0x33, 0x35, 0x31, 0x38, 0x38,
1016	0x35, 0x37, 0x35, 0x32, 0x37, 0x32, 0x34,
1017	0x38, 0x39, 0x31, 0x32, 0x32, 0x37, 0x39,
1018	0x33, 0x38, 0x31, 0x38, 0x33, 0x30, 0x31,
1019	0x31, 0x39, 0x34, 0x39, 0x31, 0x32, 0x39,
1020	0x38, 0x33, 0x33, 0x36, 0x37, 0x33, 0x33,
1021	0x36, 0x32, 0x34, 0x34, 0x30, 0x36, 0x35,
1022	0x36, 0x36, 0x34, 0x33, 0x30, 0x38, 0x36,
1023	0x30, 0x32, 0x31, 0x33, 0x39, 0x34, 0x39,
1024	0x34, 0x36, 0x33, 0x39, 0x35, 0x32, 0x32,
1025	0x34, 0x37, 0x33, 0x37, 0x31, 0x39, 0x30,
1026	0x37, 0x30, 0x32, 0x31, 0x37, 0x39, 0x38,
1027	0x40800099, 0x37, 0x30, 0x32, 0x37, 0x37, 0x30,
1028	0x35, 0x33, 0x39, 0x32, 0x31, 0x37, 0x31,
1029	0x37, 0x36, 0x32, 0x39, 0x33, 0x31, 0x37,
1030	0x36, 0x37, 0x35, 0x40800232, 0x37, 0x34, 0x38,
1031	0x31, 0x40400006, 0x36, 0x36, 0x39, 0x34, 0x30,
1032	0x404001e7, 0x30, 0x30, 0x30, 0x35, 0x36, 0x38,
1033	0x31, 0x32, 0x37, 0x31, 0x34, 0x35, 0x32,
1034	0x36, 0x33, 0x35, 0x36, 0x30, 0x38, 0x32,
1035	0x37, 0x37, 0x38, 0x35, 0x37, 0x37, 0x31,
1036	0x33, 0x34, 0x32, 0x37, 0x35, 0x37, 0x37,
1037	0x38, 0x39, 0x36, 0x40400129, 0x33, 0x36, 0x33,
1038	0x37, 0x31, 0x37, 0x38, 0x37, 0x32, 0x31,
1039	0x34, 0x36, 0x38, 0x34, 0x34, 0x30, 0x39,
1040	0x30, 0x31, 0x32, 0x32, 0x34, 0x39, 0x35,
1041	0x33, 0x34, 0x33, 0x30, 0x31, 0x34, 0x36,
1042	0x35, 0x34, 0x39, 0x35, 0x38, 0x35, 0x33,
1043	0x37, 0x31, 0x30, 0x35, 0x30, 0x37, 0x39,
1044	0x404000ca, 0x36, 0x40400153, 0x38, 0x39, 0x32, 0x33,
1045	0x35, 0x34, 0x404001c9, 0x39, 0x35, 0x36, 0x31,
1046	0x31, 0x32, 0x31, 0x32, 0x39, 0x30, 0x32,
1047	0x31, 0x39, 0x36, 0x30, 0x38, 0x36, 0x34,
1048	0x30, 0x33, 0x34, 0x34, 0x31, 0x38, 0x31,
1049	0x35, 0x39, 0x38, 0x31, 0x33, 0x36, 0x32,
1050	0x39, 0x37, 0x37, 0x34, 0x40400074, 0x30, 0x39,
1051	0x39, 0x36, 0x30, 0x35, 0x31, 0x38, 0x37,
1052	0x30, 0x37, 0x32, 0x31, 0x31, 0x33, 0x34,
1053	0x39, 0x40800000, 0x38, 0x33, 0x37, 0x32, 0x39,
1054	0x37, 0x38, 0x30, 0x34, 0x39, 0x39, 0x404002da,
1055	0x39, 0x37, 0x33, 0x31, 0x37, 0x33, 0x32,
1056	0x38, 0x4040018a, 0x36, 0x33, 0x31, 0x38, 0x35,
1057	0x40400301, 0x404002e8, 0x34, 0x35, 0x35, 0x33, 0x34,
1058	0x36, 0x39, 0x30, 0x38, 0x33, 0x30, 0x32,
1059	0x36, 0x34, 0x32, 0x35, 0x32, 0x32, 0x33,
1060	0x30, 0x404002e3, 0x40400267, 0x38, 0x35, 0x30, 0x33,
1061	0x35, 0x32, 0x36, 0x31, 0x39, 0x33, 0x31,
1062	0x31, 0x40400212, 0x31, 0x30, 0x31, 0x30, 0x30,
1063	0x30, 0x33, 0x31, 0x33, 0x37, 0x38, 0x33,
1064	0x38, 0x37, 0x35, 0x32, 0x38, 0x38, 0x36,
1065	0x35, 0x38, 0x37, 0x35, 0x33, 0x33, 0x32,
1066	0x30, 0x38, 0x33, 0x38, 0x31, 0x34, 0x32,
1067	0x30, 0x36, 0x40400140, 0x4040012b, 0x31, 0x34, 0x37,
1068	0x33, 0x30, 0x33, 0x35, 0x39, 0x4080032e, 0x39,
1069	0x30, 0x34, 0x32, 0x38, 0x37, 0x35, 0x35,
1070	0x34, 0x36, 0x38, 0x37, 0x33, 0x31, 0x31,
1071	0x35, 0x39, 0x35, 0x40400355, 0x33, 0x38, 0x38,
1072	0x32, 0x33, 0x35, 0x33, 0x37, 0x38, 0x37,
1073	0x35, 0x4080037f, 0x39, 0x4040013a, 0x31, 0x40400148, 0x38,
1074	0x30, 0x35, 0x33, 0x4040018a, 0x32, 0x32, 0x36,
1075	0x38, 0x30, 0x36, 0x36, 0x31, 0x33, 0x30,
1076	0x30, 0x31, 0x39, 0x32, 0x37, 0x38, 0x37,
1077	0x36, 0x36, 0x31, 0x31, 0x31, 0x39, 0x35,
1078	0x39, 0x40400237, 0x36, 0x40800124, 0x38, 0x39, 0x33,
1079	0x38, 0x30, 0x39, 0x35, 0x32, 0x35, 0x37,
1080	0x32, 0x30, 0x31, 0x30, 0x36, 0x35, 0x34,
1081	0x38, 0x35, 0x38, 0x36, 0x33, 0x32, 0x37,
1082	0x4040009a, 0x39, 0x33, 0x36, 0x31, 0x35, 0x33,
1083	0x40400220, 0x4080015c, 0x32, 0x33, 0x30, 0x33, 0x30,
1084	0x31, 0x39, 0x35, 0x32, 0x30, 0x33, 0x35,
1085	0x33, 0x30, 0x31, 0x38, 0x35, 0x32, 0x40400171,
1086	0x40400075, 0x33, 0x36, 0x32, 0x32, 0x35, 0x39,
1087	0x39, 0x34, 0x31, 0x33, 0x40400254, 0x34, 0x39,
1088	0x37, 0x32, 0x31, 0x37, 0x404000de, 0x33, 0x34,
1089	0x37, 0x39, 0x31, 0x33, 0x31, 0x35, 0x31,
1090	0x35, 0x35, 0x37, 0x34, 0x38, 0x35, 0x37,
1091	0x32, 0x34, 0x32, 0x34, 0x35, 0x34, 0x31,
1092	0x35, 0x30, 0x36, 0x39, 0x4040013f, 0x38, 0x32,
1093	0x39, 0x35, 0x33, 0x33, 0x31, 0x31, 0x36,
1094	0x38, 0x36, 0x31, 0x37, 0x32, 0x37, 0x38,
1095	0x40400337, 0x39, 0x30, 0x37, 0x35, 0x30, 0x39,
1096	0x4040010d, 0x37, 0x35, 0x34, 0x36, 0x33, 0x37,
1097	0x34, 0x36, 0x34, 0x39, 0x33, 0x39, 0x33,
1098	0x31, 0x39, 0x32, 0x35, 0x35, 0x30, 0x36,
1099	0x30, 0x34, 0x30, 0x30, 0x39, 0x4040026b, 0x31,
1100	0x36, 0x37, 0x31, 0x31, 0x33, 0x39, 0x30,
1101	0x30, 0x39, 0x38, 0x40400335, 0x34, 0x30, 0x31,
1102	0x32, 0x38, 0x35, 0x38, 0x33, 0x36, 0x31,
1103	0x36, 0x30, 0x33, 0x35, 0x36, 0x33, 0x37,
1104	0x30, 0x37, 0x36, 0x36, 0x30, 0x31, 0x30,
1105	0x34, 0x40400172, 0x38, 0x31, 0x39, 0x34, 0x32,
1106	0x39, 0x4080041e, 0x404000ef, 0x4040028b, 0x37, 0x38, 0x33,
1107	0x37, 0x34, 0x404004a8, 0x38, 0x32, 0x35, 0x35,
1108	0x33, 0x37, 0x40800209, 0x32, 0x36, 0x38, 0x4040002e,
1109	0x34, 0x30, 0x34, 0x37, 0x404001d1, 0x34, 0x404004b5,
1110	0x4040038d, 0x38, 0x34, 0x404003a8, 0x36, 0x40c0031f, 0x33,
1111	0x33, 0x31, 0x33, 0x36, 0x37, 0x37, 0x30,
1112	0x32, 0x38, 0x39, 0x38, 0x39, 0x31, 0x35,
1113	0x32, 0x40400062, 0x35, 0x32, 0x31, 0x36, 0x32,
1114	0x30, 0x35, 0x36, 0x39, 0x36, 0x40400411, 0x30,
1115	0x35, 0x38, 0x40400477, 0x35, 0x40400498, 0x35, 0x31,
1116	0x31, 0x40400209, 0x38, 0x32, 0x34, 0x33, 0x30,
1117	0x30, 0x33, 0x35, 0x35, 0x38, 0x37, 0x36,
1118	0x34, 0x30, 0x32, 0x34, 0x37, 0x34, 0x39,
1119	0x36, 0x34, 0x37, 0x33, 0x32, 0x36, 0x33,
1120	0x4040043e, 0x39, 0x39, 0x32, 0x4040044b, 0x34, 0x32,
1121	0x36, 0x39, 0x40c002c5, 0x37, 0x404001d6, 0x34, 0x4040053d,
1122	0x4040041d, 0x39, 0x33, 0x34, 0x31, 0x37, 0x404001ad,
1123	0x31, 0x32, 0x4040002a, 0x34, 0x4040019e, 0x31, 0x35,
1124	0x30, 0x33, 0x30, 0x32, 0x38, 0x36, 0x31,
1125	0x38, 0x32, 0x39, 0x37, 0x34, 0x35, 0x35,
1126	0x35, 0x37, 0x30, 0x36, 0x37, 0x34, 0x40400135,
1127	0x35, 0x30, 0x35, 0x34, 0x39, 0x34, 0x35,
1128	0x38, 0x404001c5, 0x39, 0x40400051, 0x35, 0x36, 0x404001ec,
1129	0x37, 0x32, 0x31, 0x30, 0x37, 0x39, 0x40400159,
1130	0x33, 0x30, 0x4040010a, 0x33, 0x32, 0x31, 0x31,
1131	0x36, 0x35, 0x33, 0x34, 0x34, 0x39, 0x38,
1132	0x37, 0x32, 0x30, 0x32, 0x37, 0x4040011b, 0x30,
1133	0x32, 0x33, 0x36, 0x34, 0x4040022e, 0x35, 0x34,
1134	0x39, 0x39, 0x31, 0x31, 0x39, 0x38, 0x40400418,
1135	0x34, 0x4040011b, 0x35, 0x33, 0x35, 0x36, 0x36,
1136	0x33, 0x36, 0x39, 0x40400450, 0x32, 0x36, 0x35,
1137	0x404002e4, 0x37, 0x38, 0x36, 0x32, 0x35, 0x35,
1138	0x31, 0x404003da, 0x31, 0x37, 0x35, 0x37, 0x34,
1139	0x36, 0x37, 0x32, 0x38, 0x39, 0x30, 0x39,
1140	0x37, 0x37, 0x37, 0x37, 0x40800453, 0x30, 0x30,
1141	0x30, 0x404005fd, 0x37, 0x30, 0x404004df, 0x36, 0x404003e9,
1142	0x34, 0x39, 0x31, 0x4040041e, 0x40400297, 0x32, 0x31,
1143	0x34, 0x37, 0x37, 0x32, 0x33, 0x35, 0x30,
1144	0x31, 0x34, 0x31, 0x34, 0x40400643, 0x33, 0x35,
1145	0x36, 0x404004af, 0x31, 0x36, 0x31, 0x33, 0x36,
1146	0x31, 0x31, 0x35, 0x37, 0x33, 0x35, 0x32,
1147	0x35, 0x40400504, 0x33, 0x34, 0x4040005b, 0x31, 0x38,
1148	0x4040047b, 0x38, 0x34, 0x404005e7, 0x33, 0x33, 0x32,
1149	0x33, 0x39, 0x30, 0x37, 0x33, 0x39, 0x34,
1150	0x31, 0x34, 0x33, 0x33, 0x33, 0x34, 0x35,
1151	0x34, 0x37, 0x37, 0x36, 0x32, 0x34, 0x40400242,
1152	0x32, 0x35, 0x31, 0x38, 0x39, 0x38, 0x33,
1153	0x35, 0x36, 0x39, 0x34, 0x38, 0x35, 0x35,
1154	0x36, 0x32, 0x30, 0x39, 0x39, 0x32, 0x31,
1155	0x39, 0x32, 0x32, 0x32, 0x31, 0x38, 0x34,
1156	0x32, 0x37, 0x4040023e, 0x32, 0x404000ba, 0x36, 0x38,
1157	0x38, 0x37, 0x36, 0x37, 0x31, 0x37, 0x39,
1158	0x30, 0x40400055, 0x30, 0x40800106, 0x36, 0x36, 0x404003e7,
1159	0x38, 0x38, 0x36, 0x32, 0x37, 0x32, 0x404006dc,
1160	0x31, 0x37, 0x38, 0x36, 0x30, 0x38, 0x35,
1161	0x37, 0x40400073, 0x33, 0x408002fc, 0x37, 0x39, 0x37,
1162	0x36, 0x36, 0x38, 0x31, 0x404002bd, 0x30, 0x30,
1163	0x39, 0x35, 0x33, 0x38, 0x38, 0x40400638, 0x33,
1164	0x404006a5, 0x30, 0x36, 0x38, 0x30, 0x30, 0x36,
1165	0x34, 0x32, 0x32, 0x35, 0x31, 0x32, 0x35,
1166	0x32, 0x4040057b, 0x37, 0x33, 0x39, 0x32, 0x40400297,
1167	0x40400474, 0x34, 0x408006b3, 0x38, 0x36, 0x32, 0x36,
1168	0x39, 0x34, 0x35, 0x404001e5, 0x34, 0x31, 0x39,
1169	0x36, 0x35, 0x32, 0x38, 0x35, 0x30, 0x40400099,
1170	0x4040039c, 0x31, 0x38, 0x36, 0x33, 0x404001be, 0x34,
1171	0x40800154, 0x32, 0x30, 0x33, 0x39, 0x4040058b, 0x34,
1172	0x35, 0x404002bc, 0x32, 0x33, 0x37, 0x4040042c, 0x36,
1173	0x40400510, 0x35, 0x36, 0x40400638, 0x37, 0x31, 0x39,
1174	0x31, 0x37, 0x32, 0x38, 0x40400171, 0x37, 0x36,
1175	0x34, 0x36, 0x35, 0x37, 0x35, 0x37, 0x33,
1176	0x39, 0x40400101, 0x33, 0x38, 0x39, 0x40400748, 0x38,
1177	0x33, 0x32, 0x36, 0x34, 0x35, 0x39, 0x39,
1178	0x35, 0x38, 0x404006a7, 0x30, 0x34, 0x37, 0x38,
1179	0x404001de, 0x40400328, 0x39, 0x4040002d, 0x36, 0x34, 0x30,
1180	0x37, 0x38, 0x39, 0x35, 0x31, 0x4040008e, 0x36,
1181	0x38, 0x33, 0x4040012f, 0x32, 0x35, 0x39, 0x35,
1182	0x37, 0x30, 0x40400468, 0x38, 0x32, 0x32, 0x404002c8,
1183	0x32, 0x4040061b, 0x34, 0x30, 0x37, 0x37, 0x32,
1184	0x36, 0x37, 0x31, 0x39, 0x34, 0x37, 0x38,
1185	0x40400319, 0x38, 0x32, 0x36, 0x30, 0x31, 0x34,
1186	0x37, 0x36, 0x39, 0x39, 0x30, 0x39, 0x404004e8,
1187	0x30, 0x31, 0x33, 0x36, 0x33, 0x39, 0x34,
1188	0x34, 0x33, 0x4040027f, 0x33, 0x30, 0x40400105, 0x32,
1189	0x30, 0x33, 0x34, 0x39, 0x36, 0x32, 0x35,
1190	0x32, 0x34, 0x35, 0x31, 0x37, 0x404003b5, 0x39,
1191	0x36, 0x35, 0x31, 0x34, 0x33, 0x31, 0x34,
1192	0x32, 0x39, 0x38, 0x30, 0x39, 0x31, 0x39,
1193	0x30, 0x36, 0x35, 0x39, 0x32, 0x40400282, 0x37,
1194	0x32, 0x32, 0x31, 0x36, 0x39, 0x36, 0x34,
1195	0x36, 0x40400419, 0x4040007a, 0x35, 0x4040050e, 0x34, 0x40800565,
1196	0x38, 0x40400559, 0x39, 0x37, 0x4040057b, 0x35, 0x34,
1197	0x4040049d, 0x4040023e, 0x37, 0x4040065a, 0x38, 0x34, 0x36,
1198	0x38, 0x31, 0x33, 0x4040008c, 0x36, 0x38, 0x33,
1199	0x38, 0x36, 0x38, 0x39, 0x34, 0x32, 0x37,
1200	0x37, 0x34, 0x31, 0x35, 0x35, 0x39, 0x39,
1201	0x31, 0x38, 0x35, 0x4040005a, 0x32, 0x34, 0x35,
1202	0x39, 0x35, 0x33, 0x39, 0x35, 0x39, 0x34,
1203	0x33, 0x31, 0x404005b7, 0x37, 0x40400012, 0x36, 0x38,
1204	0x30, 0x38, 0x34, 0x35, 0x404002e7, 0x37, 0x33,
1205	0x4040081e, 0x39, 0x35, 0x38, 0x34, 0x38, 0x36,
1206	0x35, 0x33, 0x38, 0x404006e8, 0x36, 0x32, 0x404000f2,
1207	0x36, 0x30, 0x39, 0x404004b6, 0x36, 0x30, 0x38,
1208	0x30, 0x35, 0x31, 0x32, 0x34, 0x33, 0x38,
1209	0x38, 0x34, 0x4040013a, 0x4040000b, 0x34, 0x31, 0x33,
1210	0x4040030f, 0x37, 0x36, 0x32, 0x37, 0x38, 0x40400341,
1211	0x37, 0x31, 0x35, 0x4040059b, 0x33, 0x35, 0x39,
1212	0x39, 0x37, 0x37, 0x30, 0x30, 0x31, 0x32,
1213	0x39, 0x40400472, 0x38, 0x39, 0x34, 0x34, 0x31,
1214	0x40400277, 0x36, 0x38, 0x35, 0x35, 0x4040005f, 0x34,
1215	0x30, 0x36, 0x33, 0x404008e6, 0x32, 0x30, 0x37,
1216	0x32, 0x32, 0x40400158, 0x40800203, 0x34, 0x38, 0x31,
1217	0x35, 0x38, 0x40400205, 0x404001fe, 0x4040027a, 0x40400298, 0x33,
1218	0x39, 0x34, 0x35, 0x32, 0x32, 0x36, 0x37,
1219	0x40c00496, 0x38, 0x4040058a, 0x32, 0x31, 0x404002ea, 0x32,
1220	0x40400387, 0x35, 0x34, 0x36, 0x36, 0x36, 0x4040051b,
1221	0x32, 0x33, 0x39, 0x38, 0x36, 0x34, 0x35,
1222	0x36, 0x404004c4, 0x31, 0x36, 0x33, 0x35, 0x40800253,
1223	0x40400811, 0x37, 0x404008ad, 0x39, 0x38, 0x4040045e, 0x39,
1224	0x33, 0x36, 0x33, 0x34, 0x4040075b, 0x37, 0x34,
1225	0x33, 0x32, 0x34, 0x4040047b, 0x31, 0x35, 0x30,
1226	0x37, 0x36, 0x404004bb, 0x37, 0x39, 0x34, 0x35,
1227	0x31, 0x30, 0x39, 0x4040003e, 0x30, 0x39, 0x34,
1228	0x30, 0x404006a6, 0x38, 0x38, 0x37, 0x39, 0x37,
1229	0x31, 0x30, 0x38, 0x39, 0x33, 0x404008f0, 0x36,
1230	0x39, 0x31, 0x33, 0x36, 0x38, 0x36, 0x37,
1231	0x32, 0x4040025b, 0x404001fe, 0x35, 0x4040053f, 0x40400468, 0x40400801,
1232	0x31, 0x37, 0x39, 0x32, 0x38, 0x36, 0x38,
1233	0x404008cc, 0x38, 0x37, 0x34, 0x37, 0x4080079e, 0x38,
1234	0x32, 0x34, 0x4040097a, 0x38, 0x4040025b, 0x37, 0x31,
1235	0x34, 0x39, 0x30, 0x39, 0x36, 0x37, 0x35,
1236	0x39, 0x38, 0x404006ef, 0x33, 0x36, 0x35, 0x40400134,
1237	0x38, 0x31, 0x4040005c, 0x40400745, 0x40400936, 0x36, 0x38,
1238	0x32, 0x39, 0x4040057e, 0x38, 0x37, 0x32, 0x32,
1239	0x36, 0x35, 0x38, 0x38, 0x30, 0x40400611, 0x35,
1240	0x40400249, 0x34, 0x32, 0x37, 0x30, 0x34, 0x37,
1241	0x37, 0x35, 0x35, 0x4040081e, 0x33, 0x37, 0x39,
1242	0x36, 0x34, 0x31, 0x34, 0x35, 0x31, 0x35,
1243	0x32, 0x404005fd, 0x32, 0x33, 0x34, 0x33, 0x36,
1244	0x34, 0x35, 0x34, 0x404005de, 0x34, 0x34, 0x34,
1245	0x37, 0x39, 0x35, 0x4040003c, 0x40400523, 0x408008e6, 0x34,
1246	0x31, 0x4040052a, 0x33, 0x40400304, 0x35, 0x32, 0x33,
1247	0x31, 0x40800841, 0x31, 0x36, 0x36, 0x31, 0x404008b2,
1248	0x35, 0x39, 0x36, 0x39, 0x35, 0x33, 0x36,
1249	0x32, 0x33, 0x31, 0x34, 0x404005ff, 0x32, 0x34,
1250	0x38, 0x34, 0x39, 0x33, 0x37, 0x31, 0x38,
1251	0x37, 0x31, 0x31, 0x30, 0x31, 0x34, 0x35,
1252	0x37, 0x36, 0x35, 0x34, 0x40400761, 0x30, 0x32,
1253	0x37, 0x39, 0x39, 0x33, 0x34, 0x34, 0x30,
1254	0x33, 0x37, 0x34, 0x32, 0x30, 0x30, 0x37,
1255	0x4040093f, 0x37, 0x38, 0x35, 0x33, 0x39, 0x30,
1256	0x36, 0x32, 0x31, 0x39, 0x40800299, 0x40400345, 0x38,
1257	0x34, 0x37, 0x408003d2, 0x38, 0x33, 0x33, 0x32,
1258	0x31, 0x34, 0x34, 0x35, 0x37, 0x31, 0x40400284,
1259	0x40400776, 0x34, 0x33, 0x35, 0x30, 0x40400928, 0x40400468,
1260	0x35, 0x33, 0x31, 0x39, 0x31, 0x30, 0x34,
1261	0x38, 0x34, 0x38, 0x31, 0x30, 0x30, 0x35,
1262	0x33, 0x37, 0x30, 0x36, 0x404008bc, 0x4080059d, 0x40800781,
1263	0x31, 0x40400559, 0x37, 0x4040031b, 0x35, 0x404007ec, 0x4040040c,
1264	0x36, 0x33, 0x408007dc, 0x34, 0x40400971, 0x4080034e, 0x408003f5,
1265	0x38, 0x4080052d, 0x40800887, 0x39, 0x40400187, 0x39, 0x31,
1266	0x404008ce, 0x38, 0x31, 0x34, 0x36, 0x37, 0x35,
1267	0x31, 0x4040062b, 0x31, 0x32, 0x33, 0x39, 0x40c001a9,
1268	0x39, 0x30, 0x37, 0x31, 0x38, 0x36, 0x34,
1269	0x39, 0x34, 0x32, 0x33, 0x31, 0x39, 0x36,
1270	0x31, 0x35, 0x36, 0x404001ec, 0x404006bc, 0x39, 0x35,
1271	0x40400926, 0x40400469, 0x4040011b, 0x36, 0x30, 0x33, 0x38,
1272	0x40400a25, 0x4040016f, 0x40400384, 0x36, 0x32, 0x4040045a, 0x35,
1273	0x4040084c, 0x36, 0x33, 0x38, 0x39, 0x33, 0x37,
1274	0x37, 0x38, 0x37, 0x404008c5, 0x404000f8, 0x39, 0x37,
1275	0x39, 0x32, 0x30, 0x37, 0x37, 0x33, 0x404005d7,
1276	0x32, 0x31, 0x38, 0x32, 0x35, 0x36, 0x404007df,
1277	0x36, 0x36, 0x404006d6, 0x34, 0x32, 0x4080067e, 0x36,
1278	0x404006e6, 0x34, 0x34, 0x40400024, 0x35, 0x34, 0x39,
1279	0x32, 0x30, 0x32, 0x36, 0x30, 0x35, 0x40400ab3,
1280	0x408003e4, 0x32, 0x30, 0x31, 0x34, 0x39, 0x404004d2,
1281	0x38, 0x35, 0x30, 0x37, 0x33, 0x40400599, 0x36,
1282	0x36, 0x36, 0x30, 0x40400194, 0x32, 0x34, 0x33,
1283	0x34, 0x30, 0x40400087, 0x30, 0x4040076b, 0x38, 0x36,
1284	0x33, 0x40400956, 0x404007e4, 0x4040042b, 0x40400174, 0x35, 0x37,
1285	0x39, 0x36, 0x32, 0x36, 0x38, 0x35, 0x36,
1286	0x40400140, 0x35, 0x30, 0x38, 0x40400523, 0x35, 0x38,
1287	0x37, 0x39, 0x36, 0x39, 0x39, 0x40400711, 0x35,
1288	0x37, 0x34, 0x40400a18, 0x38, 0x34, 0x30, 0x404008b3,
1289	0x31, 0x34, 0x35, 0x39, 0x31, 0x4040078c, 0x37,
1290	0x30, 0x40400234, 0x30, 0x31, 0x40400be7, 0x31, 0x32,
1291	0x40400c74, 0x30, 0x404003c3, 0x33, 0x39, 0x40400b2a, 0x40400112,
1292	0x37, 0x31, 0x35, 0x404003b0, 0x34, 0x32, 0x30,
1293	0x40800bf2, 0x39, 0x40400bc2, 0x30, 0x37, 0x40400341, 0x40400795,
1294	0x40400aaf, 0x40400c62, 0x32, 0x31, 0x40400960, 0x32, 0x35,
1295	0x31, 0x4040057b, 0x40400944, 0x39, 0x32, 0x404001b2, 0x38,
1296	0x32, 0x36, 0x40400b66, 0x32, 0x40400278, 0x33, 0x32,
1297	0x31, 0x35, 0x37, 0x39, 0x31, 0x39, 0x38,
1298	0x34, 0x31, 0x34, 0x4080087b, 0x39, 0x31, 0x36,
1299	0x34, 0x408006e8, 0x39, 0x40800b58, 0x404008db, 0x37, 0x32,
1300	0x32, 0x40400321, 0x35, 0x404008a4, 0x40400141, 0x39, 0x31,
1301	0x30, 0x404000bc, 0x40400c5b, 0x35, 0x32, 0x38, 0x30,
1302	0x31, 0x37, 0x40400231, 0x37, 0x31, 0x32, 0x40400914,
1303	0x38, 0x33, 0x32, 0x40400373, 0x31, 0x40400589, 0x30,
1304	0x39, 0x33, 0x35, 0x33, 0x39, 0x36, 0x35,
1305	0x37, 0x4040064b, 0x31, 0x30, 0x38, 0x33, 0x40400069,
1306	0x35, 0x31, 0x4040077a, 0x40400d5a, 0x31, 0x34, 0x34,
1307	0x34, 0x32, 0x31, 0x30, 0x30, 0x40400202, 0x30,
1308	0x33, 0x4040019c, 0x31, 0x31, 0x30, 0x33, 0x40400c81,
1309	0x40400009, 0x40400026, 0x40c00602, 0x35, 0x31, 0x36, 0x404005d9,
1310	0x40800883, 0x4040092a, 0x35, 0x40800c42, 0x38, 0x35, 0x31,
1311	0x37, 0x31, 0x34, 0x33, 0x37, 0x40400605, 0x4040006d,
1312	0x31, 0x35, 0x35, 0x36, 0x35, 0x30, 0x38,
1313	0x38, 0x404003b9, 0x39, 0x38, 0x39, 0x38, 0x35,
1314	0x39, 0x39, 0x38, 0x32, 0x33, 0x38, 0x404001cf,
1315	0x404009ba, 0x33, 0x4040016c, 0x4040043e, 0x404009c3, 0x38, 0x40800e05,
1316	0x33, 0x32, 0x40400107, 0x35, 0x40400305, 0x33, 0x404001ca,
1317	0x39, 0x4040041b, 0x39, 0x38, 0x4040087d, 0x34, 0x40400cb8,
1318	0x37, 0x4040064b, 0x30, 0x37, 0x404000e5, 0x34, 0x38,
1319	0x31, 0x34, 0x31, 0x40400539, 0x38, 0x35, 0x39,
1320	0x34, 0x36, 0x31, 0x40400bc9, 0x38, 0x30,
1321	},
1322	},
1323	HuffTest{
1324	.input = "huffman-rand-1k.input",
1325	.want = "huffman-rand-1k.{s}.expect",
1326	.want_no_input = "huffman-rand-1k.{s}.expect-noinput",
1327	.tokens = &[_]token.Token{
1328	0xf8, 0x8b, 0x96, 0x76, 0x48, 0xd, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d,
1329	0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b,
1330	0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4,
1331	0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde,
1332	0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14,
1333	0x13, 0x6f, 0x23, 0xa9, 0xd, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c,
1334	0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5,
1335	0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20,
1336	0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4,
1337	0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28,
1338	0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0,
1339	0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda,
1340	0xea, 0x9b, 0xa, 0x64, 0xb, 0xe0, 0x23, 0x29, 0xbd, 0xf7, 0xe7, 0x83, 0x3c, 0xfb,
1341	0xdf, 0xb3, 0xae, 0x4f, 0xa4, 0x47, 0x55, 0x99, 0xde, 0x2f, 0x96, 0x6e, 0x1c, 0x43,
1342	0x4c, 0x87, 0xe2, 0x7c, 0xd9, 0x5f, 0x4c, 0x7c, 0xe8, 0x90, 0x3, 0xdb, 0x30, 0x95,
1343	0xd6, 0x22, 0xc, 0x47, 0xb8, 0x4d, 0x6b, 0xbd, 0x24, 0x11, 0xab, 0x2c, 0xd7, 0xbe,
1344	0x6e, 0x7a, 0xd6, 0x8, 0xa3, 0x98, 0xd8, 0xdd, 0x15, 0x6a, 0xfa, 0x93, 0x30, 0x1,
1345	0x25, 0x1d, 0xa2, 0x74, 0x86, 0x4b, 0x6a, 0x95, 0xe8, 0xe1, 0x4e, 0xe, 0x76, 0xb9,
1346	0x49, 0xa9, 0x5f, 0xa0, 0xa6, 0x63, 0x3c, 0x7e, 0x7e, 0x20, 0x13, 0x4f, 0xbb, 0x66,
1347	0x92, 0xb8, 0x2e, 0xa4, 0xfa, 0x48, 0xcb, 0xae, 0xb9, 0x3c, 0xaf, 0xd3, 0x1f, 0xe1,
1348	0xd5, 0x8d, 0x42, 0x6d, 0xf0, 0xfc, 0x8c, 0xc, 0x0, 0xde, 0x40, 0xab, 0x8b, 0x47,
1349	0x97, 0x4e, 0xa8, 0xcf, 0x8e, 0xdb, 0xa6, 0x8b, 0x20, 0x9, 0x84, 0x7a, 0x66, 0xe5,
1350	0x98, 0x29, 0x2, 0x95, 0xe6, 0x38, 0x32, 0x60, 0x3, 0xe3, 0x9a, 0x1e, 0x54, 0xe8,
1351	0x63, 0x80, 0x48, 0x9c, 0xe7, 0x63, 0x33, 0x6e, 0xa0, 0x65, 0x83, 0xfa, 0xc6, 0xba,
1352	0x7a, 0x43, 0x71, 0x5, 0xf5, 0x68, 0x69, 0x85, 0x9c, 0xba, 0x45, 0xcd, 0x6b, 0xb,
1353	0x19, 0xd1, 0xbb, 0x7f, 0x70, 0x85, 0x92, 0xd1, 0xb4, 0x64, 0x82, 0xb1, 0xe4, 0x62,
1354	0xc5, 0x3c, 0x46, 0x1f, 0x92, 0x31, 0x1c, 0x4e, 0x41, 0x77, 0xf7, 0xe7, 0x87, 0xa2,
1355	0xf, 0x6e, 0xe8, 0x92, 0x3, 0x6b, 0xa, 0xe7, 0xa9, 0x3b, 0x11, 0xda, 0x66, 0x8a,
1356	0x29, 0xda, 0x79, 0xe1, 0x64, 0x8d, 0xe3, 0x54, 0xd4, 0xf5, 0xef, 0x64, 0x87, 0x3b,
1357	0xf4, 0xc2, 0xf4, 0x71, 0x13, 0xa9, 0xe9, 0xe0, 0xa2, 0x6, 0x14, 0xab, 0x5d, 0xa7,
1358	0x96, 0x0, 0xd6, 0xc3, 0xcc, 0x57, 0xed, 0x39, 0x6a, 0x25, 0xcd, 0x76, 0xea, 0xba,
1359	0x3a, 0xf2, 0xa1, 0x95, 0x5d, 0xe5, 0x71, 0xcf, 0x9c, 0x62, 0x9e, 0x6a, 0xfa, 0xd5,
1360	0x31, 0xd1, 0xa8, 0x66, 0x30, 0x33, 0xaa, 0x51, 0x17, 0x13, 0x82, 0x99, 0xc8, 0x14,
1361	0x60, 0x9f, 0x4d, 0x32, 0x6d, 0xda, 0x19, 0x26, 0x21, 0xdc, 0x7e, 0x2e, 0x25, 0x67,
1362	0x72, 0xca, 0xf, 0x92, 0xcd, 0xf6, 0xd6, 0xcb, 0x97, 0x8a, 0x33, 0x58, 0x73, 0x70,
1363	0x91, 0x1d, 0xbf, 0x28, 0x23, 0xa3, 0xc, 0xf1, 0x83, 0xc3, 0xc8, 0x56, 0x77, 0x68,
1364	0xe3, 0x82, 0xba, 0xb9, 0x57, 0x56, 0x57, 0x9c, 0xc3, 0xd6, 0x14, 0x5, 0x3c, 0xb1,
1365	0xaf, 0x93, 0xc8, 0x8a, 0x57, 0x7f, 0x53, 0xfa, 0x2f, 0xaa, 0x6e, 0x66, 0x83, 0xfa,
1366	0x33, 0xd1, 0x21, 0xab, 0x1b, 0x71, 0xb4, 0x7c, 0xda, 0xfd, 0xfb, 0x7f, 0x20, 0xab,
1367	0x5e, 0xd5, 0xca, 0xfd, 0xdd, 0xe0, 0xee, 0xda, 0xba, 0xa8, 0x27, 0x99, 0x97, 0x69,
1368	0xc1, 0x3c, 0x82, 0x8c, 0xa, 0x5c, 0x2d, 0x5b, 0x88, 0x3e, 0x34, 0x35, 0x86, 0x37,
1369	0x46, 0x79, 0xe1, 0xaa, 0x19, 0xfb, 0xaa, 0xde, 0x15, 0x9, 0xd, 0x1a, 0x57, 0xff,
1370	0xb5, 0xf, 0xf3, 0x2b, 0x5a, 0x6a, 0x4d, 0x19, 0x77, 0x71, 0x45, 0xdf, 0x4f, 0xb3,
1371	0xec, 0xf1, 0xeb, 0x18, 0x53, 0x3e, 0x3b, 0x47, 0x8, 0x9a, 0x73, 0xa0, 0x5c, 0x8c,
1372	0x5f, 0xeb, 0xf, 0x3a, 0xc2, 0x43, 0x67, 0xb4, 0x66, 0x67, 0x80, 0x58, 0xe, 0xc1,
1373	0xec, 0x40, 0xd4, 0x22, 0x94, 0xca, 0xf9, 0xe8, 0x92, 0xe4, 0x69, 0x38, 0xbe, 0x67,
1374	0x64, 0xca, 0x50, 0xc7, 0x6, 0x67, 0x42, 0x6e, 0xa3, 0xf0, 0xb7, 0x6c, 0xf2, 0xe8,
1375	0x5f, 0xb1, 0xaf, 0xe7, 0xdb, 0xbb, 0x77, 0xb5, 0xf8, 0xcb, 0x8, 0xc4, 0x75, 0x7e,
1376	0xc0, 0xf9, 0x1c, 0x7f, 0x3c, 0x89, 0x2f, 0xd2, 0x58, 0x3a, 0xe2, 0xf8, 0x91, 0xb6,
1377	0x7b, 0x24, 0x27, 0xe9, 0xae, 0x84, 0x8b, 0xde, 0x74, 0xac, 0xfd, 0xd9, 0xb7, 0x69,
1378	0x2a, 0xec, 0x32, 0x6f, 0xf0, 0x92, 0x84, 0xf1, 0x40, 0xc, 0x8a, 0xbc, 0x39, 0x6e,
1379	0x2e, 0x73, 0xd4, 0x6e, 0x8a, 0x74, 0x2a, 0xdc, 0x60, 0x1f, 0xa3, 0x7, 0xde, 0x75,
1380	0x8b, 0x74, 0xc8, 0xfe, 0x63, 0x75, 0xf6, 0x3d, 0x63, 0xac, 0x33, 0x89, 0xc3, 0xf0,
1381	0xf8, 0x2d, 0x6b, 0xb4, 0x9e, 0x74, 0x8b, 0x5c, 0x33, 0xb4, 0xca, 0xa8, 0xe4, 0x99,
1382	0xb6, 0x90, 0xa1, 0xef, 0xf, 0xd3, 0x61, 0xb2, 0xc6, 0x1a, 0x94, 0x7c, 0x44, 0x55,
1383	0xf4, 0x45, 0xff, 0x9e, 0xa5, 0x5a, 0xc6, 0xa0, 0xe8, 0x2a, 0xc1, 0x8d, 0x6f, 0x34,
1384	0x11, 0xb9, 0xbe, 0x4e, 0xd9, 0x87, 0x97, 0x73, 0xcf, 0x3d, 0x23, 0xae, 0xd5, 0x1a,
1385	0x5e, 0xae, 0x5d, 0x6a, 0x3, 0xf9, 0x22, 0xd, 0x10, 0xd9, 0x47, 0x69, 0x15, 0x3f,
1386	0xee, 0x52, 0xa3, 0x8, 0xd2, 0x3c, 0x51, 0xf4, 0xf8, 0x9d, 0xe4, 0x98, 0x89, 0xc8,
1387	0x67, 0x39, 0xd5, 0x5e, 0x35, 0x78, 0x27, 0xe8, 0x3c, 0x80, 0xae, 0x79, 0x71, 0xd2,
1388	0x93, 0xf4, 0xaa, 0x51, 0x12, 0x1c, 0x4b, 0x1b, 0xe5, 0x6e, 0x15, 0x6f, 0xe4, 0xbb,
1389	0x51, 0x9b, 0x45, 0x9f, 0xf9, 0xc4, 0x8c, 0x2a, 0xfb, 0x1a, 0xdf, 0x55, 0xd3, 0x48,
1390	0x93, 0x27, 0x1, 0x26, 0xc2, 0x6b, 0x55, 0x6d, 0xa2, 0xfb, 0x84, 0x8b, 0xc9, 0x9e,
1391	0x28, 0xc2, 0xef, 0x1a, 0x24, 0xec, 0x9b, 0xae, 0xbd, 0x60, 0xe9, 0x15, 0x35, 0xee,
1392	0x42, 0xa4, 0x33, 0x5b, 0xfa, 0xf, 0xb6, 0xf7, 0x1, 0xa6, 0x2, 0x4c, 0xca, 0x90,
1393	0x58, 0x3a, 0x96, 0x41, 0xe7, 0xcb, 0x9, 0x8c, 0xdb, 0x85, 0x4d, 0xa8, 0x89, 0xf3,
1394	0xb5, 0x8e, 0xfd, 0x75, 0x5b, 0x4f, 0xed, 0xde, 0x3f, 0xeb, 0x38, 0xa3, 0xbe, 0xb0,
1395	0x73, 0xfc, 0xb8, 0x54, 0xf7, 0x4c, 0x30, 0x67, 0x2e, 0x38, 0xa2, 0x54, 0x18, 0xba,
1396	0x8, 0xbf, 0xf2, 0x39, 0xd5, 0xfe, 0xa5, 0x41, 0xc6, 0x66, 0x66, 0xba, 0x81, 0xef,
1397	0x67, 0xe4, 0xe6, 0x3c, 0xc, 0xca, 0xa4, 0xa, 0x79, 0xb3, 0x57, 0x8b, 0x8a, 0x75,
1398	0x98, 0x18, 0x42, 0x2f, 0x29, 0xa3, 0x82, 0xef, 0x9f, 0x86, 0x6, 0x23, 0xe1, 0x75,
1399	0xfa, 0x8, 0xb1, 0xde, 0x17, 0x4a,
1400	},
1401	},
1402	HuffTest{
1403	.input = "huffman-rand-limit.input",
1404	.want = "huffman-rand-limit.{s}.expect",
1405	.want_no_input = "huffman-rand-limit.{s}.expect-noinput",
1406	.tokens = &[_]token.Token{
1407	0x61, 0x51c00000, 0xa, 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xa, 0x85, 0x94, 0x25, 0x80,
1408	0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde,
1409	0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73,
1410	0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15,
1411	0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9,
1412	0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xa,
1413	0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70,
1414	0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59,
1415	0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46,
1416	0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b,
1417	0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28,
1418	0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f,
1419	0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4,
1420	0x75, 0xda, 0xea, 0x9b, 0xa,
1421	},
1422	},
1423	HuffTest{
1424	.input = "huffman-shifts.input",
1425	.want = "huffman-shifts.{s}.expect",
1426	.want_no_input = "huffman-shifts.{s}.expect-noinput",
1427	.tokens = &[_]token.Token{
1428	0x31, 0x30, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1429	0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1430	0x7fc00001, 0x7fc00001, 0x7fc00001, 0x52400001, 0xd, 0xa, 0x32,
1431	0x33, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1432	0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7f400001,
1433	},
1434	},
1435	HuffTest{
1436	.input = "huffman-text-shift.input",
1437	.want = "huffman-text-shift.{s}.expect",
1438	.want_no_input = "huffman-text-shift.{s}.expect-noinput",
1439	.tokens = &[_]token.Token{
1440	0x2f, 0x2f, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68,
1441	0x74, 0x32, 0x30, 0x30, 0x39, 0x54, 0x68, 0x47, 0x6f, 0x41,
1442	0x75, 0x74, 0x68, 0x6f, 0x72, 0x2e, 0x41, 0x6c, 0x6c, 0x40800016,
1443	0x72, 0x72, 0x76, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x55,
1444	0x6f, 0x66, 0x74, 0x68, 0x69, 0x6f, 0x75, 0x72, 0x63, 0x63,
1445	0x6f, 0x64, 0x69, 0x67, 0x6f, 0x76, 0x72, 0x6e, 0x64, 0x62,
1446	0x79, 0x42, 0x53, 0x44, 0x2d, 0x74, 0x79, 0x6c, 0x40400020, 0x6c,
1447	0x69, 0x63, 0x6e, 0x74, 0x68, 0x74, 0x63, 0x6e, 0x62, 0x66,
1448	0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x74, 0x68, 0x4c, 0x49,
1449	0x43, 0x45, 0x4e, 0x53, 0x45, 0x66, 0x69, 0x6c, 0x2e, 0xd,
1450	0xa, 0xd, 0xa, 0x70, 0x63, 0x6b, 0x67, 0x6d, 0x69, 0x6e,
1451	0x4040000a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x6f, 0x22,
1452	0x4040000c, 0x66, 0x75, 0x6e, 0x63, 0x6d, 0x69, 0x6e, 0x28, 0x29,
1453	0x7b, 0xd, 0xa, 0x9, 0x76, 0x72, 0x62, 0x3d, 0x6d, 0x6b,
1454	0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x2c, 0x36, 0x35, 0x35,
1455	0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x5f, 0x3a,
1456	0x3d, 0x6f, 0x2e, 0x43, 0x72, 0x74, 0x28, 0x22, 0x68, 0x75,
1457	0x66, 0x66, 0x6d, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d,
1458	0x6d, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x40800021, 0x2e, 0x57, 0x72,
1459	0x69, 0x74, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa,
1460	0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a,
1461	0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54,
1462	0x55, 0x56, 0x58, 0x78, 0x79, 0x7a, 0x21, 0x22, 0x23, 0xc2,
1463	0xa4, 0x25, 0x26, 0x2f, 0x3f, 0x22,
1464	},
1465	},
1466	HuffTest{
1467	.input = "huffman-text.input",
1468	.want = "huffman-text.{s}.expect",
1469	.want_no_input = "huffman-text.{s}.expect-noinput",
1470	.tokens = &[_]token.Token{
1471	0x2f, 0x2f, 0x20, 0x7a, 0x69, 0x67, 0x20, 0x76,
1472	0x30, 0x2e, 0x31, 0x30, 0x2e, 0x30, 0x0a, 0x2f,
1473	0x2f, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65,
1474	0x20, 0x61, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x40400004,
1475	0x6c, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68,
1476	0x20, 0x30, 0x78, 0x30, 0x30, 0x0a, 0x63, 0x6f,
1477	0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, 0x64, 0x20,
1478	0x3d, 0x20, 0x40, 0x69, 0x6d, 0x70, 0x6f, 0x72,
1479	0x74, 0x28, 0x22, 0x73, 0x74, 0x64, 0x22, 0x29,
1480	0x3b, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x20, 0x66,
1481	0x6e, 0x20, 0x6d, 0x61, 0x69, 0x6e, 0x28, 0x29,
1482	0x20, 0x21, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x7b,
1483	0x0a, 0x20, 0x20, 0x20, 0x20, 0x76, 0x61, 0x72,
1484	0x20, 0x62, 0x20, 0x3d, 0x20, 0x5b, 0x31, 0x5d,
1485	0x75, 0x38, 0x7b, 0x30, 0x7d, 0x20, 0x2a, 0x2a,
1486	0x20, 0x36, 0x35, 0x35, 0x33, 0x35, 0x3b, 0x4080001e,
1487	0x40c00055, 0x66, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x79,
1488	0x4040005d, 0x2e, 0x66, 0x73, 0x2e, 0x63, 0x77, 0x64,
1489	0x28, 0x29, 0x2e, 0x40c0008f, 0x46, 0x69, 0x6c, 0x65,
1490	0x28, 0x4080002a, 0x40400000, 0x22, 0x68, 0x75, 0x66, 0x66,
1491	0x6d, 0x61, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c,
1492	0x2d, 0x6d, 0x61, 0x78, 0x2e, 0x69, 0x6e, 0x22,
1493	0x2c, 0x4180001e, 0x2e, 0x7b, 0x20, 0x2e, 0x72, 0x65,
1494	0x61, 0x64, 0x4080004e, 0x75, 0x65, 0x20, 0x7d, 0x40c0001a,
1495	0x29, 0x40c0006b, 0x64, 0x65, 0x66, 0x65, 0x72, 0x20,
1496	0x66, 0x2e, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x28,
1497	0x404000b6, 0x40400015, 0x5f, 0x4100007b, 0x66, 0x2e, 0x77, 0x72,
1498	0x69, 0x74, 0x65, 0x41, 0x6c, 0x6c, 0x28, 0x62,
1499	0x5b, 0x30, 0x2e, 0x2e, 0x5d, 0x29, 0x3b, 0x0a,
1500	0x7d, 0x0a,
1501	},
1502	},
1503	HuffTest{
1504	.input = "huffman-zero.input",
1505	.want = "huffman-zero.{s}.expect",
1506	.want_no_input = "huffman-zero.{s}.expect-noinput",
1507	.tokens = &[_]token.Token{ 0x30, ml, 0x4b800000 },
1508	},
1509	HuffTest{
1510	.input = "",
1511	.want = "",
1512	.want_no_input = "null-long-match.{s}.expect-noinput",
1513	.tokens = &[_]token.Token{
1514	0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1515	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1516	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1517	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1518	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1519	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1520	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1521	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1522	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1523	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1524	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1525	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1526	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1527	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1528	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1529	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1530	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1531	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1532	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1533	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1534	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1535	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1536	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1537	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1538	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1539	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1540	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1541	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1542	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1543	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1544	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1545	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1546	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1547	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1548	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1549	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1550	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1551	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1552	ml, ml, ml, 0x41400000,
1553	},
1554	},
1555	};
1556
1557	const TestType = enum {
1558	write_block,
1559	write_dyn_block, // write dynamic block
1560	write_huffman_block,
1561
1562	fn to_s(self: TestType) []const u8 {
1563	return switch (self) {
1564	.write_block => "wb",
1565	.write_dyn_block => "dyn",
1566	.write_huffman_block => "huff",
1567	};
1568	}
1569	};
1570
1571	test "writeBlock" {
1572	// tests if the writeBlock encoding has changed.
1573
1574	const ttype: TestType = .write_block;
1575	try testBlock(writeBlockTests[0], ttype);
1576	try testBlock(writeBlockTests[1], ttype);
1577	try testBlock(writeBlockTests[2], ttype);
1578	try testBlock(writeBlockTests[3], ttype);
1579	try testBlock(writeBlockTests[4], ttype);
1580	try testBlock(writeBlockTests[5], ttype);
1581	try testBlock(writeBlockTests[6], ttype);
1582	try testBlock(writeBlockTests[7], ttype);
1583	try testBlock(writeBlockTests[8], ttype);
1584	}
1585
1586	test "writeBlockDynamic" {
1587	// tests if the writeBlockDynamic encoding has changed.
1588
1589	const ttype: TestType = .write_dyn_block;
1590	try testBlock(writeBlockTests[0], ttype);
1591	try testBlock(writeBlockTests[1], ttype);
1592	try testBlock(writeBlockTests[2], ttype);
1593	try testBlock(writeBlockTests[3], ttype);
1594	try testBlock(writeBlockTests[4], ttype);
1595	try testBlock(writeBlockTests[5], ttype);
1596	try testBlock(writeBlockTests[6], ttype);
1597	try testBlock(writeBlockTests[7], ttype);
1598	try testBlock(writeBlockTests[8], ttype);
1599	}
1600
1601	// testBlock tests a block against its references,
1602	// or regenerate the references, if "-update" flag is set.
1603	fn testBlock(comptime ht: HuffTest, comptime ttype: TestType) !void {
1604	if (ht.input.len != 0 and ht.want.len != 0) {
1605	const want_name = comptime fmt.comptimePrint(ht.want, .{ttype.to_s()});
1606	const input = @embedFile("testdata/" ++ ht.input);
1607	const want = @embedFile("testdata/" ++ want_name);
1608
1609	var buf = ArrayList(u8).init(testing.allocator);
1610	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1611	try writeToType(ttype, &bw, ht.tokens, input);
1612
1613	var got = buf.items;
1614	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
1615
1616	// Test if the writer produces the same output after reset.
1617	buf.deinit();
1618	buf = ArrayList(u8).init(testing.allocator);
1619	defer buf.deinit();
1620
1621	bw.reset(buf.writer());
1622	defer bw.deinit();
1623
1624	try writeToType(ttype, &bw, ht.tokens, input);
1625	try bw.flush();
1626	got = buf.items;
1627	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
1628	try testWriterEOF(.write_block, ht.tokens, input);
1629	}
1630
1631	const want_name_no_input = comptime fmt.comptimePrint(ht.want_no_input, .{ttype.to_s()});
1632	const want_ni = @embedFile("testdata/" ++ want_name_no_input);
1633
1634	var buf = ArrayList(u8).init(testing.allocator);
1635	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1636
1637	try writeToType(ttype, &bw, ht.tokens, null);
1638
1639	var got = buf.items;
1640	try testing.expectEqualSlices(u8, want_ni, got); // expect writeBlock to yield expected result
1641	try expect(got[0] & 1 != 1); // expect no EOF
1642
1643	// Test if the writer produces the same output after reset.
1644	buf.deinit();
1645	buf = ArrayList(u8).init(testing.allocator);
1646	defer buf.deinit();
1647
1648	bw.reset(buf.writer());
1649	defer bw.deinit();
1650
1651	try writeToType(ttype, &bw, ht.tokens, null);
1652	try bw.flush();
1653	got = buf.items;
1654
1655	try testing.expectEqualSlices(u8, want_ni, got); // expect writeBlock to yield expected result
1656	try testWriterEOF(.write_block, ht.tokens, &[0]u8{});
1657	}
1658
1659	fn writeToType(ttype: TestType, bw: anytype, tok: []const token.Token, input: ?[]const u8) !void {
1660	switch (ttype) {
1661	.write_block => try bw.writeBlock(tok, false, input),
1662	.write_dyn_block => try bw.writeBlockDynamic(tok, false, input),
1663	else => unreachable,
1664	}
1665	try bw.flush();
1666	}
1667
1668	// Tests if the written block contains an EOF marker.
1669	fn testWriterEOF(ttype: TestType, ht_tokens: []const token.Token, input: []const u8) !void {
1670	var buf = ArrayList(u8).init(testing.allocator);
1671	defer buf.deinit();
1672	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1673	defer bw.deinit();
1674
1675	switch (ttype) {
1676	.write_block => try bw.writeBlock(ht_tokens, true, input),
1677	.write_dyn_block => try bw.writeBlockDynamic(ht_tokens, true, input),
1678	.write_huffman_block => try bw.writeBlockHuff(true, input),
1679	}
1680
1681	try bw.flush();
1682
1683	const b = buf.items;
1684	try expect(b.len > 0);
1685	try expect(b[0] & 1 == 1);
1686	}

lib/std/compress/deflate/huffman_code.zig deleted-432

...	@@ -1,432 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5	const sort = std.sort;
6	const testing = std.testing;
7
8	const Allocator = std.mem.Allocator;
9
10	const bu = @import("bits_utils.zig");
11	const deflate_const = @import("deflate_const.zig");
12
13	const max_bits_limit = 16;
14
15	const LiteralNode = struct {
16	literal: u16,
17	freq: u16,
18	};
19
20	// Describes the state of the constructed tree for a given depth.
21	const LevelInfo = struct {
22	// Our level. for better printing
23	level: u32,
24
25	// The frequency of the last node at this level
26	last_freq: u32,
27
28	// The frequency of the next character to add to this level
29	next_char_freq: u32,
30
31	// The frequency of the next pair (from level below) to add to this level.
32	// Only valid if the "needed" value of the next lower level is 0.
33	next_pair_freq: u32,
34
35	// The number of chains remaining to generate for this level before moving
36	// up to the next level
37	needed: u32,
38	};
39
40	// hcode is a huffman code with a bit code and bit length.
41	pub const HuffCode = struct {
42	code: u16 = 0,
43	len: u16 = 0,
44
45	// set sets the code and length of an hcode.
46	fn set(self: *HuffCode, code: u16, length: u16) void {
47	self.len = length;
48	self.code = code;
49	}
50	};
51
52	pub const HuffmanEncoder = struct {
53	codes: []HuffCode,
54	freq_cache: []LiteralNode = undefined,
55	bit_count: [17]u32 = undefined,
56	lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
57	lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
58	allocator: Allocator,
59
60	pub fn deinit(self: *HuffmanEncoder) void {
61	self.allocator.free(self.codes);
62	self.allocator.free(self.freq_cache);
63	}
64
65	// Update this Huffman Code object to be the minimum code for the specified frequency count.
66	//
67	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
68	// max_bits The maximum number of bits to use for any literal.
69	pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void {
70	var list = self.freq_cache[0 .. freq.len + 1];
71	// Number of non-zero literals
72	var count: u32 = 0;
73	// Set list to be the set of all non-zero literals and their frequencies
74	for (freq, 0..) \|f, i\| {
75	if (f != 0) {
76	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
77	count += 1;
78	} else {
79	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
80	self.codes[i].len = 0;
81	}
82	}
83	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
84
85	list = list[0..count];
86	if (count <= 2) {
87	// Handle the small cases here, because they are awkward for the general case code. With
88	// two or fewer literals, everything has bit length 1.
89	for (list, 0..) \|node, i\| {
90	// "list" is in order of increasing literal value.
91	self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
92	}
93	return;
94	}
95	self.lfs = list;
96	mem.sort(LiteralNode, self.lfs, {}, byFreq);
97
98	// Get the number of literals for each bit count
99	const bit_count = self.bitCounts(list, max_bits);
100	// And do the assignment
101	self.assignEncodingAndSize(bit_count, list);
102	}
103
104	pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 {
105	var total: u32 = 0;
106	for (freq, 0..) \|f, i\| {
107	if (f != 0) {
108	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
109	}
110	}
111	return total;
112	}
113
114	// Return the number of literals assigned to each bit size in the Huffman encoding
115	//
116	// This method is only called when list.len >= 3
117	// The cases of 0, 1, and 2 literals are handled by special case code.
118	//
119	// list: An array of the literals with non-zero frequencies
120	// and their associated frequencies. The array is in order of increasing
121	// frequency, and has as its last element a special element with frequency
122	// std.math.maxInt(i32)
123	//
124	// max_bits: The maximum number of bits that should be used to encode any literal.
125	// Must be less than 16.
126	//
127	// Returns an integer array in which array[i] indicates the number of literals
128	// that should be encoded in i bits.
129	fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 {
130	var max_bits = max_bits_to_use;
131	const n = list.len;
132
133	assert(max_bits < max_bits_limit);
134
135	// The tree can't have greater depth than n - 1, no matter what. This
136	// saves a little bit of work in some small cases
137	max_bits = @min(max_bits, n - 1);
138
139	// Create information about each of the levels.
140	// A bogus "Level 0" whose sole purpose is so that
141	// level1.prev.needed == 0. This makes level1.next_pair_freq
142	// be a legitimate value that never gets chosen.
143	var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
144	// leaf_counts[i] counts the number of literals at the left
145	// of ancestors of the rightmost node at level i.
146	// leaf_counts[i][j] is the number of literals at the left
147	// of the level j ancestor.
148	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
149
150	{
151	var level = @as(u32, 1);
152	while (level <= max_bits) : (level += 1) {
153	// For every level, the first two items are the first two characters.
154	// We initialize the levels as if we had already figured this out.
155	levels[level] = LevelInfo{
156	.level = level,
157	.last_freq = list[1].freq,
158	.next_char_freq = list[2].freq,
159	.next_pair_freq = list[0].freq + list[1].freq,
160	.needed = 0,
161	};
162	leaf_counts[level][level] = 2;
163	if (level == 1) {
164	levels[level].next_pair_freq = math.maxInt(i32);
165	}
166	}
167	}
168
169	// We need a total of 2*n - 2 items at top level and have already generated 2.
170	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
171
172	{
173	var level = max_bits;
174	while (true) {
175	var l = &levels[level];
176	if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
177	// We've run out of both leafs and pairs.
178	// End all calculations for this level.
179	// To make sure we never come back to this level or any lower level,
180	// set next_pair_freq impossibly large.
181	l.needed = 0;
182	levels[level + 1].next_pair_freq = math.maxInt(i32);
183	level += 1;
184	continue;
185	}
186
187	const prev_freq = l.last_freq;
188	if (l.next_char_freq < l.next_pair_freq) {
189	// The next item on this row is a leaf node.
190	const next = leaf_counts[level][level] + 1;
191	l.last_freq = l.next_char_freq;
192	// Lower leaf_counts are the same of the previous node.
193	leaf_counts[level][level] = next;
194	if (next >= list.len) {
195	l.next_char_freq = maxNode().freq;
196	} else {
197	l.next_char_freq = list[next].freq;
198	}
199	} else {
200	// The next item on this row is a pair from the previous row.
201	// next_pair_freq isn't valid until we generate two
202	// more values in the level below
203	l.last_freq = l.next_pair_freq;
204	// Take leaf counts from the lower level, except counts[level] remains the same.
205	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
206	levels[l.level - 1].needed = 2;
207	}
208
209	l.needed -= 1;
210	if (l.needed == 0) {
211	// We've done everything we need to do for this level.
212	// Continue calculating one level up. Fill in next_pair_freq
213	// of that level with the sum of the two nodes we've just calculated on
214	// this level.
215	if (l.level == max_bits) {
216	// All done!
217	break;
218	}
219	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
220	level += 1;
221	} else {
222	// If we stole from below, move down temporarily to replenish it.
223	while (levels[level - 1].needed > 0) {
224	level -= 1;
225	if (level == 0) {
226	break;
227	}
228	}
229	}
230	}
231	}
232
233	// Somethings is wrong if at the end, the top level is null or hasn't used
234	// all of the leaves.
235	assert(leaf_counts[max_bits][max_bits] == n);
236
237	var bit_count = self.bit_count[0 .. max_bits + 1];
238	var bits: u32 = 1;
239	const counts = &leaf_counts[max_bits];
240	{
241	var level = max_bits;
242	while (level > 0) : (level -= 1) {
243	// counts[level] gives the number of literals requiring at least "bits"
244	// bits to encode.
245	bit_count[bits] = counts[level] - counts[level - 1];
246	bits += 1;
247	if (level == 0) {
248	break;
249	}
250	}
251	}
252	return bit_count;
253	}
254
255	// Look at the leaves and assign them a bit count and an encoding as specified
256	// in RFC 1951 3.2.2
257	fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void {
258	var code = @as(u16, 0);
259	var list = list_arg;
260
261	for (bit_count, 0..) \|bits, n\| {
262	code <<= 1;
263	if (n == 0 or bits == 0) {
264	continue;
265	}
266	// The literals list[list.len-bits] .. list[list.len-bits]
267	// are encoded using "bits" bits, and get the values
268	// code, code + 1, .... The code values are
269	// assigned in literal order (not frequency order).
270	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
271
272	self.lns = chunk;
273	mem.sort(LiteralNode, self.lns, {}, byLiteral);
274
275	for (chunk) \|node\| {
276	self.codes[node.literal] = HuffCode{
277	.code = bu.bitReverse(u16, code, @as(u5, @intCast(n))),
278	.len = @as(u16, @intCast(n)),
279	};
280	code += 1;
281	}
282	list = list[0 .. list.len - @as(u32, @intCast(bits))];
283	}
284	}
285	};
286
287	fn maxNode() LiteralNode {
288	return LiteralNode{
289	.literal = math.maxInt(u16),
290	.freq = math.maxInt(u16),
291	};
292	}
293
294	pub fn newHuffmanEncoder(allocator: Allocator, size: u32) !HuffmanEncoder {
295	return HuffmanEncoder{
296	.codes = try allocator.alloc(HuffCode, size),
297	// Allocate a reusable buffer with the longest possible frequency table.
298	// (deflate_const.max_num_frequencies).
299	.freq_cache = try allocator.alloc(LiteralNode, deflate_const.max_num_frequencies + 1),
300	.allocator = allocator,
301	};
302	}
303
304	// Generates a HuffmanCode corresponding to the fixed literal table
305	pub fn generateFixedLiteralEncoding(allocator: Allocator) !HuffmanEncoder {
306	const h = try newHuffmanEncoder(allocator, deflate_const.max_num_frequencies);
307	var codes = h.codes;
308	var ch: u16 = 0;
309
310	while (ch < deflate_const.max_num_frequencies) : (ch += 1) {
311	var bits: u16 = undefined;
312	var size: u16 = undefined;
313	switch (ch) {
314	0...143 => {
315	// size 8, 000110000 .. 10111111
316	bits = ch + 48;
317	size = 8;
318	},
319	144...255 => {
320	// size 9, 110010000 .. 111111111
321	bits = ch + 400 - 144;
322	size = 9;
323	},
324	256...279 => {
325	// size 7, 0000000 .. 0010111
326	bits = ch - 256;
327	size = 7;
328	},
329	else => {
330	// size 8, 11000000 .. 11000111
331	bits = ch + 192 - 280;
332	size = 8;
333	},
334	}
335	codes[ch] = HuffCode{ .code = bu.bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
336	}
337	return h;
338	}
339
340	pub fn generateFixedOffsetEncoding(allocator: Allocator) !HuffmanEncoder {
341	const h = try newHuffmanEncoder(allocator, 30);
342	var codes = h.codes;
343	for (codes, 0..) \|_, ch\| {
344	codes[ch] = HuffCode{ .code = bu.bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
345	}
346	return h;
347	}
348
349	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
350	_ = context;
351	return a.literal < b.literal;
352	}
353
354	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
355	_ = context;
356	if (a.freq == b.freq) {
357	return a.literal < b.literal;
358	}
359	return a.freq < b.freq;
360	}
361
362	test "generate a Huffman code from an array of frequencies" {
363	var freqs: [19]u16 = [_]u16{
364	8, // 0
365	1, // 1
366	1, // 2
367	2, // 3
368	5, // 4
369	10, // 5
370	9, // 6
371	1, // 7
372	0, // 8
373	0, // 9
374	0, // 10
375	0, // 11
376	0, // 12
377	0, // 13
378	0, // 14
379	0, // 15
380	1, // 16
381	3, // 17
382	5, // 18
383	};
384
385	var enc = try newHuffmanEncoder(testing.allocator, freqs.len);
386	defer enc.deinit();
387	enc.generate(freqs[0..], 7);
388
389	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422	}
423
424	test "generate a Huffman code for the fixed literal table specific to Deflate" {
425	var enc = try generateFixedLiteralEncoding(testing.allocator);
426	defer enc.deinit();
427	}
428
429	test "generate a Huffman code for the 30 possible relative offsets (LZ77 distances) of Deflate" {
430	var enc = try generateFixedOffsetEncoding(testing.allocator);
431	defer enc.deinit();
432	}

lib/std/compress/deflate/testdata/compress-e.txt deleted-1

...	@@ -1 +0,0 @@
1	2.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788567430285974600143785483237068701190078499404930918919181649327259774030074879681484882342932023012128032327460392219687528340516906974194257614673978110715464186273369091584973185011183960482533518748438923177292613543024932562896371361977285456622924461644497284597867711574125670307871885109336344480149675240618536569532074170533486782754827815415561966911055101472799040386897220465550833170782394808785990501947563108984124144672821865459971596639015641941751820935932616316888380132758752601460507676098392625726411120135288591317848299475682472564885533357279772205543568126302535748216585414000805314820697137262149755576051890481622376790414926742600071045922695314835188137463887104273544767623577933993970632396604969145303273887874557905934937772320142954803345000695256980935282887783710670585567749481373858630385762823040694005665340584887527005308832459182183494318049834199639981458773435863115940570443683515285383609442955964360676090221741896883548131643997437764158365242234642619597390455450680695232850751868719449064767791886720306418630751053512149851051207313846648717547518382979990189317751550639981016466414592102406838294603208535554058147159273220677567669213664081505900806952540610628536408293276621931939933861623836069111767785448236129326858199965239275488427435414402884536455595124735546139403154952097397051896240157976832639450633230452192645049651735466775699295718989690470902730288544945416699791992948038254980285946029052763145580316514066229171223429375806143993484914362107993576737317948964252488813720435579287511385856973381976083524423240466778020948399639946684833774706725483618848273000648319163826022110555221246733323184463005504481849916996622087746140216157021029603318588727333298779352570182393861244026868339555870607758169954398469568540671174444932479519572159419645863736126915526457574786985964242176592896862383506370433939811671397544736228625506803682664135541448048997721373174119199970017293907303350869020922519124447393278376156321810842898207706974138707053266117683698647741787180202729412982310888796831880854367327806879771659111654224453806625861711729498038248879986504061563975629936962809358189761491017145343556659542757064194408833816841111166200759787244137082333917886114708228657531078536674695018462140736493917366254937783014074302668422150335117736471853872324040421037907750266020114814935482228916663640782450166815341213505278578539332606110249802273093636740213515386431693015267460536064351732154701091440650878823636764236831187390937464232609021646365627553976834019482932795750624399645272578624400375983422050808935129023122475970644105678361870877172333555465482598906861201410107222465904008553798235253885171623518256518482203125214950700378300411216212126052726059944320443056274522916128891766814160639131235975350390320077529587392412476451850809163911459296071156344204347133544720981178461451077872399140606290228276664309264900592249810291068759434533858330391178747575977065953570979640012224092199031158229259667913153991561438070129260780197022589662923368154312499412259460023399472228171056603931877226800493833148980338548909468685130789292064242819174795866199944411196208730498064385006852620258432842085582338566936649849720817046135376163584015342840674118587581546514598270228676671855309311923340191286170613364873183197560812569460089402953094429119590295968563923037689976327462283900735457144596414108229285922239332836210192822937243590283003884445701383771632056518351970100115722010956997890484964453434612129224964732356126321951155701565824427661599326463155806672053127596948538057364208384918887095176052287817339462747644656858900936266123311152910816041524100214195937349786431661556732702792109593543055579732660554677963552005378304619540636971842916168582734122217145885870814274090248185446421774876925093328785670674677381226752831653559245204578070541352576903253522738963847495646255940378924925007624386893776475310102323746733771474581625530698032499033676455430305274561512961214585944432150749051491453950981001388737926379964873728396416897555132275962011838248650746985492038097691932606437608743209385602815642849756549307909733854185583515789409814007691892389063090542534883896831762904120212949167195811935791203162514344096503132835216728021372415947344095498316138322505486708172221475138425166790445416617303200820330902895488808516797258495813407132180533988828139346049850532340472595097214331492586604248511405819579711564191458842833000525684776874305916390494306871343118796189637475503362820939949343690321031976898112055595369465424704173323895394046035325396758354395350516720261647961347790912327995264929045151148307923369382166010702872651938143844844532639517394110131152502750465749343063766541866128915264446926222884366299462732467958736383501937142786471398054038215513463223702071533134887083174146591492406359493020921122052610312390682941345696785958518393491382340884274312419099152870804332809132993078936867127413922890033069995875921815297612482409116951587789964090352577345938248232053055567238095022266790439614231852991989181065554412477204508510210071522352342792531266930108270633942321762570076323139159349709946933241013908779161651226804414809765618979735043151396066913258379033748620836695475083280318786707751177525663963479259219733577949555498655214193398170268639987388347010255262052312317215254062571636771270010760912281528326508984359568975961038372157726831170734552250194121701541318793651818502020877326906133592182000762327269503283827391243828198170871168108951187896746707073377869592565542713340052326706040004348843432902760360498027862160749469654989210474443927871934536701798673920803845633723311983855862638008516345597194441994344624761123844617615736242015935078520825600604101556889899501732554337298073561699861101908472096600708320280569917042590103876928658336557728758684250492690370934262028022399861803400211320742198642917383679176232826444645756330336556777374808644109969141827774253417010988435853189339175934511574023847292909015468559163792696196841000676598399744972047287881831200233383298030567865480871476464512824264478216644266616732096012564794514827125671326697067367144617795643752391742928503987022583734069852309190464967260243411270345611114149835783901793499713790913696706497637127248466613279908254305449295528594932793818341607827091326680865655921102733746700132583428715240835661522165574998431236278287106649401564670141943713823863454729606978693335973109537126499416282656463708490580151538205338326511289504938566468752921135932220265681856418260827538790002407915892646028490894922299966167437731347776134150965262448332709343898412056926145108857812249139616912534202918139898683901335795857624435194008943955180554746554000051766240202825944828833811886381749594284892013520090951007864941868256009273977667585642598378587497776669563350170748579027248701370264203283965756348010818356182372177082236423186591595883669487322411726504487268392328453010991677518376831599821263237123854357312681202445175401852132663740538802901249728180895021553100673598184430429105288459323064725590442355960551978839325930339572934663055160430923785677229293537208416693134575284011873746854691620648991164726909428982971065606801805807843600461866223562874591385185904416250663222249561448724413813849763797102676020845531824111963927941069619465426480006761727618115630063644321116224837379105623611358836334550102286170517890440570419577859833348463317921904494652923021469259756566389965893747728751393377105569802455757436190501772466214587592374418657530064998056688376964229825501195065837843125232135309371235243969149662310110328243570065781487677299160941153954063362752423712935549926713485031578238899567545287915578420483105749330060197958207739558522807307048950936235550769837881926357141779338750216344391014187576711938914416277109602859415809719913429313295145924373636456473035037374538503489286113141638094752301745088784885645741275003353303416138096560043105860548355773946625033230034341587814634602169235079216111013148948281895391028916816328709309713184139815427678818067628650978085718262117003140003377301581536334149093237034703637513354537634521050370995452942055232078817449370937677056009306353645510913481627378204985657055608784211964039972344556458607689515569686899384896439195225232309703301037277227710870564912966121061494072782442033414057441446459968236966118878411656290355117839944070961772567164919790168195234523807446299877664824873753313018142763910519234685081979001796519907050490865237442841652776611425351538665162781316090964802801234493372427866930894827913465443931965254154829494577875758599482099181824522449312077768250830768282335001597040419199560509705364696473142448453825888112602753909548852639708652339052941829691802357120545328231809270356491743371932080628731303589640570873779967845174740515317401384878082881006046388936711640477755985481263907504747295012609419990373721246201677030517790352952793168766305099837441859803498821239340919805055103821539827677291373138006715339240126954586376422065097810852907639079727841301764553247527073788764069366420012194745702358295481365781809867944020220280822637957006755393575808086318932075864444206644691649334467698180811716568665213389686173592450920801465312529777966137198695916451869432324246404401672381978020728394418264502183131483366019384891972317817154372192103946638473715630226701801343515930442853848941825678870721238520597263859224934763623122188113706307506918260109689069251417142514218153491532129077723748506635489170892850760234351768218355008829647410655814882049239533702270536705630750317499788187009989251020178015601042277836283644323729779929935160925884515772055232896978333126427671291093993103773425910592303277652667641874842441076564447767097790392324958416348527735171981064673837142742974468992320406932506062834468937543016787815320616009057693404906146176607094380110915443261929000745209895959201159412324102274845482605404361871836330268992858623582145643879695210235266673372434423091577183277565800211928270391042391966426911155333594569685782817020325495552528875464466074620294766116004435551604735044292127916358748473501590215522120388281168021413865865168464569964810015633741255098479730138656275460161279246359783661480163871602794405482710196290774543628092612567507181773641749763254436773503632580004042919906963117397787875081560227368824967077635559869284901628768699628053790181848148810833946900016380791075960745504688912686792812391148880036720729730801354431325347713094186717178607522981373539126772812593958220524289991371690685650421575056729991274177149279608831502358697816190894908487717722503860872618384947939757440664912760518878124233683125467278331513186758915668300679210215947336858591201395360301678110413444411030903388761520488296909104689167671555373346622545575975202624771242796225983278405833585897671474205724047439720232895903726148688388003174146490203843590358527993123871042845981608996101945691646983837718267264685264869172948414153004604004299585035164101899027529366867431834955447458124140190754681607770977920579383895378192128847409929537040546962226547278807248685508046571043123854873351653070570784584243335550958221912862797205455466267099131902370311779690892786623112661337671178512943059323281605826535623848164192144732543731002062738466812351691016359252588256806438946389880872735284406462208149513862275239938938734905082625472417781702582044129853760499827899020083498387362992498125742354568439023012261733665820546785671147973065077035475620567428300187473019197310881157516777005071432012726354601912460800451608108641835539669946936947322271670748972850464195392966434725254724357659192969949061670189061433616907056148280980363243454128229968275980226694045642181328624517549652147221620839824594576613342710564957193564431561774500828376935700995419541839029151033187933907614207467028867968594985439789457300768939890070073924697461812855764662265412913204052279071212820653775058280040897163467163709024906774736309136904002615646432159560910851092445162454420141442641660181385990017417408244245378610158433361777292580611159192008414091888191208858207627011483671760749046980914443057262211104583300789331698191603917150622792986282709446275915009683226345073725451366858172483498470080840163868209726371345205439802277866337293290829914010645589761697455978409211409167684020269370229231743334499986901841510888993165125090001163719114994852024821586396216294981753094623047604832399379391002142532996476235163569009445086058091202459904612118623318278614464727795523218635916551883057930657703331498510068357135624341881884405780028844018129031378653794869614630467726914552953690154167025838032477842272417994513653582260971652588356712133519546838335349801503269359798167463231847628306340588324731228951257944267639877946713121042763380872695738609314631539148548792514028885025189788076023838995615684850391995855029256054176767663145354058496296796781349420116003325874431438746248313850214980401681940795687219268462617287403480967931949965604299190281810597603263251746405016454606266765529010639868703668263299050577706266397868453584384057673298268163448646707439990917504018892319267557518354054956017732907127219134577524905771512773358423314008356080926962298894163047287780054743798498545562870729968407382937218623831766524716090967192007237658894226186550487552614557855898773008703234726418384831040394818743616224455286163287628541175946460497027724490799275146445792982549802258601001772437840167723166802004162547244179415547810554178036773553354467030326469619447560812831933095679685582771932031205941616693902049665352189672822671972640029493307384717544753761937017882976382487233361813499414541694736549254840633793674361541081593464960431603544354737728802361047743115330785159902977771499610274627769759612488879448609863349422852847651310277926279743981957617505591300993377368240510902583759345170015340522266144077237050890044496613295859536020556034009492820943862994618834790932894161098856594954213114335608810239423706087108026465913203560121875933791639666437282836752328391688865373751335794859860107569374889645657187292540448508624449947816273842517229343960137212406286783636675845331904743954740664015260871940915743955282773904303868772728262065663129387459875317749973799293043294371763801856280061141619563942414312254397099163565102848315765427037906837175764870230052388197498746636856292655058222887713221781440489538099681072143012394693530931524054081215705402274414521876541901428386744260011889041724570537470755550581632831687247110220353727166112304857340460879272501694701067831178927095527253222125224361673343366384756590949728221809418684074238351567868893421148203905824224324264643630201441787982022116248471657468291146315407563770222740135841109076078464780070182766336227978104546331131294044833570134869585165267459515187680033395522410548181767867772152798270250117195816577603549732923724732067853690257536233971216884390878879262188202305529937132397194333083536231248870386416194361506529551267334207198502259771408638122015980894363561808597010080081622557455039101321981979045520049618583777721048046635533806616517023595097133203631578945644487800945620369784973459902004606886572701865867757842758530645706617127194967371083950603267501532435909029491516973738110897934782297684100117657987098185725131372267749706609250481876835516003714638685918913011736805218743265426063700710595364425062760458252336880552521181566417553430681181548267844169315284408461087588214317641649835663127518728182948655658524206852221830755306118393326934164459415342651778653397980580828158806300749952897558204686612590853678738603318442905510689778698417735603118111677563872589911516803236547002987989628986181014596471307916144369564690909518788574398821730583884980809523077569358851616027719521488998358632323127308909861560777386006984035267826785387215920936255817889813416247486456433211043194821421299793188104636399541496539441501383868748384870224681829391860319598667962363489309283087840712400431022706137591368056518861313458307990705003607588327248867879324093380071864152853317943535073401891193638546730000660453783784472469288830546979000131248952100446949032058838294923613919284305249167833012980192255157050378521810552961623637523647962685751660066539364142273063001648652613891842243501797455993616794063303522111829071597538821839777552812981538570168702202620274678647916644030729018445497956399844836807851997088201407769199261674991148329821854382718946282165387064858588646221611410343570342878862979083418871606214430014533275029715104673156021000043869510583773779766003460887624861640938645252177935289947578496255243925598620521409052346250847830487046492688313289470553891357290706967599556298586669559721686506052072801342104355762779184021797626656484580261591407173477009039475168017709900129391137881248534255949312866653465033728846390649968460644741907524313323903404908195233044389559060547854954620263256676813262435925020249516275607080900436460421497025691488555265022810327762115842282433269528629137662675481993546118143913367579700141255870143319434764035725376914388899683088262844616425575034001428982557620386364384137906519612917777354183694676232982904981261717676191554292570438432239918482261744350470199171258214687683172646078959690569981353264435973965173473319484798758064137926885413552523275720457329477215706850016950046959758389373527538622664943456437071610511521617176237598050900553232154896062817794302268640579555845730600598376482703339859420098582351400179507104569019191359062304102336798080907240196312675268916362136351032648077232914950859151265812143823371072949148088472355286394195993455684156344577951727033374238129903260198160571971183950662758220321837136059718025940870615534713104482272716848395524105913605919812444978458110854511231668173534838253724825347636777581712867205865148285317273569069839935110763432091319780314031658897379628301178409806410175016511072932907832177487566289310650383806093372841399226733384778203302020700517188941706465146238366720632742644336612174011766914919235570905644803016342294301837655263108450172510307540942604409687066288066265900569082451407632599158164499361455172452057020443093722305550217222299706209749268609762787409626448772056043078634808885709143464793241536214303199965695610753570417207285334250171325558818113295504095217830139465216436594262960768570585698507157151317262928960072587601564840556088613165411835958628710665496282599535127193244635791046554389165150954187306071015034430609582302257455974944275067630926322529966338219395202927917973247094559691016402983683080426309910481567503623509654924302589575273521412445149542462972258510120707802110188106722347972579330653187713438466713807546383471635428854957610942841898601794658721444495198801550804042506452191484989920400007310672369944655246020908767882300064337725657385010969899058191290957079866699453765080407917852438222041070599278889267745752084287526377986730360561230710723922581504781379172731261234878334034473833573601973235946604273704635201327182592410906040097638585857716958419563109577748529579836844756803121874818202833941887076311731615289811756429711334181497218078040465077657204457082859417475114926179367379999220181789399433337731146911970737861041963986422166045588965683206701337505745038872111332436739840284188639147633491695114032583475841514170325690161784931455706904169858050217798497637014758914810543205854914100662201721719726878930012101267481270235940855162601689425111458499658315589660460091525797881670384625905383256920520425791378948827579603278877535466861441826827797651258953563761485994485049706638406266121957141911063246061774180577212381659872472432252969098533628440799030007594546281549235506086481557928961969617060715201589825299772803520002610888814176506636216905928021516429198484077446143617891415191517976537848282687018750030264867608433204658525470555882410254654806040437372771834769014720664234434374255514129178503032471263418076525187802925534774001104853996960549926508093910691337614841834884596365621526610332239417467064368340504749943339802285610313083038484571294767389856293937641914407036507544622061186499127249643799875806537850203753189972618014404667793050140301580709266213229273649718653952866567538572115133606114457222800851183757899219543063413692302293139751143702404830227357629039911794499248480915071002444078482866598579406525539141041497342780203520135419925977628178182825372022920108186449448349255421793982723279357095828748597126780783134286180750497175747373730296280477376908932558914598141724852658299510882230055223242218586191394795184220131553319634363922684259164168669438122537135960710031743651959027712571604588486044820674410935215327906816032054215967959066411120187618531256710150212239401285668608469435937408158536481912528004920724042172170913983123118054043277015835629513656274610248827706488865037765175678806872498861657094846665770674577000207144332525555736557083150320019082992096545498737419756608619533492312940263904930982014700371161829485939931199955070455381196711289367735249958182011774799788636393286405807810818657337668157893827656450642917396685579555053188715314552353070355994740186225988149854660737787698781542360397080977412361518245964026869979609564523828584235953564615185448165799966460648261396618720304839119560250381111550938420209894591555760083897989949964566262540514195610780090298667014635238532066032574466820259430618801773091109212741138269148784355679352572808875543164693077235363768226036080174040660997151176880434927489197133087822951123746632635635328517394189466510943745768270782209928468034684157443127739811044186762032954475468077511126663685479944460934809992951875666499902261686019672053749149951226823637895865245462813439289338365156536992413109638102559114643923805213907862893561660998836479175633176725856523591069520326895990054884753424160586689820067483163174286329119633399132709086065074595260357157323069712106423424081597068328707624437165532750228797802598690981111226558888151520837482450034463046505984569690276166958278982913613535306291331427881888249342136442417833519319786543940201465328083410341785272489879050919932369270996567133507711905899945951923990615156165480300145359212550696405345263823452155999210578191371030188979206408883974767667144727314254467923500524618849237455307575734902707342496298879996942094595961008702501329453325358045689285707241207965919809225550560061971283541270202072583994171175520920820151096509526685113897577150810849443508285458749912943857563115668324566827992991861539009255871716840495663991959154034218364537212023678608655364745175654879318925644085274489190918193411667583563439758886046349413111875241038425467937999203546910411935443113219136068129657568583611774564654674861061988591414805799318725367531243470335482637527081353105570818049642498584646147973467599315946514787025065271083508782350656532331797738656666181652390017664988485456054961300215776115255813396184027067814900350252876823607822107397102339146870159735868589015297010347780503292154014359595298683404657471756232196640515401477953167461726208727304820634652469109953327375561090578378455945469160223687689641425960164689647106348074109928546482353083540132332924864037318003195202317476206537726163717445360549726690601711176761047774971666890152163838974311714180622222345718567941507299526201086205084783127474791909996889937275229053674785020500038630036526218800670926674104806027341997756660029427941090400064654281074454007616429525362460261476180471744322889953285828397762184600967669267581270302806519535452053173536808954589902180783145775891280203970053633193821100095443241244197949192916205234421346395653840781209416214835001155883618421164283992454027590719621537570187067083731012246141362048926555668109467076386536083015847614512581588569610030337081197058344452874666198891534664244887911940711423940115986970795745946337170243268484864632018986352827092313047089215684758207753034387689978702323438584381125011714013265769320554911860153519551654627941175593967947958810333935413289702528893533748106257875620364294270257512121137330213811951395756419122685155962476203282038726342066227347868223036522019655729325905068134849292299647248229359787842720945578267329975853818536442370617353517653060396801087899490506654491544577952166038552398013798104340564182403396162494910454712104839439200945914647542424785991096900046541371091630096785951563947332190934511838669964622788855817353221326876634958059123761251203010983867841195725887799206041260049865895027247133146763722204388398558347770112599424691208308595666787531942465131444389971195968105937957532155524204659410081418351120174196853432672343271868099625045432475688702055341969199545300952644398446384346598830418262932239295612610045884644244285011551557765935780379565026806130721758672048541797157896401554276881090475899564605488362989140226580026134158039480357971019004151547655018391755772677897148793477372747525743898158705040701968215101218826088040084551332795162841280679678965570163917067779841529149397403158167896865448841319046368332179115059107813898261026271979696826411179918656038993895418928488851750122504754778999508544083983800725431468842988412616042682248823097788556495765424017114510393927980290997604904428832198976751320535115230545666467143795931915272680278210241540629795828828466355623580986725638200565215519951793551069127710538552661926903526081367717666435071213453983711357500975854405939558661737828297120544693182260401670308530911657973113259516101749193468250063285777004686987177255226525708428745733039859744230639751837209975339055095883623642814493247460522424051972825153787541962759327436278819283740253185668545040893929401040561666867664402868211607294830305236465560955351079987185041352121321534713770667681396211443891632403235741573773787908838267618458756361026435182951815392455211729022985278518025598478407179607904114472041476091765804302984501746867981277584971731733287305281134969591668387877072315968334322509070204019030503595891994666652037530271923764252552910347950343816357721698115464329245608951158732012675424975710520894362639501382962152214033621065422821876739580121286442788547491928976959315766891987305176388698461503354594898541849550251690616888419122873385522699976822609645007504500096116866129171093180282355042553653997166054753907348915189650027442328981181709248273610863801576007240601649547082331349361582435128299050405405333992577071321011503713898695076713447940748097845416328110406350804863393555238405735580863718763530261867971725608155328716436111474875107033512913923595452951407437943144900950809932872153235195999616750297532475931909938012968640379783553559071355708369947311923538531051736669154087312467233440702525006918026747725078958903448856673081487299464807786497709361969389290891718228134002845552513917355978456150353144603409441211512001738697261466786933733154341007587514908295822756919350542184106448264951943804240543255345965248373785310657979037977505031436474651422484768831323479762673689855474944277949916560108528257618964374464656819789319422077536824661110427671936481836360534108748971066866318805026555929568123959680449295166615409802610781691689418764353363449482900125929366840591370059526914934421861891742142561071896846626335874414976973921566392767687720145153302241853125308442727245771161505550519076276250016522166274796257424425420546785767478190959486500575711016264847833741198041625940813327229905891486422127968042984725356237202887830051788539737909455265135144073130049869453403245984236934627060242579432563660640597549471239092372458126154582526667304702319359866523378856244229188278436440434628094888288712101968642736370461639297485616780079779959696843367730352483047478240669928277140069031660709951473154191919911453182543906294573298686613524886500574780251977607442660798300291573030523199052185718628543687577860915726925232573171665625274275808460620177046433101212443409281314659760221360416223031167750085960128475289259463348312408766740128170543067985261868949895004918275008304998926472034986965363326210919830621495095877228260815566702155693484634079776879525038204442326697479264829899016938511552124688935873289878336267819361764023681714606495185508780596635354698788205094762016350757090024201498400967867845405354130050482404996646978558002628931826518708714613909521454987992300431779500489569529280112698632533646737179519363094399609176354568799002814515169743717518330632232942199132137614506411391269837128970829395360832883050256072727563548374205497856659895469089938558918441085605111510354367477810778500572718180809661542709143010161515013086522842238721618109043183163796046431523184434669799904865336375319295967726080853457652274714047941973192220960296582500937408249714373040087376988068797038047223488825819819025644086847749767508999164153502160223967816357097637814023962825054332801828798160046910336602415904504637333597488119998663995617171089911809851197616486499233594328274275983382931099806461605360243604040848379619072542165869409486682092396143083817303621520642297839982533698027039931804024928814430649614747600087654305571672697259114631990688823893005380061568007730984416061355843701277573463708822073792921409548717956947854414951731561828176343929570234710460088230637509877521391223419548471196982303169544468045517922669260631327498272520906329003279972932906827204647650366969765227673645419031639887433042226322021325368176044169612053532174352764937901877252263626883107879345194133825996368795020985033021472307603375442346871647223795507794130304865403488955400210765171630884759704098331306109510294140865574071074640401937347718815339902047036749084359309086354777210564861918603858715882024476138160390378532660185842568914109194464566162667753712365992832481865739251429498555141512136758288423285957759412684479036912662015308418041737698963759002546999454131659341985624780714434977201991702665380714107259910648709897259362243300706760476097690456341576573395549588448948093604077155688747288451838106069038026528318275560395905381507241627615047252487759578650784894547389096573312763852962664517004459626327934637721151028545472312880039058405918498833810711366073657536918428084655898982349219315205257478363855266205400703561310260405145079325925798227406012199249391735122145336707913500607486561657301854049217477162051678486507913573336334257685988361252720250944019430674728667983441293018131344299088234006652915385763779110955708000600143579956351811596764725075668367726052352939773016348235753572874236648294604770429166438403558846422370760111774821079625901180265548868995181239470625954254584491340203400196442965370643088660925268811549596291166168612036195319253262662271108142149856132646467211954801142455133946382385908540917878668826947602781853283155445565265933912487885639504644196022475186011405239187543742526581685003052301877096152411653980646785444273124462179491306502631062903402737260479940181929954454297256377507172705659271779285537195547433852182309492703218343678206382655341157162788603990157495208065443409462446634653253581574814022471260618973060860559065082163068709634119751925774318683671722139063093061019303182326666420628155129647685313861018672921889347039342072245556791239578260248978371473556820782675452142687314252252601795889759116238720807580527221031327444754083319215135934526961397220564699247718289310588394769170851420631557192703636345039529604362885088555160008371973526383838996789184600327073682083234847108471706160879195227388252347506380811606090840124222431476103563328940609282430125462013806032608121942876847907192546246309055749298781661271916548229644317263587524548607563020667656942355342774617635549231817456159185668061686428714964129290560130053913469569829490891003991259088290348791943368696942620662946948514931472688923571615032405542263391673583102728579723061998175868700492227418629077079508809336215346303842967525604369606110193842723883107587771653594778681499030978765900869583480043137176832954871752604714113064847270887246697164585218774442100900090916189819413456305028950484575822161887397443918833085509908566008543102796375247476265353031558684515120283396640547496946343986288291957510384781539068343717740714095628337554413567955424664601335663617305811711646062717854078898495334329100315985673932305693426085376230981047171826940937686754301837015557540822371538037838383342702379535934403549452173960327095407712107332936507766465603712364707109272580867897181182493799540477008369348889220963814281561595610931815183701135104790176383595168144627670903450457460997444500166918675661035889313483800512736411157304599205955471122443903196476642761038164285918037488354360663299436899730090925177601162043761411616688128178292382311221745850238080733727204908880095181889576314103157447684338100457385008523652069340710078955916549813037292944462306371284357984809871964143085146878525033128989319500645722582281175483887671061073178169281242483613796475692482076321356427357261609825142445262515952514875273805633150964052552659776922077806644338105562443538136258941809788015677378951310313157361136026047890761945591820289365770116416881703644242694283057457471567494391573593353763114830246668754727566653059819746822346578699972291792416156043557665183382167059157867799311835820189855730344883681934418305987021880502259192818047775223884407167894780414701414651073580452021499197980812095692195622632313741870979731320870864552236740416185590793816745658234353037283309503729022429802768451559528656923189798000383061378732434546500582722712325031420712488100290697226311129067629080951145758060270806092801504406139446350643069742785469477459876821004441453438033759717384777232052065301037861326418823586036569054773343070911759152582503029410738914441818378779490613137536794654893375260322906277631983337976816641721083140551864133302224787118511817036598365960493964571491686005656771360533192423185262166760222073368844844409234470948568027905894191829969467724456269443308241243846160408284006424867072583661011433404214473683453638496544701067827313169538435919120440283949541956874453676459875488726170687163109591315801609722382049772577307454562979127906177531663252857205858766376754282917933549923678212008601904369428956102301731743150352204665675088491593025926618816581008701658499456495586855628208747248318351516339189292646558880593601275151838235485893426165223086697314511412035659916934103076974774451947043836739600076578628245472064617380804602903639144493859012422380173377038154675297645596518492676039300171943042511794045679862114630138402371099347243455794730048929825402680821621522346560274258486595687074510352794291633405915025075992398611224340312056999780516223878772230396359709132856830486160362127579561601328561866388146004722200580017580282279272167842720649966956840905752590774886105493806116954293569077377792821084159737469613143291808510446953973485067590503662391722108732333169909603363771705474725026941732982890400239372879549386540463828596742216318201530139629734398479588628632934746650690284066719018081265539973675916799759010867483920062877888531102781695087545740384607594616919584610655963327283485609570305572502494416337066573150237126843581984154103154401008430380631442183776750349813408169325201240813452285974626715177152223063741359255747513535160669108359443999692315898156732033027129284241219651936303734407981204656795322986357374589031654007016472204989445629050395873788912680565516464274460174738175296313458739390484560414203426465560422112239134631023161290836446988901247285192778589195228773637440432659264672239982186452797664826673070168802722052338600372842903155828454593854349099449420750911108532138744823216151007808922516285123275724355101999038195993350032641446053470357293073912578481757987468353429629749652545426864234949270336399427519354240001973125098882419600095766257217621860474573769577649582201796258392376391717855799468922496750179251915218219624653575570564228220399546682648329822996167217080156801080799777126517156274295763666959661983507435667132218383358509536665806605597148376773866922551603463644386269977295750658468929599809168949981898588529537874489519527097766262684177088590284321676352132630838812766335363319004134332844347630067982023716933653652880580156390360562722752187272454764258840995216482554453662083811789117725225682611478014242896970967121967502094421226279437073328703410646312100557376727450271638975234111426287828736758358819056742163061523416789476056879277154789714326222041069587947186435439940738639948986836168919377836648327137363654676901173760246643082285362494712605173293777247276797635865806019396287718060679122426813922872134061694882029506831654589707623668302556167559477498715183426989208952182644710514911419441192277010977616645850068963849426165593473112961064282379048216056210094265076173838082479030510998790719611852832556787472942907151041468948104916751035295897242381802288151276582257190705537652455285511598636421244284176256230139538669970308943645907600684938040875210854159851278070333207779865635907968462191534944587677170063778573171211036517486371634098385626541555573292664616402279791195975248525300376741774056125700303625811704838385391207273191845064713669122576415213769896260940351804147432053600369234179035440735703058314741623452840188940808983125191307741823338981880316339159565954543405777784331681162551898060409183018907512170192983622897099598983405484962284289398469847938668614293324543983592637036699355184231661615244505980576745765335552338715678211466689996845227042954589710922163652573965950289645637766038988037941517917867910675199009966139206238732318786758420544279396366759104126821843375015743069045967947046685602358283919759975285865384338189120042853787549302768972168199113340697282255535300044743958830079799736518459131437946494086272149669719100359399974735262764126125995350902609540048669398955899487421379590802893196914845826873123710180229775301190684280440780938156598081694611679374425663244656799606363751546304833112722231812338371779800439731087402647536582575657351059978314264831879619843765495877803685261751835391844920488198629786329743136948511780579298636452193232481339393090754566368038513630619718033957979522539508697432546502659123585049283028832934489284591373621624852528877442891851104093746333590660233239711922814450735588373324057814862662207486215513375036775585494138678352928273109003823116855374520901095101174796663003330352534143230024288248051396631446632656081582045216883922312025671065388459503224002320453633895521539919011035217362720909565500846486605368975498478995875596103167696587161281951919668893326641203784750417081752273735270989343717167642329956935697166213782736138899530515711822960896394055380431939398453970864418654291655853168697537052760701061488025700785387150835779480952313152747735711713643356413242974208137266896149109564214803567792270566625834289773407718710649866150447478726164249976671481383053947984958938064202886667951943482750168192023591633247099185942520392818083953020434979919361853380201407072481627304313418985942503858404365993281651941497377286729589582881907490040331593436076189609669494800067194371424058105327517721952474344983414191979918179909864631583246021516575531754156198940698289315745851842783390581029411600498699307751428513021286202539508732388779357409781288187000829944831476678183644656510024467827445695591845768068704978044824105799710771577579093525803824227377612436908709875189149049904225568041463131309240101049368241449253427992201346380538342369643767428862595140146178201810734100565466708236854312816339049676558789901487477972479202502227218169405159042170892104287552188658308608452708423928652597536146290037780167001654671681605343292907573031466562485809639550080023347676187068086526878722783177420214068980703410506200235273632267291964034093571225623659496432076928058165514428643204955256838543079254299909353199329432966018220787933122323225928276556048763399988478426451731890365879756498207607478270258861409976050788036706732268192473513646356758611212953074644777149423343867876705824452296605797007134458987594126654609414211447540007211790607458330686866231309155780005966522736183536340439991445294960728379007338249976020630448806064574892740547730693971337007962746135534442514745423654662752252624869916077111131569725392943756732215758704952417232428206555322808868670153681482911738542735797154157943689491063759749151524510096986573825654899585216747260540468342338610760823605782941948009334370046866568258579827323875158302566720152604684361412652956519894291184887986819088277339147282063794512260294515707367105637720023427811802621502691790400488001808901847311751199425460594416773315777951735444490965752131026306836047140331442314298077895617051256930051804287472368435536402764392777908638966566390166776625678575354239947427919442544664643315554138265543388487778859972063679660692327601733858843763144148113561693030468420017434061395220072403658812798249143261731617813894970955038369479594617979829257740992171922783223006387384996138434398468502234780438733784470928703890536420557474836284616809363650973790900204118525835525201575239280826462555785658190226958376345342663420946214426672453987171047721482128157607275305173330963455909323664528978019175132987747952929099598069790148515839540444283988381797511245355548426126784217797728268989735007954505834273726937288386902125284843370917479603207479554080911491866208687184899550445210616155437083299502854903659617362726552868081324793106686855857401668022408227992433394360936223390321499357262507480617409173636062365464458476384647869520547719533384203403990244761056010612777546471464177412625548519830144627405538601855708359981544891286863480720710061787059669365218674805943569985859699554089329219507269337550235821561424994538234781138316591662683103065194730233419384164076823699357668723462219641322516076261161976034708844046473083172682611277723613381938490606534404043904909864126903479263503943531836741051762565704797064478004684323069430241749029731181951132935746854550484711078742905499870600373983113761544808189067620753424526993443755719446665453524088287267537759197074526286322840219629557247932987132852479994638938924943286917770190128914220188747760484939855471168524810559991574441551507431214406120333762869533792439547155394213121021954430556748370425907553004950664994802614794524739012802842646689229455664958621308118913500279654910344806150170407268010067948926855360944990373928383520627992820181576427054962997401900837493444950600754365525758905546552402103412862124809003162941975876195941956592556732874237856112669741771367104424821916671499611728903944393665340294226514575682907490402153401026923964977275904729573320027982816062130523130658731513076913832317193626664465502290735017347656293033318520949298475227462534564256702254695786484819977513326393221579478212493307051107367474918016345667888810782101151826314878755138027101379868751299375133303843885631415175908928986956197561123025310875057188962535763225834275763348421016668109884514141469311719314272028007223449941999003964948245457520704922091620614222912795322688239046498239081592961111003756999529251250673688233852648213896986384052437049402152187547825163347082430303521036927849762517317825860862215614519165573478940019558704784741658847364803865995119651409542615026615147651220820245816010801218275982577477652393859159165067449846149161165153821266726927461290533753163055654440793427876550267301214578324885948736899073512166118397877342715872870912311383472485146035661382188014840560716074652441118841800734067898587159273982452147328317214621907330492060817440914125388918087968538960627860118193099489240811702350413554126823863744341209267781729790694714759018264824761112414556423937732224538665992861551475342773370683344173073150805440138894084087253197595538897613986400165639906934600670780501058567196636796167140097031535132386972899001749862948883362389858632127176571330142071330179992326381982094042993377790345261665892577931395405145369730429462079488033141099249907113241694504241391265397274078984953073730364134893688060340009640631540701820289244667315059736321311926231179142794944897281477264038321021720718017561601025111179022163703476297572233435788863537030535008357679180120653016668316780269873860755423748298548246360981608957670421903145684942967286646362305101773132268579232832164818921732941553151386988781837232271364011755881332524294135348699384658137175857614330952147617551708342432434174779579226338663454959438736807839569911987059388085500837507984051126658973018149321061950769007587519836861526164087252594820126991923916722273718430385263107266000047367872474915828601694439920041571102706081507270147619679971490141639274282889578424398001497985658130305740620028554097382687819891158955487586486645709231721825870342960508203415938806006561845735081804032347750084214100574577342802985404049555529215986404933246481040773076611691605586804857302606467764258503301836174306413323887707999698641372275526317649662882467901094531117120243890323410259937511584651917675138077575448307953064925086002835629697045016137935696266759775923436166369375035368699454550392874449940328328128905560530091416446608691247256021455381248285307613556149618444364923014290938289373215312818797541139219415606631622784836152140668972661027123715779503062132916001988806369127647416567067485490795342762338253943990022498972883660263920518704790601584084302914787302246651371144395418253441269003331181914268070735159284180415100555199146564934872796969351992963117195821262627236458009708099166752820365818699111948365866102758375863322993225541477479210421324166848264953111826527351008031659958888814809945737293785681411438021523876706455063233067233939551964260397443829874822322662036352861302543796600943104500158604854027036789711934695579989189112302233381602302236277726084846296189550730850698061500281436425336666311433321645213882557346329366870956708432252564333895997812402164189946978348320376011613913855499933990786652305860332060641949298931012423081105800169745975038516887112037747631577311831360002742502722451570906304496369230938382329175076469684003556425503797106891999812319602533733677437970687713814747552190142928586781724044248049323750330957002929126630316970587409214456472022710796484778657310660832173093768033821742156446602190335203981531618935787083561603302255162155107179460621892674335641960083663483835896703409115513087820138723494714321400450513941428998350576038799343355677628023346565854351219361896876831439866735726040869511136649881229957801618882834124004126142251475184552502502640896823664946401177803776799157180146386554733265278569418005501363433953502870836220605121839418516239153709790768084909674194289061134979961034672077354959593868862427986411437928435620575955500144308051267664432183688321434583708549082240014585748228606859593502657405750939203135881722442164955416889785558265198046245527898343289578416968890756237467281044803018524217706136533236073856228166664597654076844715963930782091017090763377917711485205493367936868430832404126789220929930411890501756484917499452393770674524578019171841679541825554377930299249277892416277257788147974770446005423669346157135208417428211847353652367573702352791459837645712257646122605628127852169580892808988394594406165340521932514843306105322700231133680378433377389724881307874325614952744243584753011150345103737688223837573804282007358586938044331529253129961025096113761670187568525921208929131354473196308440066835155160913925692912175784379179004808848023029304392630921342768601226558630456913133560978156776098711809238440656353136182676923761613389237802972720736243967239854144480757286813436768000573823963610796223140429490728058551444771338682314499547929338131259971996894072233847404542592316639781608209399269744676323921370773991899853301483814622364299493902073285072098040905300059160091641710175605409814301906444379905831277826625762288108104414704097708248077905168225857235732665234414956169007985520848841886027352780861218049418060017941147110410688703738674378147161236141950474056521041002268987858525470689031657094677131822113205505046579701869337769278257145248837213394613987859786320048011792814546859096532616616068403160077901584946840224344163938313618742275417712170336151163782359059685168880561304838542087505126933144171705880517278127917564053282929427357971823360842784676292324980318169828654166132873909074116734612367109059236155113860447246378721244612580406931724769152219217409096880209008801535633471775664392125733993165330324425899852598966724744126503608416484160724482125980550754851232313331300621490042708542735985913041306918279258584509440150719217604794274047740253314305451367710311947544521321732225875550489799267468541529538871443696399406391099267018219539890685186755868574434469213792094590683677929528246795437302263472495359466300235998990248299853826140395410812427393530207575128774273992824866921285637240069184859771126480352376025469714309316636539718514623865421671429236191647402172547787238964043145364190541101514371773797752463632741619269990461595895793940622986041489302535678633503526382069821487003578061101552210224486633247184367035502326672749787730470216165019711937442505629639916559369593557640005236360445141148916155147776301876302136068825296274460238077523189646894043033182148655637014692476427395401909403584437251915352134557610698046469739424511797999048754951422010043090235713636892619493763602673645872492900162675597083797995647487354531686531900176427222751039446099641439322672532108666047912598938351926694497553568096931962642014042788365702610390456105151611792018698900673027082384103280213487456720062839744828713298223957579105420819286308176631987048287388639069922461848323992902685392499812367091421613488781501234093387999776097433615750910992585468475923085725368613605356762146929424264323906626708602846163376051573599050869800314239735368928435294958099434465414316189806451480849292695749412903363373410480943579407321266012450796613789442208485840536446021616517885568969302685188950832476793300404851688934411125834396590422211152736276278672366665845757559585409486248261694480201791748223085835007862255216359325125768382924978090431102048708975715033330963651576804501966025215527080352103848176167004443740572131294252820989545456276344353575741673638980108310579931697917916718271145837435222026387771805250290791645414791173616253155840768495583288190293564201219633684854080865928095131505012602919562576032932512847250469881908146475324342363863860247943921015193235101390117789997483527186469346024554247028375300033725403910085997650987642832802908445662021678362267272292737780213652404028817217012490974899454430826861772239385250883760749742195942655217301733355851389407457348144161511380845358039740277795072051893487170722955427683655826706766313911972211811528466502223383490906676554168336907959409404576472940901354356409277969379842065738891481990225399022315913388145851487225126560927576795873759207013915029216513720851137197522734365458411622066281660256333632074449918511469174455062297146086578736313585389023662557285424516018080487167823688885575325066254262367702604215835160174851981885460860036597606743233346410471991027562358645341748631726556391320606407754779439671383653877377610828300019937359760370467245737880967939894493795829602910746901609451288456550071458091887879542641820145369659962842686882363495879277007025298960996798975941955735253914237782443302746708282008722602053415292735847582937522487377937899136764642153727843553986244015856488692101644781661602962113570056638347990334049623875941092886778920270077504951511405782565295015024484968204744379710872943108541684540513016310902267112951959140520827546866418137305837933236150599142045255880213558474751516267815309465541240524091663857551298894834797423322854504140527354235070335984964593699534959698554244978249586929179182415068053002553370412778703476446244329205906832901886692400222391918714603175399666877477960121790688623311002908668305431787009355066944389131913333586368037447530664502418437136030852288582121720231274167009740351431532131803978033680228154223490183737494117973254478594157962104378787072154814091725163615415163381388912588517924237727229603497305533840942889918919161186249580560073570527227874940321250645426206304469470804277945973817146810395192821550688079136701210109944220737024613687196031491162370967939354636396448139025711768057799751751298979667073292674886430097398814873780767363792886767781170520534367705731566895899181530825761606591843760505051704242093231358724816618683821026679970982966436224723644898648976857100173643547336955619347638598187756855912376232580849341570570863450733443976604780386678461711520325115528237161469200634713570383377229877321365028868868859434051205798386937002783312365427450532283462669786446920780944052138528653384627970748017872477988461146015077617116261800781557915472305214759943058006652042710117125674185860274188801377931279938153727692612114066810156521441903567333926116697140453812010040811760123270513163743154487571768761575554916236601762880220601068655524141619314312671535587154866747899398685510873576261006923021359580838145290642217792987748784161516349497309700794368305080955621264592795333690631936594413261117944256602433064619312002953123619348034504503004315096798588111896950537335671086336886944665564112662287921812114121425167348136472449021275252555647623248505638391391630760976364990288930588053406631352470996993362568102360392264043588787550723319888417590521211390376609272658409023873553418516426444865247805763826160023858280693148922231457758783791564902227590699346481624734399733206013058796068136378152964615963260698744961105368384203105364183675373594176373955988088591188920114871545460924735613515979992999722298041707112256996310945945097765566409972722824015293663094891067963296735505830412258608050740410916678539569261234499102819759563955711753011823480304181029089719655278245770283085321733741593938595853203645590564229716679900322284081259569032886928291260139267587858284765599075828016611120063145411315144108875767081854894287737618991537664505164279985451077400771946398046265077776614053524831090497899859510873112620613018757108643735744708366215377470972660188656210681516328000908086198554303597948479869789466434027029290899143432223920333487108261968698934611177160561910681226015874410833093070377506876977485840324132474643763087889666151972556180371472590029550718424245405129246729039791532535999005557334600111693557020225722442772950263840538309433999383388018839553821540371447394465152512354603526742382254148328248990134023054550811390236768038649723899924257800315803725555410178461863478690646045865826036072306952576113184134225274786464852363324759102670562466350802553058142201552282050989197818420425028259521880098846231828512448393059455162005455907776121981297954040150653985341579053629101777939776957892084510979265382905626736402636703151957650493344879513766262192237185642999150828898080904189181015450813145034385734032579549707819385285699926238835221520814478940626889936085239827537174490903769904145555260249190126341431327373827075950390882531223536876389814182564965563294518709637484074360669912550026080424160562533591856230955376566866124027875883101021495284600804805028045254063691285010599912421270508133194975917146762267305044225075915290251742774636494555052325186322411388406191257012917881384181566918237215400893603475101448554254698937834239606460813666829750019379115061709452680984785152862123171377897417492087541064556959508967969794980679770961683057941674310519254486327358885118436597143583348756027405400165571178309126113117314169066606067613797690123141099672013123730329707678988740099317309687380126740538923612230370779727025191340850390101739924877352408881040807749924412635346413181858792480760553268122881584307471326768283097203149049868884456187976015468233715478415429742230166504759393312132256510189175368566338139736836336126010908419590215582111816677413843969205870515074254852744810154541079359513596653630049188769523677579147319184225806802539818418929888943038224766186405856591859943091324575886587044653095332668532261321209825839180538360814144791320319699276037194760191286674308615217243049852806380129834255379486287824758850820609389214668693729881191560115633701248675404205911464930888219050248857645752083363921499441937170268576222251074166230901665867067714568862793343153513505688216165112807318529333124070912343832502302341169501745502360505475824093175657701604884577017762183184615567978427541088499501610912720817913532406784267161792013428902861583277304794830971705537485109380418091491750245433432217445924133037928381694330975012918544596923388733288616144238100112755828623259628572648121538348900698511503485369544461542161283241700533583180520082915722904696365553178152398468725451306350506984981006205514844020769539324155096762680887603572463913955278222246439122592651921288446961107463586148252820017348957533954255019475442643148903233373926763409115527189768429887783617346613535388507656327107814312435018965109238453660236940276060642119384227665755210663671879603217527184404651560427289869560206997012906367847161654793068868305846508082886614111979138822898112498261434559408961813509226857611474609406147937240008842153535862052780125014270055274468359151840373309373580494342483940467505708347927948338133276237937844629209323999417593374917899786484958148818865149169302451512835579818112344900827168644548306546633975256079615935830821400021951611342337058359111545217293721664061708131602078213341260356852013161345136871600980378712556766143923146458085652084039744217352744813741215277475202259244561520365608268890193913957991844109971588312780020898275935898106482117936157951837937026741451400902833064466209280549839169261068975151083963132117128513257434964510681479694782619701483204392206140109523453209269311762298139422044308117317394338867965739135764377642819353621467837436136161591167926578700137748127848510041447845416464568496606699139509524527949914769441031612575776863713634644477006787131066832417871556281779122339077841275184193161188155887229676749605752053192594847679397486414128879475647133049543555044790277128690095643357913405127375570391806822344718167939329121448449553897728696601037841520390662890781218240141299368590465146519209198605347788576842696538459445700169758422531241268031418456268722581132040056433413524302102739213788415250475704533878002467378571470021087314693254557923134757243640544448132093266582986850659125571745568328831440322798049274104403921761438405750750288608423536966715191668510428001748971774811216784160854454400190449242294333666338347684438072624307319019363571067447363413698467328522605570126450123348367412135721830146848071241856625742852208909104583727386227300781566668914250733456373259567253354316171586533339843321723688126003809020585719930855573100508771533737446465211874481748868710652311198691114058503492239156755462142467550498676710264926176510110766876596258810039163948397811986615585196216487695936398904500383258041054420595482859955239065758108017936807080830518996468540836412752905182813744878769639548306385089756146421874889271294890398025623046812175145502330254086076115859321603465240763923593699949180470780496764486889980902123735780457040380820770357387588525976042434608851075199334470112741787878845674656640471901619633546770714090590826954225196409446319547658653032104723804625249971910690110456227579220926904132753699634145768795242244563973018311291451151322757841320376225862458224784696669785947914981610522628786944136373683125108310682898766123782697506343047263278453719024447970975017396831214493357290791648779915089163278018852504558488782722376705263811803792477835540018117452957747339714012352011459901984753358434861297092928529424139865507522507808919352104173963493428604871342370429572757862549365917805401652536330410692033704691093097588782938291296447890613200063096560747882082122140978472301680600835812336957051454650181292694364578357815608503303392466039553797630836137289498678842851139853615593352782103740733076818433040893624460576706096188294529171362940967592507631348636606011346115980434147450705511490716640635688739020690279453438236930531133440901381392849163507484449076828386687476663619303412376248380175840467851210698290605196112357188811150723607303158506622574566366740720668999061320627793994112805759798332878792144188725498543014546662945079670707688135022230580562225942983096887732856788971494623888272184647618153045844390967248232348259587963698908456664795754200195991919240707615823002328977439748112690476546256873684352229063217889227643289360535947903046811114130586348244566489159211382258867880972564351646404364328416076247766114349880319792230537889671148058968061594279189647401954989466232962162567264739015818692956765601444248501821713300527995551312539849919933907083138030214072556753022600033565715934283182650908979350869698950542635843046765145668997627989606295925119763672907762567862769469947280606094290314917493590511523235698715397127866718077578671910380368991445381484562682604003456798248689847811138328054940490519768008320299631757043011485087384048591850157264392187414592464617404735275250506783992273121600117160338604710710015235631159734711153198198710616109850375758965576728904060387168114313084172893710817412764581206119054145955378853200366615264923610030157044627231777788649806700723598889528747481372190175074700005571108178930354895017924552067329003818814068686247959272205591627902292600592107710510448103392878991286820705448979977319695574374529708195463942431669050083984398993036790655541596099324867822475424361758944371791403787168166189093900243862038610001362193667280872414291108080291896093127526202667881902085595708111853836166128848729527875143202956393295910508349687029060692838441522579419764824996318479414814660898281725690484184326061946254276693688953540732363428302189694947766126078346328490315128061501009539164530614554234923393806214007779256337619373052025699319099789404390847443596972052065999017828537676265683558625452697455260991024576619614037537859594506363227095122489241931813728141668427013096050734578659047904243852086508154491350136491698639048125666610843702294730266721499164849610746803261583352580352858275799038584091667618877199539888680431991650866887781701439663176815592262016991396613153738021294160006906947533431677802632207226265881842757216055461439677336258462997385077307751473833315101468395296411397329672457933540390136107395245686243008096720460995545708974893048753897955544443791303790422346037768729236001386569593952300768091377768847789746299699489949016141866131552200856673695770822720338936659590666350594330040363762591189195691561626122704788696510356062748423100605472091437069471661080277379848576543481249822444235828329813543645124092220896643987201997945619030397327254617823136363375927622656301565813545578319730419339269008282952718252138855126583037630477490625995514925943105307478901043009876580816508144862607975129633326675259272351611791836777128931053144471668835182920514343609292493191180249366051791485330421043899773019267686085347768149502299280938065840007311767895491286098112311307002535600347898600653805084532572431553654422067661352337408211307834360326940015926958459588297845649462271300855594293344520727007718206398887404742186697709349647758173683580193168322111365547392288184271373843690526638607662451284299368435082612881367358536293873792369928837047900484722240370919885912556341130849457067599032002751632513926694249485692320904596897775676762684224768120033279577059394613185252356456291805905295974791266162882381429824622654141067246487216174351317397697122228010100668178786776119825961537643641828573481088089988571570279722274734750248439022607880448075724807701621064670166965100202654371260046641935546165838945950143502160890185703558173661823437491622669077311800121188299737319891006060966841193266075165452741829459541189277264192546108246351931647783837078295218389645376236304858042774417907169146356546201215125418664885396161542055152375000426794253417764590821513675258479774465114750438460596325820468809667795709044645884673847481638045635188183210386594798204376334738389017759714236223057776395541011294523488098341476645559342209402059733452337956309441446698222457026367119493286653989491344225517746402732596722993581333110831711807234044326813737231209669052411856734897392234152750707954137453460386506786693396236535556479102508529284294227710593056660625152290924148057080971159783458351173168204129645967070633303569271821496292272073250126955216172649821895790908865085382490848904421755530946832055636316431893917626269931034289485184392539670922412565933079102365485294162132200251193795272480340133135247014182195618419055761030190199521647459734401211601239235679307823190770288415814605647291481745105388060109787505925537152356112290181284710137917215124667428500061818271276125025241876177485994084521492727902567005925854431027704636911098800554312457229683836980470864041706010966962231877065395275783874454229129966623016408054769705821417128636329650130416501278156397799631957412627634011130135082721772287129164002237230234809031485343677016544959380750634285293053131127965945266651960426350406454862543383772209428482543536823186182982713182489884498260285705690699045790998144649193654563259496570044689011049923939218088155626191834404362264965506449848521612498442375928443642612004256628602157801140467879662339228190804577624109076487087406157070486658398144845855803277997327929143195789110373530019873110486895656281917362036703039179710646309906285483702836118486672219457621775034511770110458001291255925462680537427727378863726783016568351092332280649908459179620305691566806180826586923920561895421631986004793961133953226395999749526798801074576466538377400437463695133685671362553184054638475191646737948743270916620098057717103475575333102702706317395612448413745782734376330101853438497450236265733191742446567787499665000938706441886733491099877926005340862442833450486907338279348425305698737469497333364267191968992849534561045719338665222471536681145666596959735075972188416698767321649331898967182978657974612216573922404856900225324160367805329990925438960169901664189038843548375648056012628830409421321300206164540821986138099462721214327234457806819925823202851398237118926541234460723597174777907172041523181575194793527456442984630888846385381068621715274531612303165705848974316209831401326306699896632888532682145204083110738032052784669279984003137878996525635126885368435559620598057278951754498694219326972133205286374577983487319388899574634252048213337552584571056619586932031563299451502519194559691231437579991138301656117185508816658756751184338145761060365142858427872190232598107834593970738225147111878311540875777560020664124562293239116606733386480367086953749244898068000217666674827426925968686433731916548717750106343608307376281613984107392410037196754833838054369880310983922140260514297591221159148505938770679068701351029862207502287721123345624421024715163941251258954337788492834236361124473822814504596821452253550035968325337489186278678359443979041598043992124889848660795045011701169092519383155609441705397900600291315024253848282782826223304151370929502192196508374714697845805550615914539506437316401173317807741497557116733034632008408954066541694665746735785483133770133628948904397670025863002540635264006601631712883920305576358989492412827022489373848906764385339931878608019223108328847459816417701264089078551777830131616162049792779670521847212730327970738223860581986744668610994383049960437407323195784473254857416239738852016202384784256163512597161783106850156299135559874758848151014815490937380933394074455700842090155903853444962128368313687375166780513082594599771257467939781491953642874321122421579851584491669362551569370916855252644720786527971466476760328471332985501945689772758983450586004316822658631176606237201721007922216410188299330808409384014213759697185976897042759041500946595252763487628135867117352364964121058854934496645898651826545634382851159137631569519895230262881794959971545221250667461174394884433312659432286710965281109501693028351496524082850120190831078678067061851145740970787563117610746428835593915985421673115153096948758378955979586132649569817205284291038172721213138681565524428109871168862743968021885581515367531218374119972919471325465199144188500672036481975944167950887487934416759598361960010994838744709079104099785974656112459851972157558134628546189728615020774374529539536929655449012953097288963767713353842429715394179547179095580120134210175150931491664699052366350233024087218654727629639065723341455005903913890253699317155917179823065162679744711857951506573868504088229934804445549850597823297898617029498418376255258757455303112991914341109413088238114443068843062655305601658801408561023324210300218460588586954418502977463085858496130037238190325162225570729975710727306066072916922978033647048840958711228045188511908718588299514331534128549297173849768523136276076868494780364948299904475715771141080958058141208956059471668626290036145602625334863284986816039463372436667112964460292915746181117789169695839947080954788863503281129626899231110099889317815313946681882028368363373822281414974006917942192888817139116283910295684918233358930813360131488748366464224381776081007739183393749346933644748150564933649323157235306109385796839902153381449126925350768211098738352197507736653475499431740580563099143218212547336281359488317681489194306530426029773885492974570569448783077945878865062970895499843760181694031056909587141386804846359853684034105948341788438963179956468815791937174656705047441528027712541569401365862097760735632832966564135817028088013546326104892768731829917950379944446328158595181380144716817284996793061814177131912099236282922612543236071226270324572637946863533391758737446552006008819975294017572421299723542069630427857950608911113416534893431149175314953530067419744979017235181671568754163484949491289001739377451431928382431183263265079530371177806185851153508809998200482761808307209649636476943066172549186143700971387567940218696710148540307471561091358933165600167252126542502898612259306484105898847129649230941215144563947889999327145875969555737090855150648002321476443037232466147111552578583071024936898814562568786834745518893385181791667579054210421036349316257870476543126790661216644142285017446278477132740595579600648343288827864837043456066966456899746910373987712891593313271266247505582258634928427718355831641593667712218537642376222104779338956378722902509543014182257180331300148113377736941508488867501893156994849838936052666818012783912005801431596441910546663236810148207799356523056490420711364192200177189107935243234322761787712568251126481332974354926568682748715986654943041648468220593921673359485057849622807932422649812705271398407720995707236227009245067665680069149966555737866411877079767754867028786431817941521796178310655030287157272282250812017060713380339641841211253856248920130010782462165136989511064611133562443838185366273563783436921279354709230119655914915800561707258518503167289370411936374780625824298250726464801821523430268081486978164824349353456855843696378384153838051184406043696871666416514036129729992912630842812149152469877429332305214999981829046119471676727503742221367186614654042534463141660649871499001000660041544868437352208483059495953182872280520828676300361091734508632133033647289584176588755345227938480297724485711815574893561311524926772006362198369980664159549388683836411891430443767715498026544959061738265591178545999378510861446014967645550103653971251138583505085112442517772923814396233043724036032603181442991365750246012787514117944901305803452199992701148071712847770301254994886841867572975189214295652512486943983729047410363121899124217339550688778643130750024823361832738729697376598820053895902935486054979802320400472236873557411858132734337978931582039412878989728973298812553514507641535360519462112217000676321611195841029252568536561813138784086477147099724553013170761712163186600291464501378587854802096244703771373587720086738054108140042311418525803293267396324596914044834665722042880679280616029884043400536534009706581694636096660911110968789751801325224478246957913251892122653056085866541115373584912790254654369020869419871125588453729063224423222287139122012248769976837147645598526739225904997885514250047585260297929306159913444898341973583316070107516452301310796620382579278533125161760789984630103493496981494261055367836366022561213767081421091373531780682420175737470287189310207606953355721704357535177461573524838432101571399813798596607129664438314791296359275429627129436142685922138993054980645399144588692472767598544271527788443836760149912897358259961869729756588978741082189422337344547375227693199222635973520722998387368484349176841191020246627479579564349615012657433845758638834735832242535328142047826934473129971189346354502994681747128179298167439644524956655532311649920677163664580318205849626132234652606175413532444702007661807418914040158148560001030119994109595492321434406067634769713089513389171050503856336503545166431774489640061738861761193622676890576955693918707703942304940038440622614449572516631017080642923345170422426679607075404028551182398361531383751432493056398381877995594942545196756559181968690885283434886050828529642437578712929439366177362830136595872723080969468398938676366226456791132977469812675226595621009318322081754694778878755356188335083870248295346078597023609865656376722755704495258739871812593441903785275571333409842450127258596692434317689018966145404453679047136294238156127656824247864736176671770647002431119711090007474065945650315375044177982192306323700872039212085499569681061379189029961178936752146022386905665481382858280449537530160921422195940638787074787991194920898374091788534417523064715030278397979864517336625329511775105559014160459873338186887977858817291976604516353353556047648420520888811722831990044504284486852338334530105533929637308039738230604714104525470094899407601215247602819963846343554852932377161410869591950786873276075400085220065031871239272857835807010762542769655355964789450166013816295177908531139811092831583216931563867459747449584385282701658246192092219529134323496779345585613140207765996142546463288677356891785576835169608392864188830094883324700447958316931533832382377876344426323456301679513671047510469669001217777128065522453689371871451567394733440447280450959433090683667110655953338602938000999949010642769859623260401863733572846679531229683156358145420890540651226419162015504500430562136991850941034609601030543816694795964585804425194905110733387679946734471718615647723811737035654917628707589456035519195603962301157866323750234725054461073979402475184415558178087962822231972692984516683306919505079993357259165675557294585962182052650473353712351623662770479333289322136141858785972771685682725303734836891911847197133753088446777943274857148827821608844765700041403499921376794209627560883081509438030705666022764678117533361028187800710219794428777313146387857817205661409023041499923248268982477222109852189758140879763486146763606368674611966620347304608917277240045953051376938375381543486981101990651706961774052218247422657652138152740612699012706880875386408669901461740890540981877671880076124151967064152117653084325544261017536348281196837493395825742541244634247233586360777980960199745187758845459645895956779558869098404768259253477849930457883128541747079059795909431627722327844578918694214929451540174214623240300841907975296782445969183509474202123617940309048634960534054931299919496087957952586977170236680033862505764938088740994009589948109397983231108838769236490221499111120870639202892490698435333152727991330986335454324971441378059132240814960156485679843966464780280409057580889190254236606774500413415794312112501275232250148067232979652230488493751166084976116412777395311302041566848265531411348993243747890268935173904043294851610659785832253168204202834993641595980197343889883020994152152288611175126686173051956249367180053845637855129171848417841594797435580617856680758491080185805695567990185198397660693358224779136504562705766735170961550493338390452612404395517449136885115987454340932040102218982707539212403241042424451570052968378815749468441508011138612561164102477190903050040240662278945607061512108266146098662040425010583978098192019726759010749924884966139441184159734610382401178556739080566483321039073867083298691078093495828888707110651559651222542929154212923108071159723275797510859911398076844732639426419452063138217862260999160086752446265457028969067192282283045169111363652774517975842147102219099906257373383472726498678244401048998507631630668050267115944636293525120269424810854530602810627264236538250773340575475701704367039596467715959261029438313074897245505729085688496091346323165819468660587092144653716755655531962091865952628448253731353698162517351930115341581171353292035873164168839107994000677266031617527582917398395852606454113318985505747847121053505795649095931672167565624818782002769963734155880000867852567422461511406015760115910256449002264980039498403358091309140197877843650167960167465370287466062584346329708303725980494653589318912163976013193079476972058034710553111117215859219066231028099212084069283091906017370764654655683413207556315315006453462321007133584907633048328153458698497332599801187479664273140279381289961720524540674695271948079930396730194274036466594154400092799908634806622334906695224044652158992864203435098858422692019340575496840904812955522654754650713532842543496616084954788090727649930252702815067862810825243222979985391759845188868387004477101866772159439708514664612871148749531862180941719676843144666435175837688436786081446319641912566574047718699160915550910878919431253671945651261878486910876729910565595155159739659034383628124629118117760949411880105946336671039049777312004243578115790429823045072038322781246413671297959415082918378213212876890545963586369344879749784841123274921331663162812456388238288715648447883142417650147980187858215768793063001153788998014623690135803753306246148576074932567807682651045738059018831237617271889933790487113395588485234240255002352200613574914318259142479829367775490496399350755839668967578364316618369307625603528602940662803255416535431518013714821941772672244005268401996533334184004345525296592918502940131600651124395297874364222806977720437363717873457948420238745151249157913139411148608416429347958793681868609689684640858334131017858142710955416293375915178392341303110543328703526599993904966822112768158316511246866451167351378214345336650598328347443536290312393672084593164394941881138607974670134709640378534907149089842317891739783650654751982883367395714360000003439863363212091718954899055748693397700245632475954504411422582410783866837655467400137324322809113692670682805397549111166171102397437749479335174036135005397581475520834285772800986189401984375446435081498218360112577632447389452051636938585136484259964518361856989088721789764694721246807900330925083496645841656554261294195108847197209106605105540933731954888406444080280579549008076040034154662137669606444293774985897353625591959618552448187940317374508256072895120945456562159540405425814886929842786582357673195799285293120866275922366115137445767916063621675267440451221051052090834707443986137829082352772895849625656881972792768694795806100573787084121444815034797422312103295359297822377134077549545477791813823542607184617108389097825964406170543546968567030745411634244134486308676327949177682923093183221341455482591367202823284396549001805653203960795517074496039006696990334199278212696767771835209083959545341866777944872740383733381985235884202840150981579594685874537989503257362809837592216229258598599123843993575573285028613155970362934249814178056461615863415338635077223269996508860870999964899373049307170967888740149746147542880387421250689212155876692242387434701120990859082164073576380817386959755176083877600277517253037133445654852635661720197563001580049790223419586738061442401502436288957503206533690825756785507020555105572381878574650371086308158185862815883054564662297694803970618265491385181326737485227188267917919091354407852685476254126683398240534022469989966652573155637645862251862823092085424412805997628505488913098331761884983352975136073772030571342739638126588567405013841074788943393996603591853934198416322617654857376671943132840050626295140357877264680649549355746326408186979718630218760025813995719923601345374229758918285167511358171472625828596940798518571870075823122317068134867930884899275181661399609753105295773584618525865211893339375771859916335112163441037910451845019023066893064178977808158101360449495409665363660370075881004450265734935127707426742578608784898185628869980851665713320835842613381142623855420315774246613108873106318111989880289722849790551075148403702290580483052731884959994156606537314021296702220821915862905952604040620011815269664910068587592655660567562963361434230232810747488395040380984981860056164646099819257616235478710913832967563761506732550860683433720438748186791668975746563456020002562889601191100980453350423842063824039434163502977688802779835087481178298349417211674919425601608685332435385951152061809031241698182079314615062073826097180458265687043623935757495737332781578904386011378078508110273049446611821957450170106059384336519458628360682108585130499820420578458577175933849015564447305834515291412561679970569657426139901681932056241927977282026714297258700193234337873153939403115411184101414292741703537542003698760608765500109345299007034032401334806388514095769557147190364152027721127070187421548123931953220997506553022646844227700020589045922742423904937051507367764629844971682121994198274794049092601715727439368569721862936007387077810797440975556627807371228030350048829843919546433753355787895064018998685060281902452191177018634505171087023903398550540704454189088472042376499749035038518949505897971286631644699407490959473411581934618336692169573605081585080837952036335619947691937965065016808710250735070825260046821242820434367245824478859256555487861614478717581068572356895150707602217433511627331709472765932413249132702425519391509083601346239612335001086614623850633127072987745618984384288764099836164964775714638573247333226653894523588365972955159905187411779288608760239306160016168434070611663449248395156319152882728822831375458678269830696691220130954815935450754923554167766876455212545681242936427474153815692219503331560151614492247512488957534835926226263545406704767033866410025277276800886383266629488582740369655329362236090572479794734434077704284318507901973469071141230364111729224929307731939309795452877412451183953480382210373644697046967493042810911797232448615413264031578430955396671061468083815548947146733652483679138566431084747848676243012018489329109615281108087617422779131629345494425395422727309645057976122885347393189600810965202090151104579377602529543130188938184010247010134929317443562883578609861545691161669857388024973756940558138630581099823372565164920155443216861690537054630176154809626620800633059320775897175589925862195462096455464624399535391743228225433267174308492508396461328929584567927365409119947616225155964704061297047759818551878441419948614013153859322060745185909608884280218943358691959604936409651570327527570641500776261323783648149005245481413195989296398441371781402764122087644989688629798910870164270169014007825748311598976330612951195680427485317886333041169767175063822135213839779138443325644288490872919067009802496281560626258636942322658490628628035057282983101266919109637258378149363774960594515216932644945188292639525772348420077356021656909077097264985642831778694777804964343991762549216500608626285329471055602670413384500507827390640287529864161287496473708235188892189612641279553536442286955430551308700009878557534223100547153412810957024870812654319123261956462149376527526356402127388765103883255007364899937167183280028398832319373301564123277185395654932422977953016534830128490677845037490891749347389015649588574802194996722621185874361039774946338633057887487405540005440439344888192044102134790034598411927024921557026873700970995205391930979319495883265922171508324621942300185974396706491149559411733728199869021311629886680267446443489233020607003821262841723679627307191405008084085703978151998148822390059948911946474438682533745889962375133378280532928272016815977970066488394482446332210928320504045983008943565954267256879714918703447338237767914829203283196838105907715727191903042365315650957464549643425328069510396558733549803850995143463506175361480050195045201350200180281506933241918267855737764414097080945745624854867704904368368717590918057269794010465019484853146726642978667687697789291431128505043098192949736165944259471754765135205245072597538577958372797702972231435199958499522344049394502115428867244188717409524554771867484911475031801773304689909317974472957035192387686405544278134169807249382219749124257510162187439772902147704638010731470653154201300583810458905006764557332998149945854655105526374914354195867992595981412218735238407957416123372264063860431988936249867649693592569592128495906254446474331759999685163660305216426770428154681777589339252115538590526823311608302751194384823861552852465010329467297198112105314125898165100120742688143577590825227466863206188376830450921784582526239594189673003640808624233657620979111641766331328852352062487922978959456450333733139422384778582717195412347860434376165241568717943562570215636666680088531006728947033079540804583324192188488870712275670333173939262509073556164513677064199539111948881240659821685787131385056850623094155206877987539740658484250135205615103489821873770245063583314243624807432542464195984647411575625441010389671576677263196442524931941806472423789334668561083789808830313571333157729435664956078125304917594015895146954965223118559669048559467607968190167266634650186182955669893965019614544401768162810604465068448139561667220729261210164692339016793399632833013163850830967942792934551268435760356901970523138364640961311774904600772840862214747547653221505518116489887879087780918009050706040061220010051271575991225725282523378026809030528461581739558198122397010092017202251606352922464781615533532275453264543087093320924631855976580561717446840450048285353396546862678852330044967795580761661801833668792312510460809773895565488962815089519622093675058841609752282328250433712970186608193748968699961301486924694482420723632912367052542145464162968910442981633373266871675946715392611950649224725627254543274193495995569590243279097174392258098103601486364409101491734183079646345064833303404765711827040276868271418084574998493392039317445402616663674646668754385093967129918067471909885312710726724428584870694307099756567949198418996425748884764622030325637751112534060087936904565779272035205921345924272965206683338510673615276261016026647772485083344719891986802656197236420847504962661607797092906844757798251795569758235084371746103310387911789239441630112634077535773520558040066982523191225570519133631407211349723226549151062961739050617857127509403623146700931176133132018631158730886798239298009805089491510788371194099750375473674305745187265414016446924576792185753680363289139664155342066705623272936001177781498886100830877849571709880858667023104043242526785955562077310543072298032125941107957349146684680220501816192150766649106862033378713826058987655210423668198670177861672671972374156917880001690656659046965316154923604061891820982414006103779407166342002735828911994182647812782659666207030384795881442790246669264032799404016800137293477301530941805070587421153284642203006550763966756168318897005152026656649929417382840327305940740147117478464839241225676523593418554066440983706083636457657081801664285044258224551650808864421212113914352453935225522162483791737330329812349528984098613273709957407786789349311975204237925022851375880436791854547836416773151821457226504640800104202100410766027807729152555503218182387221708112766208665317651926458452495269685376314437998340336947124447247796973890514941120010934140073794061859447165516612674930799374705772930521750426383798367668159183589049652163726492960837147204067428996276720315410211504333742057182854090136325721437592054640471894328548696883599785122262130812989581571391597464534806099601555877223193450760315411663112963843719400333736013305526352571490454327925190794007111504785378036370897340146753465517470747096935814912797188187854376797751675927822300312945518595042883902735494672667647506072643698761394806879080593531793001711000214417701504495496412454361656210150919997862972495905809191825255486358703529320142005857057855419217730505342687533799076038746689684283402648733290888881745453047194740939258407362058242849349024756883352446212456101562729065130618520732925434179252299417447855189995098959999877410951464170076989305620163502192692653166599093238118295411937545448509428621839424186218067457128099385258842631930670182098008050900019819621758458932516877698594110522845465835679362969619219080897536813210484518784516230623911878024604050824909336069998094776253792973597037759066145994638578378211017122446355845171941670344732162722443265914858595797823752976323442911242311368603724514438765801271594060878788638511089680883165505046309006148832545452819908256238805872042843941834687865142541377686054291079721004271658

lib/std/compress/deflate/testdata/compress-gettysburg.txt deleted-29

...	@@ -1,29 +0,0 @@
1	Four score and seven years ago our fathers brought forth on
2	this continent, a new nation, conceived in Liberty, and dedicated
3	to the proposition that all men are created equal.
4	Now we are engaged in a great Civil War, testing whether that
5	nation, or any nation so conceived and so dedicated, can long
6	endure.
7	We are met on a great battle-field of that war.
8	We have come to dedicate a portion of that field, as a final
9	resting place for those who here gave their lives that that
10	nation might live. It is altogether fitting and proper that
11	we should do this.
12	But, in a larger sense, we can not dedicate - we can not
13	consecrate - we can not hallow - this ground.
14	The brave men, living and dead, who struggled here, have
15	consecrated it, far above our poor power to add or detract.
16	The world will little note, nor long remember what we say here,
17	but it can never forget what they did here.
18	It is for us the living, rather, to be dedicated here to the
19	unfinished work which they who fought here have thus far so
20	nobly advanced. It is rather for us to be here dedicated to
21	the great task remaining before us - that from these honored
22	dead we take increased devotion to that cause for which they
23	gave the last full measure of devotion -
24	that we here highly resolve that these dead shall not have
25	died in vain - that this nation, under God, shall have a new
26	birth of freedom - and that government of the people, by the
27	people, for the people, shall not perish from this earth.
28
29	Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania

lib/std/compress/deflate/testdata/compress-pi.txt deleted-1

...	@@ -1 +0,0 @@
1	3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678566722796619885782794848855834397518744545512965634434803966420557982936804352202770984294232533022576341807039476994159791594530069752148293366555661567873640053666564165473217043903521329543529169414599041608753201868379370234888689479151071637852902345292440773659495630510074210871426134974595615138498713757047101787957310422969066670214498637464595280824369445789772330048764765241339075920434019634039114732023380715095222010682563427471646024335440051521266932493419673977041595683753555166730273900749729736354964533288869844061196496162773449518273695588220757355176651589855190986665393549481068873206859907540792342402300925900701731960362254756478940647548346647760411463233905651343306844953979070903023460461470961696886885014083470405460742958699138296682468185710318879065287036650832431974404771855678934823089431068287027228097362480939962706074726455399253994428081137369433887294063079261595995462624629707062594845569034711972996409089418059534393251236235508134949004364278527138315912568989295196427287573946914272534366941532361004537304881985517065941217352462589548730167600298865925786628561249665523533829428785425340483083307016537228563559152534784459818313411290019992059813522051173365856407826484942764411376393866924803118364453698589175442647399882284621844900877769776312795722672655562596282542765318300134070922334365779160128093179401718598599933849235495640057099558561134980252499066984233017350358044081168552653117099570899427328709258487894436460050410892266917835258707859512983441729535195378855345737426085902908176515578039059464087350612322611200937310804854852635722825768203416050484662775045003126200800799804925485346941469775164932709504934639382432227188515974054702148289711177792376122578873477188196825462981268685817050740272550263329044976277894423621674119186269439650671515779586756482399391760426017633870454990176143641204692182370764887834196896861181558158736062938603810171215855272668300823834046564758804051380801633638874216371406435495561868964112282140753302655100424104896783528588290243670904887118190909494533144218287661810310073547705498159680772009474696134360928614849417850171807793068108546900094458995279424398139213505586422196483491512639012803832001097738680662877923971801461343244572640097374257007359210031541508936793008169980536520276007277496745840028362405346037263416554259027601834840306811381855105979705664007509426087885735796037324514146786703688098806097164258497595138069309449401515422221943291302173912538355915031003330325111749156969174502714943315155885403922164097229101129035521815762823283182342548326111912800928252561902052630163911477247331485739107775874425387611746578671169414776421441111263583553871361011023267987756410246824032264834641766369806637857681349204530224081972785647198396308781543221166912246415911776732253264335686146186545222681268872684459684424161078540167681420808850280054143613146230821025941737562389942075713627516745731891894562835257044133543758575342698699472547031656613991999682628247270641336222178923903176085428943733935618891651250424404008952719837873864805847268954624388234375178852014395600571048119498842390606136957342315590796703461491434478863604103182350736502778590897578272731305048893989009923913503373250855982655867089242612429473670193907727130706869170926462548423240748550366080136046689511840093668609546325002145852930950000907151058236267293264537382104938724996699339424685516483261134146110680267446637334375340764294026682973865220935701626384648528514903629320199199688285171839536691345222444708045923966028171565515656661113598231122506289058549145097157553900243931535190902107119457300243880176615035270862602537881797519478061013715004489917210022201335013106016391541589578037117792775225978742891917915522417189585361680594741234193398420218745649256443462392531953135103311476394911995072858430658361935369329699289837914941939406085724863968836903265564364216644257607914710869984315733749648835292769328220762947282381537409961545598798259891093717126218283025848112389011968221429457667580718653806506487026133892822994972574530332838963818439447707794022843598834100358385423897354243956475556840952248445541392394100016207693636846776413017819659379971557468541946334893748439129742391433659360410035234377706588867781139498616478747140793263858738624732889645643598774667638479466504074111825658378878454858148962961273998413442726086061872455452360643153710112746809778704464094758280348769758948328241239292960582948619196670918958089833201210318430340128495116203534280144127617285830243559830032042024512072872535581195840149180969253395075778400067465526031446167050827682772223534191102634163157147406123850425845988419907611287258059113935689601431668283176323567325417073420817332230462987992804908514094790368878687894930546955703072619009502076433493359106024545086453628935456862958531315337183868265617862273637169757741830239860065914816164049449650117321313895747062088474802365371031150898427992754426853277974311395143574172219759799359685252285745263796289612691572357986620573408375766873884266405990993505000813375432454635967504844235284874701443545419576258473564216198134073468541117668831186544893776979566517279662326714810338643913751865946730024434500544995399742372328712494834706044063471606325830649829795510109541836235030309453097335834462839476304775645015008507578949548931393944899216125525597701436858943585877526379625597081677643800125436502371412783467926101995585224717220177723700417808419423948725406801556035998390548985723546745642390585850216719031395262944554391316631345308939062046784387785054239390524731362012947691874975191011472315289326772533918146607300089027768963114810902209724520759167297007850580717186381054967973100167870850694207092232908070383263453452038027860990556900134137182368370991949516489600755049341267876436746384902063964019766685592335654639138363185745698147196210841080961884605456039038455343729141446513474940784884423772175154334260306698831768331001133108690421939031080143784334151370924353013677631084913516156422698475074303297167469640666531527035325467112667522460551199581831963763707617991919203579582007595605302346267757943936307463056901080114942714100939136913810725813781357894005599500183542511841721360557275221035268037357265279224173736057511278872181908449006178013889710770822931002797665935838758909395688148560263224393726562472776037890814458837855019702843779362407825052704875816470324581290878395232453237896029841669225489649715606981192186584926770403956481278102179913217416305810554598801300484562997651121241536374515005635070127815926714241342103301566165356024733807843028655257222753049998837015348793008062601809623815161366903341111386538510919367393835229345888322550887064507539473952043968079067086806445096986548801682874343786126453815834280753061845485903798217994599681154419742536344399602902510015888272164745006820704193761584547123183460072629339550548239557137256840232268213012476794522644820910235647752723082081063518899152692889108455571126603965034397896278250016110153235160519655904211844949907789992007329476905868577878720982901352956613978884860509786085957017731298155314951681467176959760994210036183559138777817698458758104466283998806006162298486169353373865787735983361613384133853684211978938900185295691967804554482858483701170967212535338758621582310133103877668272115726949518179589754693992642197915523385766231676275475703546994148929041301863861194391962838870543677743224276809132365449485366768000001065262485473055861598999140170769838548318875014293890899506854530765116803337322265175662207526951791442252808165171667766727930354851542040238174608923283917032754257508676551178593950027933895920576682789677644531840404185540104351348389531201326378369283580827193783126549617459970567450718332065034556644034490453627560011250184335607361222765949278393706478426456763388188075656121689605041611390390639601620221536849410926053876887148379895599991120991646464411918568277004574243434021672276445589330127781586869525069499364610175685060167145354315814801054588605645501332037586454858403240298717093480910556211671546848477803944756979804263180991756422809873998766973237695737015808068229045992123661689025962730430679316531149401764737693873514093361833216142802149763399189835484875625298752423873077559555955465196394401821840998412489826236737714672260616336432964063357281070788758164043814850188411431885988276944901193212968271588841338694346828590066640806314077757725705630729400492940302420498416565479736705485580445865720227637840466823379852827105784319753541795011347273625774080213476826045022851579795797647467022840999561601569108903845824502679265942055503958792298185264800706837650418365620945554346135134152570065974881916341359556719649654032187271602648593049039787489589066127250794828276938953521753621850796297785146188432719223223810158744450528665238022532843891375273845892384422535472653098171578447834215822327020690287232330053862163479885094695472004795231120150432932266282727632177908840087861480221475376578105819702226309717495072127248479478169572961423658595782090830733233560348465318730293026659645013718375428897557971449924654038681799213893469244741985097334626793321072686870768062639919361965044099542167627840914669856925715074315740793805323925239477557441591845821562518192155233709607483329234921034514626437449805596103307994145347784574699992128599999399612281615219314888769388022281083001986016549416542616968586788372609587745676182507275992950893180521872924610867639958916145855058397274209809097817293239301067663868240401113040247007350857828724627134946368531815469690466968693925472519413992914652423857762550047485295476814795467007050347999588867695016124972282040303995463278830695976249361510102436555352230690612949388599015734661023712235478911292547696176005047974928060721268039226911027772261025441492215765045081206771735712027180242968106203776578837166909109418074487814049075517820385653909910477594141321543284406250301802757169650820964273484146957263978842560084531214065935809041271135920041975985136254796160632288736181367373244506079244117639975974619383584574915988097667447093006546342423460634237474666080431701260052055928493695941434081468529815053947178900451835755154125223590590687264878635752541911288877371766374860276606349603536794702692322971868327717393236192007774522126247518698334951510198642698878471719396649769070825217423365662725928440620430214113719922785269984698847702323823840055655517889087661360130477098438611687052310553149162517283732728676007248172987637569816335415074608838663640693470437206688651275688266149730788657015685016918647488541679154596507234287730699853713904300266530783987763850323818215535597323530686043010675760838908627049841888595138091030423595782495143988590113185835840667472370297149785084145853085781339156270760356390763947311455495832266945702494139831634332378975955680856836297253867913275055542524491943589128405045226953812179131914513500993846311774017971512283785460116035955402864405902496466930707769055481028850208085800878115773817191741776017330738554758006056014337743299012728677253043182519757916792969965041460706645712588834697979642931622965520168797300035646304579308840327480771811555330909887025505207680463034608658165394876951960044084820659673794731680864156456505300498816164905788311543454850526600698230931577765003780704661264706021457505793270962047825615247145918965223608396645624105195510522357239739512881816405978591427914816542632892004281609136937773722299983327082082969955737727375667615527113922588055201898876201141680054687365580633471603734291703907986396522961312801782679717289822936070288069087768660593252746378405397691848082041021944719713869256084162451123980620113184541244782050110798760717155683154078865439041210873032402010685341947230476666721749869868547076781205124736792479193150856444775379853799732234456122785843296846647513336573692387201464723679427870042503255589926884349592876124007558756946413705625140011797133166207153715436006876477318675587148783989081074295309410605969443158477539700943988394914432353668539209946879645066533985738887866147629443414010498889931600512076781035886116602029611936396821349607501116498327856353161451684576956871090029997698412632665023477167286573785790857466460772283415403114415294188047825438761770790430001566986776795760909966936075594965152736349811896413043311662774712338817406037317439705406703109676765748695358789670031925866259410510533584384656023391796749267844763708474978333655579007384191473198862713525954625181604342253729962863267496824058060296421146386436864224724887283434170441573482481833301640566959668866769563491416328426414974533349999480002669987588815935073578151958899005395120853510357261373640343675347141048360175464883004078464167452167371904831096767113443494819262681110739948250607394950735031690197318521195526356325843390998224986240670310768318446607291248747540316179699411397387765899868554170318847788675929026070043212666179192235209382278788809886335991160819235355570464634911320859189796132791319756490976000139962344455350143464268604644958624769094347048293294140411146540923988344435159133201077394411184074107684981066347241048239358274019449356651610884631256785297769734684303061462418035852933159734583038455410337010916767763742762102137013548544509263071901147318485749233181672072137279355679528443925481560913728128406333039373562420016045664557414588166052166608738748047243391212955877763906969037078828527753894052460758496231574369171131761347838827194168606625721036851321566478001476752310393578606896111259960281839309548709059073861351914591819510297327875571049729011487171897180046961697770017913919613791417162707018958469214343696762927459109940060084983568425201915593703701011049747339493877885989417433031785348707603221982970579751191440510994235883034546353492349826883624043327267415540301619505680654180939409982020609994140216890900708213307230896621197755306659188141191577836272927461561857103721724710095214236964830864102592887457999322374955191221951903424452307535133806856807354464995127203174487195403976107308060269906258076020292731455252078079914184290638844373499681458273372072663917670201183004648190002413083508846584152148991276106513741539435657211390328574918769094413702090517031487773461652879848235338297260136110984514841823808120540996125274580881099486972216128524897425555516076371675054896173016809613803811914361143992106380050832140987604599309324851025168294467260666138151745712559754953580239983146982203613380828499356705575524712902745397762140493182014658008021566536067765508783804304134310591804606800834591136640834887408005741272586704792258319127415739080914383138456424150940849133918096840251163991936853225557338966953749026620923261318855891580832455571948453875628786128859004106006073746501402627824027346962528217174941582331749239683530136178653673760642166778137739951006589528877427662636841830680190804609849809469763667335662282915132352788806157768278159588669180238940333076441912403412022316368577860357276941541778826435238131905028087018575047046312933353757285386605888904583111450773942935201994321971171642235005644042979892081594307167019857469273848653833436145794634175922573898588001698014757420542995801242958105456510831046297282937584161162532562516572498078492099897990620035936509934721582965174135798491047111660791587436986541222348341887722929446335178653856731962559852026072947674072616767145573649812105677716893484917660771705277187601199908144113058645577910525684304811440261938402322470939249802933550731845890355397133088446174107959162511714864874468611247605428673436709046678468670274091881014249711149657817724279347070216688295610877794405048437528443375108828264771978540006509704033021862556147332117771174413350281608840351781452541964320309576018694649088681545285621346988355444560249556668436602922195124830910605377201980218310103270417838665447181260397190688462370857518080035327047185659499476124248110999288679158969049563947624608424065930948621507690314987020673533848349550836366017848771060809804269247132410009464014373603265645184566792456669551001502298330798496079949882497061723674493612262229617908143114146609412341593593095854079139087208322733549572080757165171876599449856937956238755516175754380917805280294642004472153962807463602113294255916002570735628126387331060058910652457080244749375431841494014821199962764531068006631183823761639663180931444671298615527598201451410275600689297502463040173514891945763607893528555053173314164570504996443890936308438744847839616840518452732884032345202470568516465716477139323775517294795126132398229602394548579754586517458787713318138752959809412174227300352296508089177705068259248822322154938048371454781647213976820963320508305647920482085920475499857320388876391601995240918938945576768749730856955958010659526503036266159750662225084067428898265907510637563569968211510949669744580547288693631020367823250182323708459790111548472087618212477813266330412076216587312970811230758159821248639807212407868878114501655825136178903070860870198975889807456643955157415363193191981070575336633738038272152798849350397480015890519420879711308051233933221903466249917169150948541401871060354603794643379005890957721180804465743962806186717861017156740967662080295766577051291209907944304632892947306159510430902221439371849560634056189342513057268291465783293340524635028929175470872564842600349629611654138230077313327298305001602567240141851520418907011542885799208121984493156999059182011819733500126187728036812481995877070207532406361259313438595542547781961142935163561223496661522614735399674051584998603552953329245752388810136202347624669055816438967863097627365504724348643071218494373485300606387644566272186661701238127715621379746149861328744117714552444708997144522885662942440230184791205478498574521634696448973892062401943518310088283480249249085403077863875165911302873958787098100772718271874529013972836614842142871705531796543076504534324600536361472618180969976933486264077435199928686323835088756683595097265574815431940195576850437248001020413749831872259677387154958399718444907279141965845930083942637020875635398216962055324803212267498911402678528599673405242031091797899905718821949391320753431707980023736590985375520238911643467185582906853711897952626234492483392496342449714656846591248918556629589329909035239233333647435203707701010843880032907598342170185542283861617210417603011645918780539367447472059985023582891833692922337323999480437108419659473162654825748099482509991833006976569367159689364493348864744213500840700660883597235039532340179582557036016936990988671132109798897070517280755855191269930673099250704070245568507786790694766126298082251633136399521170984528092630375922426742575599892892783704744452189363203489415521044597261883800300677617931381399162058062701651024458869247649246891924612125310275731390840470007143561362316992371694848132554200914530410371354532966206392105479824392125172540132314902740585892063217589494345489068463993137570910346332714153162232805522972979538018801628590735729554162788676498274186164218789885741071649069191851162815285486794173638906653885764229158342500673612453849160674137340173572779956341043326883569507814931378007362354180070619180267328551191942676091221035987469241172837493126163395001239599240508454375698507957046222664619000103500490183034153545842833764378111988556318777792537201166718539541835984438305203762819440761594106820716970302285152250573126093046898423433152732131361216582808075212631547730604423774753505952287174402666389148817173086436111389069420279088143119448799417154042103412190847094080254023932942945493878640230512927119097513536000921971105412096683111516328705423028470073120658032626417116165957613272351566662536672718998534199895236884830999302757419916463841427077988708874229277053891227172486322028898425125287217826030500994510824783572905691988555467886079462805371227042466543192145281760741482403827835829719301017888345674167811398954750448339314689630763396657226727043393216745421824557062524797219978668542798977992339579057581890622525473582205236424850783407110144980478726691990186438822932305382318559732869780922253529591017341407334884761005564018242392192695062083183814546983923664613639891012102177095976704908305081854704194664371312299692358895384930136356576186106062228705599423371631021278457446463989738188566746260879482018647487672727222062676465338099801966883680994159075776852639865146253336312450536402610569605513183813174261184420189088853196356986962795036738424313011331753305329802016688817481342988681585577810343231753064784983210629718425184385534427620128234570716988530518326179641178579608888150329602290705614476220915094739035946646916235396809201394578175891088931992112260073928149169481615273842736264298098234063200244024495894456129167049508235812487391799648641133480324757775219708932772262349486015046652681439877051615317026696929704928316285504212898146706195331970269507214378230476875280287354126166391708245925170010714180854800636923259462019002278087409859771921805158532147392653251559035410209284665925299914353791825314545290598415817637058927906909896911164381187809435371521332261443625314490127454772695739393481546916311624928873574718824071503995009446731954316193855485207665738825139639163576723151005556037263394867208207808653734942440115799667507360711159351331959197120948964717553024531364770942094635696982226673775209945168450643623824211853534887989395673187806606107885440005508276570305587448541805778891719207881423351138662929667179643468760077047999537883387870348718021842437342112273940255717690819603092018240188427057046092622564178375265263358324240661253311529423457965569502506810018310900411245379015332966156970522379210325706937051090830789479999004999395322153622748476603613677697978567386584670936679588583788795625946464891376652199588286933801836011932368578558558195556042156250883650203322024513762158204618106705195330653060606501054887167245377942831338871631395596905832083416898476065607118347136218123246227258841990286142087284956879639325464285343075301105285713829643709990356948885285190402956047346131138263878897551788560424998748316382804046848618938189590542039889872650697620201995548412650005394428203930127481638158530396439925470201672759328574366661644110962566337305409219519675148328734808957477775278344221091073111351828046036347198185655572957144747682552857863349342858423118749440003229690697758315903858039353521358860079600342097547392296733310649395601812237812854584317605561733861126734780745850676063048229409653041118306671081893031108871728167519579675347188537229309616143204006381322465841111157758358581135018569047815368938137718472814751998350504781297718599084707621974605887423256995828892535041937958260616211842368768511418316068315867994601652057740529423053601780313357263267054790338401257305912339601880137825421927094767337191987287385248057421248921183470876629667207272325650565129333126059505777727542471241648312832982072361750574673870128209575544305968395555686861188397135522084452852640081252027665557677495969626612604565245684086139238265768583384698499778726706555191854468698469478495734622606294219624557085371272776523098955450193037732166649182578154677292005212667143463209637891852323215018976126034373684067194193037746880999296877582441047878123266253181845960453853543839114496775312864260925211537673258866722604042523491087026958099647595805794663973419064010036361904042033113579336542426303561457009011244800890020801478056603710154122328891465722393145076071670643556827437743965789067972687438473076346451677562103098604092717090951280863090297385044527182892749689212106670081648583395537735919136950153162018908887484210798706899114804669270650940762046502772528650728905328548561433160812693005693785417861096969202538865034577183176686885923681488475276498468821949739729707737187188400414323127636504814531122850990020742409255859252926103021067368154347015252348786351643976235860419194129697690405264832347009911154242601273438022089331096686367898694977994001260164227609260823493041180643829138347354679725399262338791582998486459271734059225620749105308531537182911681637219395188700957788181586850464507699343940987433514431626330317247747486897918209239480833143970840673084079589358108966564775859905563769525232653614424780230826811831037735887089240613031336477371011628214614661679404090518615260360092521947218890918107335871964142144478654899528582343947050079830388538860831035719306002771194558021911942899922722353458707566246926177663178855144350218287026685610665003531050216318206017609217984684936863161293727951873078972637353717150256378733579771808184878458866504335824377004147710414934927438457587107159731559439426412570270965125108115548247939403597681188117282472158250109496096625393395380922195591918188552678062149923172763163218339896938075616855911752998450132067129392404144593862398809381240452191484831646210147389182510109096773869066404158973610476436500068077105656718486281496371118832192445663945814491486165500495676982690308911185687986929470513524816091743243015383684707292898982846022237301452655679898627767968091469798378268764311598832109043715611299766521539635464420869197567370005738764978437686287681792497469438427465256316323005551304174227341646455127812784577772457520386543754282825671412885834544435132562054464241011037955464190581168623059644769587054072141985212106734332410756767575818456990693046047522770167005684543969234041711089888993416350585157887353430815520811772071880379104046983069578685473937656433631979786803671873079693924236321448450354776315670255390065423117920153464977929066241508328858395290542637687668968805033317227800185885069736232403894700471897619347344308437443759925034178807972235859134245813144049847701732361694719765715353197754997162785663119046912609182591249890367654176979903623755286526375733763526969344354400473067198868901968147428767790866979688522501636949856730217523132529265375896415171479559538784278499866456302878831962099830494519874396369070682762657485810439112232618794059941554063270131989895703761105323606298674803779153767511583043208498720920280929752649812569163425000522908872646925284666104665392171482080130502298052637836426959733707053922789153510568883938113249757071331029504430346715989448786847116438328050692507766274500122003526203709466023414648998390252588830148678162196775194583167718762757200505439794412459900771152051546199305098386982542846407255540927403132571632640792934183342147090412542533523248021932277075355546795871638358750181593387174236061551171013123525633485820365146141870049205704372018261733194715700867578539336078622739558185797587258744102542077105475361294047460100094095444959662881486915903899071865980563617137692227290764197755177720104276496949611056220592502420217704269622154958726453989227697660310524980855759471631075870133208861463266412591148633881220284440694169488261529577625325019870359870674380469821942056381255833436421949232275937221289056420943082352544084110864545369404969271494003319782861318186188811118408257865928757426384450059944229568586460481033015388911499486935436030221810943466764000022362550573631294626296096198760564259963946138692330837196265954739234624134597795748524647837980795693198650815977675350553918991151335252298736112779182748542008689539658359421963331502869561192012298889887006079992795411188269023078913107603617634779489432032102773359416908650071932804017163840644987871753756781185321328408216571107549528294974936214608215583205687232185574065161096274874375098092230211609982633033915469494644491004515280925089745074896760324090768983652940657920198315265410658136823791984090645712468948470209357761193139980246813405200394781949866202624008902150166163813538381515037735022966074627952910384068685569070157516624192987244482719429331004854824454580718897633003232525821581280327467962002814762431828622171054352898348208273451680186131719593324711074662228508710666117703465352839577625997744672185715816126411143271794347885990892808486694914139097716736900277758502686646540565950394867841110790116104008572744562938425494167594605487117235946429105850909950214958793112196135908315882620682332156153086833730838173279328196983875087083483880463884784418840031847126974543709373298362402875197920802321878744882872843727378017827008058782410749357514889978911739746129320351081432703251409030487462262942344327571260086642508333187688650756429271605525289544921537651751492196367181049435317858383453865255656640657251363575064353236508936790431702597878177190314867963840828810209461490079715137717099061954969640070867667102330048672631475510537231757114322317411411680622864206388906210192355223546711662137499693269321737043105987225039456574924616978260970253359475020913836673772894438696400028110344026084712899000746807764844088711341352503367877316797709372778682166117865344231732264637847697875144332095340001650692130546476890985050203015044880834261845208730530973189492916425322933612431514306578264070283898409841602950309241897120971601649265613413433422298827909921786042679812457285345801338260995877178113102167340256562744007296834066198480676615805021691833723680399027931606420436812079900316264449146190219458229690992122788553948783538305646864881655562294315673128274390826450611628942803501661336697824051770155219626522725455850738640585299830379180350432876703809252167907571204061237596327685674845079151147313440001832570344920909712435809447900462494313455028900680648704293534037436032625820535790118395649089354345101342969617545249573960621490288728932792520696535386396443225388327522499605986974759882329916263545973324445163755334377492928990581175786355555626937426910947117002165411718219750519831787137106051063795558588905568852887989084750915764639074693619881507814685262133252473837651192990156109189777922008705793396463827490680698769168197492365624226087154176100430608904377976678519661891404144925270480881971498801542057787006521594009289777601330756847966992955433656139847738060394368895887646054983871478968482805384701730871117761159663505039979343869339119789887109156541709133082607647406305711411098839388095481437828474528838368079418884342666222070438722887413947801017721392281911992365405516395893474263953824829609036900288359327745855060801317988407162446563997948275783650195514221551339281978226984278638391679715091262410548725700924070045488485692950448110738087996547481568913935380943474556972128919827177020766613602489581468119133614121258783895577357194986317210844398901423948496659251731388171602663261931065366535041473070804414939169363262373767777095850313255990095762731957308648042467701212327020533742667053142448208168130306397378736642483672539837487690980602182785786216512738563513290148903509883270617258932575363993979055729175160097615459044771692265806315111028038436017374742152476085152099016158582312571590733421736576267142390478279587281505095633092802668458937649649770232973641319060982740633531089792464242134583740901169391964250459128813403498810635400887596820054408364386516617880557608956896727531538081942077332597917278437625661184319891025007491829086475149794003160703845549465385946027452447466812314687943441610993338908992638411847425257044572517459325738989565185716575961481266020310797628254165590506042479114016957900338356574869252800743025623419498286467914476322774005529460903940177536335655471931000175430047504719144899841040015867946179241610016454716551337074073950260442769538553834397550548871099785205401175169747581344926079433689543783221172450687344231989878844128542064742809735625807066983106979935260693392135685881391214807354728463227784908087002467776303605551232386656295178853719673034634701222939581606792509153217489030840886516061119011498443412350124646928028805996134283511884715449771278473361766285062169778717743824362565711779450064477718370221999106695021656757644044997940765037999954845002710665987813603802314126836905783190460792765297277694043613023051787080546511542469395265127101052927070306673024447125973939950514628404767431363739978259184541176413327906460636584152927019030276017339474866960348694976541752429306040727005059039503148522921392575594845078867977925253931765156416197168443524369794447355964260633391055126826061595726217036698506473281266724521989060549880280782881429796336696744124805982192146339565745722102298677599746738126069367069134081559412016115960190237753525556300606247983261249881288192937343476862689219239777833910733106588256813777172328315329082525092733047850724977139448333892552081175608452966590553940965568541706001179857293813998258319293679100391844099286575605993598910002969864460974714718470101531283762631146774209145574041815908800064943237855839308530828305476076799524357391631221886057549673832243195650655460852881201902363644712703748634421727257879503428486312944916318475347531435041392096108796057730987201352484075057637199253650470908582513936863463863368042891767107602111159828875539940120076013947033661793715396306139863655492213741597905119083588290097656647300733879314678913181465109316761575821351424860442292445304113160652700974330088499034675405518640677342603583409608605533747362760935658853109760994238347382222087292464497684560579562516765574088410321731345627735856052358236389532038534024842273371639123973215995440828421666636023296545694703577184873442034227706653837387506169212768015766181095420097708363604361110592409117889540338021426523948929686439808926114635414571535194342850721353453018315875628275733898268898523557799295727645229391567477566676051087887648453493636068278050564622813598885879259940946446041705204470046315137975431737187756039815962647501410906658866162180038266989961965580587208639721176995219466789857011798332440601811575658074284182910615193917630059194314434605154047710570054339000182453117733718955857603607182860506356479979004139761808955363669603162193113250223851791672055180659263518036251214575926238369348222665895576994660491938112486609099798128571823494006615552196112207203092277646200999315244273589488710576623894693889446495093960330454340842102462401048723328750081749179875543879387381439894238011762700837196053094383940063756116458560943129517597713935396074322792489221267045808183313764165818269562105872892447740035947009268662659651422050630078592002488291860839743732353849083964326147000532423540647042089499210250404726781059083644007466380020870126664209457181702946752278540074508552377720890581683918446592829417018288233014971554235235911774818628592967605048203864343108779562892925405638946621948268711042828163893975711757786915430165058602965217459581988878680408110328432739867198621306205559855266036405046282152306154594474489908839081999738747452969810776201487134000122535522246695409315213115337915798026979555710508507473874750758068765376445782524432638046143042889235934852961058269382103498000405248407084403561167817170512813378805705643450616119330424440798260377951198548694559152051960093041271007277849301555038895360338261929343797081874320949914159593396368110627557295278004254863060054523839151068998913578820019411786535682149118528207852130125518518493711503422159542244511900207393539627400208110465530207932867254740543652717595893500716336076321614725815407642053020045340183572338292661915308354095120226329165054426123619197051613839357326693760156914429944943744856809775696303129588719161129294681884936338647392747601226964158848900965717086160598147204467428664208765334799858222090619802173211614230419477754990738738567941189824660913091691772274207233367635032678340586301930193242996397204445179288122854478211953530898910125342975524727635730226281382091807439748671453590778633530160821559911314144205091447293535022230817193663509346865858656314855575862447818620108711889760652969899269328178705576435143382060141077329261063431525337182243385263520217735440715281898137698755157574546939727150488469793619500477720970561793913828989845327426227288647108883270173723258818244658436249580592560338105215606206155713299156084892064340303395262263451454283678698288074251422567451806184149564686111635404971897682154227722479474033571527436819409892050113653400123846714296551867344153741615042563256713430247655125219218035780169240326699541746087592409207004669340396510178134857835694440760470232540755557764728450751826890418293966113310160131119077398632462778219023650660374041606724962490137433217246454097412995570529142438208076098364823465973886691349919784013108015581343979194852830436739012482082444814128095443773898320059864909159505322857914576884962578665885999179867520554558099004556461178755249370124553217170194282884617402736649978475508294228020232901221630102309772151569446427909802190826689868834263071609207914085197695235553488657743425277531197247430873043619511396119080030255878387644206085044730631299277888942729189727169890575925244679660189707482960949190648764693702750773866432391919042254290235318923377293166736086996228032557185308919284403805071030064776847863243191000223929785255372375566213644740096760539439838235764606992465260089090624105904215453927904411529580345334500256244101006359530039598864466169595626351878060688513723462707997327233134693971456285542615467650632465676620279245208581347717608521691340946520307673391841147504140168924121319826881568664561485380287539331160232292555618941042995335640095786495340935115266454024418775949316930560448686420862757201172319526405023099774567647838488973464317215980626787671838005247696884084989185086149003432403476742686245952395890358582135006450998178244636087317754378859677672919526111213859194725451400301180503437875277664402762618941017576872680428176623860680477885242887430259145247073950546525135339459598789619778911041890292943818567205070964606263541732944649576612651953495701860015412623962286413897796733329070567376962156498184506842263690367849555970026079867996261019039331263768556968767029295371162528005543100786408728939225714512481135778627664902425161990277471090335933309304948380597856628844787441469841499067123764789582263294904679812089984857163571087831191848630254501620929805829208334813638405421720056121989353669371336733392464416125223196943471206417375491216357008573694397305979709719726666642267431117762176403068681310351899112271339724036887000996862922546465006385288620393800504778276912835603372548255793912985251506829969107754257647488325341412132800626717094009098223529657957997803018282428490221470748111124018607613415150387569830918652780658896682362523937845272634530420418802508442363190383318384550522367992357752929106925043261446950109861088899914658551881873582528164302520939285258077969737620845637482114433988162710031703151334402309526351929588680690821355853680161000213740851154484912685841268695899174149133820578492800698255195740201818105641297250836070356851055331787840829000041552511865779453963317538532092149720526607831260281961164858098684587525129997404092797683176639914655386108937587952214971731728131517932904431121815871023518740757222100123768721944747209349312324107065080618562372526732540733324875754482967573450019321902199119960797989373383673242576103938985349278777473980508080015544764061053522202325409443567718794565430406735896491017610775948364540823486130254718476485189575836674399791508512858020607820554462991723202028222914886959399729974297471155371858924238493855858595407438104882624648788053304271463011941589896328792678327322456103852197011130466587100500083285177311776489735230926661234588873102883515626446023671996644554727608310118788389151149340939344750073025855814756190881398752357812331342279866503522725367171230756861045004548970360079569827626392344107146584895780241408158405229536937499710665594894459246286619963556350652623405339439142111271810691052290024657423604130093691889255865784668461215679554256605416005071276641766056874274200329577160643448606201239821698271723197826816628249938714995449137302051843669076723577400053932662622760323659751718925901801104290384274185507894887438832703063283279963007200698012244365116394086922220745320244624121155804354542064215121585056896157356414313068883443185280853975927734433655384188340303517822946253702015782157373265523185763554098954033236382319219892171177449469403678296185920803403867575834111518824177439145077366384071880489358256868542011645031357633355509440319236720348651010561049872726472131986543435450409131859513145181276437310438972507004981987052176272494065214619959232142314439776546708351714749367986186552791715824080651063799500184295938799158350171580759883784962257398512129810326379376218322456594236685376799113140108043139732335449090824910499143325843298821033984698141715756010829706583065211347076803680695322971990599904451209087275776225351040902392888779424630483280319132710495478599180196967835321464441189260631526618167443193550817081875477050802654025294109218264858213857526688155584113198560022135158887210365696087515063187533002942118682221893775546027227291290504292259787710667873840000616772154638441292371193521828499824350920891801685572798156421858191197490985730570332667646460728757430565372602768982373259745084479649545648030771598153955827779139373601717422996027353102768719449444917939785144631597314435351850491413941557329382048542123508173912549749819308714396615132942045919380106231421774199184060180347949887691051557905554806953878540066453375981862846419905220452803306263695626490910827627115903856995051246529996062855443838330327638599800792922846659503551211245284087516229060262011857775313747949362055496401073001348853150735487353905602908933526400713274732621960311773433943673385759124508149335736911664541281788171454023054750667136518258284898099512139193995633241336556777098003081910272040997148687418134667006094051021462690280449159646545330107754695413088714165312544813061192407821188690056027781824235022696189344352547633573536485619363254417756613981703930632872166905722259745209192917262199844409646158269456380239502837121686446561785235565164127712826918688615572716201474934052276946595712198314943381622114006936307430444173284786101777743837977037231795255434107223445512555589998646183876764903972461167959018100035098928641204195163551108763204267612979826529425882951141275841262732790798807559751851576841264742209479721843309352972665210015662514552994745127631550917636730259462132930190402837954246323258550301096706922720227074863419005438302650681214142135057154175057508639907673946335146209082888934938376439399256900604067311422093312195936202982972351163259386772241477911629572780752395056251581603133359382311500518626890530658368129988108663263271980611271548858798093487912913707498230575929091862939195014721197586067270092547718025750337730799397134539532646195269996596385654917590458333585799102012713204583903200853878881633637685182083727885131175227769609787962142372162545214591281831798216044111311671406914827170981015457781939202311563871950805024679725792497605772625913328559726371211201905720771409148645074094926718035815157571514050397610963846755569298970383547314100223802583468767350129775413279532060971154506484212185936490997917766874774481882870632315515865032898164228288232746866106592732197907162384642153489852476216789050260998045266483929542357287343977680495774091449538391575565485459058976495198513801007958010783759945775299196700547602252552034453988712538780171960718164078124847847257912407824544361682345239570689514272269750431873633263011103053423335821609333191218806608268341428910415173247216053355849993224548730778822905252324234861531520976938461042582849714963475341837562003014915703279685301868631572488401526639835689563634657435321783493199825542117308467745297085839507616458229630324424328237737450517028560698067889521768198156710781633405266759539424926280756968326107495323390536223090807081455919837355377748742029039018142937311529334644468151212945097596534306284215319445727118614900017650558177095302468875263250119705209476159416768727784472000192789137251841622857783792284439084301181121496366424659033634194540657183544771912446621259392656620306888520055599121235363718226922531781458792593750441448933981608657900876165024635197045828895481793756681046474614105142498870252139936870509372305447734112641354892806841059107716677821238332810262185587751312721179344448201440425745083063944738363793906283008973306241380614589414227694747931665717623182472168350678076487573420491557628217583972975134478990696589532548940335615613167403276472469212505759116251529654568544633498114317670257295661844775487469378464233737238981920662048511894378868224807279352022501796545343757274163910791972952950812942922205347717304184477915673991738418311710362524395716152714669005814700002633010452643547865903290733205468338872078735444762647925297690170912007874183736735087713376977683496344252419949951388315074877537433849458259765560996555954318040920178497184685497370696212088524377013853757681416632722412634423982152941645378000492507262765150789085071265997036708726692764308377229685985169122305037462744310852934305273078865283977335246017463527703205938179125396915621063637625882937571373840754406468964783100704580613446731271591194608435935825987782835266531151065041623295329047772174083559349723758552138048305090009646676088301540612824308740645594431853413755220166305812111033453120745086824339432159043594430312431227471385842030390106070940315235556172767994160020393975099897629335325855575624808996691829864222677502360193257974726742578211119734709402357457222271212526852384295874273501563660093188045493338989741571490544182559738080871565281430102670460284316819230392535297795765862414392701549740879273131051636119137577008929564823323648298263024607975875767745377160102490804624301856524161756655600160859121534556267602192689982855377872583145144082654583484409478463178777374794653580169960779405568701192328608041130904629350871827125934668712766694873899824598527786499569165464029458935064964335809824765965165142090986755203808309203230487342703468288751604071546653834619611223013759451579252696743642531927390036038608236450762698827497618723575476762889950752114804852527950845033958570838130476937881321123674281319487950228066320170022460331989671970649163741175854851878484012054844672588851401562725019821719066960812627785485964818369621410721714214986361918774754509650308957099470934337856981674465828267911940611956037845397855839240761276344105766751024307559814552786167815949657062559755074306521085301597908073343736079432866757890533483669555486803913433720156498834220893399971641479746938696905480089193067138057171505857307148815649920714086758259602876056459782423770242469805328056632787041926768467116266879463486950464507420219373945259262668613552940624781361206202636498199999498405143868285258956342264328707663299304891723400725471764188685351372332667877921738347541480022803392997357936152412755829569276837231234798989446274330454566790062032420516396282588443085438307201495672106460533238537203143242112607424485845094580494081820927639140008540422023556260218564348994145439950410980591817948882628052066441086319001688568155169229486203010738897181007709290590480749092427141018933542818429995988169660993836961644381528877214085268088757488293258735809905670755817017949161906114001908553744882726200936685604475596557476485674008177381703307380305476973609786543859382187220583902344443508867499866506040645874346005331827436296177862518081893144363251205107094690813586440519229512932450078833398788429339342435126343365204385812912834345297308652909783300671261798130316794385535726296998740359570458452230856390098913179475948752126397078375944861139451960286751210561638976008880092746115860800207803341591451797073036835196977766076373785333012024120112046988609209339085365773222392412449051532780950955866459477634482269986074813297302630975028812103517723124465095349653693090018637764094094349837313251321862080214809922685502948454661814715557444709669530177690434272031892770604717784527939160472281534379803539679861424370956683221491465438014593829277393396032754048009552231816667380357183932757077142046723838624617803976292377131209580789363841447929802588065522129262093623930637313496640186619510811583471173312025805866727639992763579078063818813069156366274125431259589936119647626101405563503399523140323113819656236327198961837254845333702062563464223952766943568376761368711962921818754576081617053031590728828700712313666308722754918661395773730546065997437810987649802414011242142773668082751390959313404155826266789510846776118665957660165998178089414985754976284387856100263796543178313634025135814161151902096499133548733131115022700681930135929595971640197196053625033558479980963488718039111612813595968565478868325856437896173159762002419621552896297904819822199462269487137462444729093456470028537694958859591606789282491054412515996300781368367490209374915732896270028656829344431342347351239298259166739503425995868970697267332582735903121288746660451461487850346142827765991608090398652575717263081833494441820193533385071292345774375579344062178711330063106003324053991693682603746176638565758877580201229366353270267100681261825172914608202541892885935244491070138206211553827793565296914576502048643282865557934707209634807372692141186895467322767751335690190153723669036865389161291688887876407525493494249733427181178892759931596719354758988097924525262363659036320070854440784544797348291802082044926670634420437555325050527522833778887040804033531923407685630109347772125639088640413101073817853338316038135280828119040832564401842053746792992622037698718018061122624490909242641985820861751177113789051609140381575003366424156095216328197122335023167422600567941281406217219641842705784328959802882335059828208196666249035857789940333152274817776952843681630088531769694783690580671064828083598046698841098135158654906933319522394363287923990534810987830274500172065433699066117784554364687723631844464768069142828004551074686645392805399409108754939166095731619715033166968309929466349142798780842257220697148875580637480308862995118473187124777291910070227588893486939456289515802965372150409603107761289831263589964893410247036036645058687287589051406841238124247386385427908282733827973326885504935874303160274749063129572349742611221517417153133618622410913869500688835898962349276317316478340077460886655598733382113829928776911495492184192087771606068472874673681886167507221017261103830671787856694812948785048943063086169948798703160515884108282351274153538513365895332948629494495061868514779105804696039069372662670386512905201137810858616188886947957607413585534585151768051973334433495230120395770739623771316030242887200537320998253008977618973129817881944671731160647231476248457551928732782825127182446807824215216469567819294098238926284943760248852279003620219386696482215628093605373178040863727268426696421929946819214908701707533361094791381804063287387593848269535583077395761447997270003472880182785281389503217986345216111066608839314053226944905455527867894417579202440021450780192099804461382547805858048442416404775031536054906591430078158372430123137511562284015838644270890718284816757527123846782459534334449622010096071051370608461801187543120725491334994247617115633321408934609156561550600317384218701570226103101916603887064661438897736318780940711527528174689576401581047016965247557740891644568677717158500583269943401677202156767724068128366565264122982439465133197359199709403275938502669557470231813203243716420586141033606524536939160050644953060161267822648942437397166717661231048975031885732165554988342121802846912529086101485527815277625623750456375769497734336846015607727035509629049392487088406281067943622418704747008368842671022558302403599841645951122485272633632645114017395248086194635840783753556885622317115520947223065437092606797351000565549381224575483728545711797393615756167641692895805257297522338558611388322171107362265816218842443178857488798109026653793426664216990914056536432249301334867988154886628665052346997235574738424830590423677143278792316422403877764330192600192284778313837632536121025336935812624086866699738275977365682227907215832478888642369346396164363308730139814211430306008730666164803678984091335926293402304324974926887831643602681011309570716141912830686577323532639653677390317661361315965553584999398600565155921936759977717933019744688148371103206503693192894521402650915465184309936553493337183425298433679915939417466223900389527673813330617747629574943868716978453767219493506590875711917720875477107189937960894774512654757501871194870738736785890200617373321075693302216320628432065671192096950585761173961632326217708945426214609858410237813215817727602222738133495410481003073275107799948991977963883530734443457532975914263768405442264784216063122769646967156473999043715903323906560726644116438605404838847161912109008701019130726071044114143241976796828547885524779476481802959736049439700479596040292746299203572099761950140348315380947714601056333446998820822120587281510729182971211917876424880354672316916541852256729234429187128163232596965413548589577133208339911288775917226115273379010341362085614577992398778325083550730199818459025958355989260553299673770491722454935329683300002230181517226575787524058832249085821280089747909326100762578770428656006996176212176845478996440705066241710213327486796237430229155358200780141165348065647488230615003392068983794766255036549822805329662862117930628430170492402301985719978948836897183043805182174419147660429752437251683435411217038631379411422095295885798060152938752753799030938871683572095760715221900279379292786303637268765822681241993384808166021603722154710143007377537792699069587121289288019052031601285861825494413353820784883465311632650407642428390870121015194231961652268422003711230464300673442064747718021353070124098860353399152667923871101706221865883573781210935179775604425634694999787251125440854522274810914874307259869602040275941178942581281882159952359658979181144077653354321757595255536158128001163846720319346507296807990793963714961774312119402021297573125165253768017359101557338153772001952444543620071848475663415407442328621060997613243487548847434539665981338717466093020535070271952983943271425371155766600025784423031073429551533945060486222764966687624079324353192992639253731076892135352572321080889819339168668278948281170472624501948409700975760920983724090074717973340788141825195842598096241747610138252643955135259311885045636264188300338539652435997416931322894719878308427600401368074703904097238473945834896186539790594118599310356168436869219485382055780395773881360679549900085123259442529724486666766834641402189915944565309423440650667851948417766779470472041958822043295380326310537494883122180391279678446100139726753892195119117836587662528083690053249004597410947068772912328214304635337283519953648274325833119144459017809607782883583730111857543659958982724531925310588115026307542571493943024453931870179923608166611305426253995833897942971602070338767815033010280120095997252222280801423571094760351925544434929986767817891045559063015953809761875920358937341978962358931125983902598310267193304189215109689156225069659119828323455503059081730735195503721665870288053992138576037035377105178021280129566841984140362872725623214428754302210909472721073474134975514190737043318276626177275996888826027225247133683353452816692779591328861381766349857728936900965749562287103024362590772412219094300871755692625758065709912016659622436080242870024547362036394841255954881727272473653467783647201918303998717627037515724649922289467932322693619177641614618795613956699567783068290316589699430767333508234990790624100202506134057344300695745474682175690441651540636584680463692621274211075399042188716127617787014258864825775223889184599523376292377915585744549477361295525952226578636462118377598473700347971408206994145580719080213590732269233100831759510659019121294795408603640757358750205890208704579670007055262505811420663907459215273309406823649441590891009220296680523325266198911311842016291631076894084723564366808182168657219688268358402785500782804043453710183651096951782335743030504852653738073531074185917705610397395062640355442275156101107261779370634723804990666922161971194259120445084641746383589938239946517395509000859479990136026674261494290066467115067175422177038774507673563742154782905911012619157555870238957001405117822646989944917908301795475876760168094100135837613578591356924455647764464178667115391951357696104864922490083446715486383054477914330097680486878348184672733758436892724310447406807685278625585165092088263813233623148733336714764520450876627614950389949504809560460989604329123358348859990294526400284994280878624039811814884767301216754161106629995553668193123287425702063738352020086863691311733469731741219153633246745325630871347302792174956227014687325867891734558379964351358800959350877556356248810493852999007675135513527792412429277488565888566513247302514710210575352516511814850902750476845518252096331899068527614435138213662152368890578786699432288816028377482035506016029894009119713850179871683633744139275973644017007014763706655703504338121113576415018451821413619823495159601064752712575935185304332875537783057509567425442684712219618709178560783936144511383335649103256405733898667178123972237519316430617013859539474367843392670986712452211189690840236327411496601243483098929941738030588417166613073040067588380432111555379440605497721705942821514886165672771240903387727745629097110134885184374118695655449745736845218066982911045058004299887953899027804383596282409421860556287788428802127553884803728640019441614257499904272009595204654170598104989967504511936471172772220436102614079750809686975176600237187748348016120310234680567112644766123747627852190241202569943534716226660893675219833111813511146503854895025120655772636145473604426859498074396932331297127377157347099713952291182653485155587137336629120242714302503763269501350911612952993785864681307226486008270881333538193703682598867893321238327053297625857382790097826460545598555131836688844628265133798491667839409761353766251798258249663458771950124384040359140849209733754642474488176184070023569580177410177696925077814893386672557898564589851056891960924398841569280696983352240225634570497312245269354193837004843183357196516626721575524193401933099018319309196582920969656247667683659647019595754739345514337413708761517323677204227385674279170698204549953095918872434939524094441678998846319845504852393662972079777452814399418256789457795712552426826089940863317371538896262889629402112108884427376568624527612130371017300785135715404533041507959447776143597437803742436646973247138410492124314138903579092416036406314038149831481905251720937103964026808994832572297954564042701757722904173234796073618787889913318305843069394825961318713816423467218730845133877219086975104942843769325024981656673816260615941768252509993741672883951744066932549653403101452225316189009235376486378482881344209870048096227171226407489571939002918573307460104360729190945767994614929290427981687729426487729952858434647775386906950148984133924540394144680263625402118614317031251117577642829914644533408920976961699098372652361768745605894704968170136974909523072082682887890730190018253425805343421705928713931737993142410852647390948284596418093614138475831136130576108462366837237695913492615824516221552134879244145041756848064120636520170386330129532777699023118648020067556905682295016354931992305914246396217025329747573114094220180199368035026495636955866425906762685687372110339156793839895765565193177883000241613539562437777840801748819373095020699900890899328088397430367736595524891300156633294077907139615464534088791510300651321934486673248275907946807879819425019582622320395131252014109960531260696555404248670549986786923021746989009547850725672978794769888831093487464426400718183160331655511534276155622405474473378049246214952133258527698847336269182649174338987824789278468918828054669982303689939783413747587025805716349413568433929396068192061773331791738208562436433635359863494496890781064019674074436583667071586924521182997893804077137501290858646578905771426833582768978554717687184427726120509266486102051535642840632368481807287940717127966820060727559555904040233178749447346454760628189541512139162918444297651066947969354016866010055196077687335396511614930937570968554559381513789569039251014953265628147011998326992200066392875374713135236421589265126204072887716578358405219646054105435443642166562244565042999010256586927279142752931172082793937751326106052881235373451068372939893580871243869385934389175713376300720319760816604464683937725806909237297523486702916910426369262090199605204121024077648190316014085863558427609537086558164273995349346546314504040199528537252004957805254656251154109252437991326262713609099402902262062836752132305065183934057450112099341464918433323646569371725914489324159006242020612885732926133596808726500045628284557574596592120530341310111827501306961509835515632004310784601906565493806542525229161991819959602752327702249855738824899882707465936355768582560518068964285376850772012220347920993936179268206590142165615925306737944568949070853263568196831861772268249911472615732035807646298116244013316737892788689229032593349861797021994981925739617673075834417098559222170171825712777534491508205278430904619460835217402005838672849709411023266953921445461066215006410674740207009189911951376466904481267253691537162290791385403937560077835153374167747942100384002308951850994548779039346122220865060160500351776264831611153325587705073541279249909859373473787081194253055121436979749914951860535920403830235716352727630874693219622190064260886183676103346002255477477813641012691906569686495012688376296907233961276287223041141813610060264044030035996988919945827397624114613744804059697062576764723766065541618574690527229238228275186799156983390747671146103022776606020061246876477728819096791613354019881402757992174167678799231603963569492851513633647219540611171767387372555728522940054361785176502307544693869307873499110352182532929726044553210797887711449898870911511237250604238753734841257086064069052058452122754533848008205302450456517669518576913200042816758054924811780519832646032445792829730129105318385636821206215531288668564956512613892261367064093953334570526986959692350353094224543865278677673027540402702246384483553239914751363441044050092330361271496081355490531539021002299595756583705381261965683144286057956696622154721695620870013727768536960840704833325132793112232507148630206951245395003735723346807094656483089209801534878705633491092366057554050864111521441481434630437273271045027768661953107858323334857840297160925215326092558932655600672124359464255065996771770388445396181632879614460817789272171836908880126778207430106422524634807454300476492885553409062185153654355474125476152769772667769772777058315801412185688011705028365275543214803488004442979998062157904564161957212784508928489806426497427090579129069217807298769477975112447305991406050629946894280931034216416629935614828130998870745292716048433630818404126469637925843094185442216359084576146078558562473814931427078266215185541603870206876980461747400808324343665382354555109449498431093494759944672673665352517662706772194183191977196378015702169933675083760057163454643671776723387588643405644871566964321041282595645349841388412890420682047007615596916843038999348366793542549210328113363184722592305554383058206941675629992013373175489122037230349072681068534454035993561823576312837767640631013125335212141994611869350833176587852047112364331226765129964171325217513553261867681942338790365468908001827135283584888444111761234101179918709236507184857856221021104009776994453121795022479578069506532965940383987369907240797679040826794007618729547835963492793904576973661643405359792219285870574957481696694062334272619733518136626063735982575552496509807260123668283605928341855848026958413772558970883789942910549800331113884603401939166122186696058491571485733568286149500019097591125218800396419762163559375743718011480559442298730418196808085647265713547612831629200449880315402105530597076666362749328308916880932359290081787411985738317192616728834918402429721290434965526942726402559641463525914348400675867690350382320572934132981593533044446496829441367323442158380761694831219333119819061096142952201536170298575105594326461468505452684975764807808009221335811378197749271768545075538328768874474591593731162470601091244609829424841287520224462594477638749491997840446829257360968534549843266536862844489365704111817793806441616531223600214918768769467398407517176307516849856359201486892943105940202457969622924566644881967576294349535326382171613395757790766370764569570259738800438415805894336137106551859987600754924187211714889295221737721146081154344982665479872580056674724051122007383459271575727715218589946948117940644466399432370044291140747218180224825837736017346685300744985564715420036123593397312914458591522887408719508708632218837288262822884631843717261903305777147651564143822306791847386039147683108141358275755853643597721650028277803713422869688787349795096031108899196143386664068450697420787700280509367203387232629637856038653216432348815557557018469089074647879122436375556668678067610544955017260791142930831285761254481944449473244819093795369008206384631678225064809531810406570254327604385703505922818919878065865412184299217273720955103242251079718077833042609086794273428955735559252723805511440438001239041687716445180226491681641927401106451622431101700056691121733189423400547959684669804298017362570406733282129962153684881404102194463424646220745575643960452985313071409084608499653767803793201899140865814662175319337665970114330608625009829566917638846056762972931464911493704624469351984039534449135141193667933301936617663652555149174982307987072280860859626112660504289296966535652516688885572112276802772743708917389639772257564890533401038855931125679991516589025016486961427207005916056166159702451989051832969278935550303934681219761582183980483960562523091462638447386296039848924386187298507775928792722068554807210497817653286210187476766897248841139560349480376727036316921007350834073865261684507482496448597428134936480372426116704266870831925040997615319076855770327421785010006441984124207396400139603601583810565928413684574119102736420274163723488214524101347716529603128408658419787951116511529827814620379139855006399960326591248525308493690313130100799977191362230866011099929142871249388541612038020411340188887219693477904497527454288072803509305828754420755134816660927879353566521255620139988249628478726214432362853676502591450468377635282587652139156480972141929675549384375582600253168536356731379262475878049445944183429172756988376226261846365452743497662411138451305481449836311789784489732076719508784158618879692955819733250699951402601511675529750575437810242238957925786562128432731202200716730574069286869363930186765958251326499145950260917069347519408975357464016830811798846452473618956056479426358070562563281189269663026479535951097127659136233180866921535788607812759910537171402204506186075374866306350591483916467656723205714516886170790984695932236724946737583099607042589220481550799132752088583781117685214269334786921895240622657921043620348852926267984013953216458791151579050460579710838983371864038024417511347226472547010794793996953554669619726763255229914654933499663234185951450360980344092212206712567698723427940708857070474293173329188523896721971353924492426178641188637790962814486917869468177591717150669111480020759432012061969637795103227089029566085562225452602610460736131368869009281721068198618553780982018471154163630326265699283424155023600978046417108525537612728905335045506135684143775854429677977014660294387687225115363801191758154028120818255606485410787933598921064427244898618961629413418001295130683638609294100083136673372153008352696235737175330738653338204842190308186449184093723944033405244909554558016406460761581010301767488475017661908692946098769201691202181688291040870709560951470416921147027413390052253340834812870353031023919699978597413908593605433599697075604460134242453682496098772581311024732798562072126572499003468293886872304895562253204463602639854225258416464324271611419817802482595563544907219226583863662663750835944314877635156145710745528016159677048442714194435183275698407552677926411261765250615965235457187956673170913319358761628255920783080185206890151504713340386100310055914817852110384754542933389188444120517943969970194112695119526564919594189975418393234647424290702718875223534393673633663200307232747037407123982562024662651974090199762452056198557625760008708173083288344381831070054514493545885422678578551915372292379555494333410174420169600090696415612732297770221217951868376359082255128816470021992348864043959153018464004714321186360622527011541122283802778538911098490201342741014121559769965438877197485376431158229838533123071751132961904559007938064276695819014842627991221792947987348901868471676503827328552059082984529806259250352128451925927986593506132961946796252373972565584157853744567558998032405492186962888490332560851455344391660226257775512916200772796852629387937530454181080729285891989715381797343496187232927614747850192611450413274873242970583408471112333746274617274626582415324271059322506255302314738759251724787322881491455915605036334575424233779160374952502493022351481961381162563911415610326844958072508273431765944054098269765269344579863479709743124498271933113863873159636361218623497261409556079920628316999420072054811525353393946076850019909886553861433495781650089961649079678142901148387645682174914075623767618453775144031475411206760160726460556859257799322070337333398916369504346690694828436629980037414527627716547623825546170883189810868806847853705536480469350958818025360529740793538676511195079373282083146268960071075175520614433784114549950136432446328193346389050936545714506900864483440180428363390513578157273973334537284263372174065775771079830517555721036795976901889958494130195999573017901240193908681356585539661941371794487632079868800371607303220547423572266896801882123424391885984168972277652194032493227314793669234004848976059037958094696041754279613782553781223947646147832926976545162290281701100437846038756544151739433960048915318817576650500951697402415644771293656614253949368884230517400129920556854289853897942669956777027089146513736892206104415481662156804219838476730871787590279209175900695273456682026513373111518000181434120962601658629821076663523361774007837783423709152644063054071807843358061072961105550020415131696373046849213356837265400307509829089364612047891114753037049893952833457824082817386441322710002968311940203323456420826473276233830294639378998375836554559919340866235090967961134004867027123176526663710778725111860354037554487418693519733656621772359229396776463251562023487570113795712096237723431370212031004965152111976013176419408203437348512852602913334915125083119802850177855710725373149139215709105130965059885999931560863655477403551898166733535880048214665099741433761182777723351910741217572841592580872591315074606025634903777263373914461377038021318347447301113032670296917335047701632106616227830027269283365584011791419447808748253360714403296252285775009808599609040936312635621328162071453406104224112083010008587264252112262480142647519426184325853386753874054743491072710049754281159466017136122590440158991600229827801796035194080046513534752698777609527839984368086908989197839693532179980139135442552717910225397010810632143048511378291498511381969143043497500189980681644412123273328307192824362406733196554692677851193152775113446468905504248113361434984604849051258345683266441528489713972376040328212660253516693914082049947320486021627759791771234751097502403078935759937715095021751693555827072533911892334070223832077585802137174778378778391015234132098489423459613692340497998279304144463162707214796117456975719681239291913740982925805561955207434243295982898980529233366415419256367380689494201471241340525072204061794355252555225008748790086568314542835167750542294803274783044056438581591952666758282929705226127628711040134801787224801789684052407924360582742467443076721645270313451354167649668901274786801010295133862698649748212118629040337691568576240699296372493097201628707200189835423690364149270236961938547372480329855045112089192879829874467864129159417531675602533435310626745254507114181483239880607297140234725520713490798398982355268723950909365667878992383712578976248755990443228895388377317348941122757071410959790047919301046740750411435381782464630795989555638991884773781341347070246747362112048986226991888517456251732519341352038115863350123913054441910073628447567514161050410973505852762044489190978901984315485280533985777844313933883994310444465669244550885946314081751220331390681596592510546858013133838152176418210433429788826119630443111388796258746090226130900849975430395771243230616906262919403921439740270894777663702488155499322458825979020631257436910946393252806241642476868495455324938017639371615636847859823715902385421265840615367228607131702674740131145261063765383390315921943469817605358380310612887852051546933639241088467632009567089718367490578163085158138161966882222047570437590614338040725853862083565176998426774523195824182683698270160237414938363496629351576854061397342746470899685618170160551104880971554859118617189668025973541705423985135560018720335079060946421271143993196046527424050882225359773481519135438571253258540493946010865793798058620143366078825219717809025817370870916460452727977153509910340736425020386386718220522879694458387652947951048660717390229327455426785669776865939923416834122274663015062155320502655341460995249356050854921756549134830958906536175693817637473644183378974229700703545206663170929607591989627732423090252397443861014263098687733913882518684316501027964911497737582888913450341148865948670215492101084328080783428089417298008983297536940644969903125399863919581601468995220880662285408414864274786281975546629278814621607171381880180840572084715868906836919393381864278454537956719272397972364651667592011057995663962598535512763558768140213409829016296873429850792471846056874828331381259161962476156902875901072733103299140623864608333378638257926302391590003557609032477281338887339178096966601469615031754226751125993315529674213336300222964906480934582008181061802100227664580400278213336758573019011371754672763059044353131319036092489097246427928455549913490005180295707082919052556781889913899625138662319380053611346224294610248954072404857123256628888931722116432947816190554868054943441034090680716088028227959686950133643814268252170472870863010137301155236861416908375675747637239763185757038109443390564564468524183028148107998376918512127201935044041804604721626939445788377090105974693219720558114078775989772072009689382249303236830515862657281114637996983137517937623215111252349734305240622105244234353732905655163406669506165892878218707756794176080712973781335187117931650033155523822487730653444179453415395202424449703410120874072188109388268167512042299404948179449472732894770111574139441228455521828424922240658752689172272780607116754046973008037039618787796694882555614674384392570115829546661358678671897661297311267200072971553613027503556167817765442287442114729881614802705243806817653573275578602505847084013208837932816008769081300492491473682517035382219619039014999523495387105997351143478292339499187936608692301375596368532373806703591144243268561512109404259582639301678017128669239283231057658851714020211196957064799814031505633045141564414623163763809904402816256917576489142569714163598439317433270237812336938043012892626375382667795034169334323607500248175741808750388475094939454896209740485442635637164995949920980884294790363666297526003243856352945844728944547166209297495496616877414120882130477022816116456044007236351581149729739218966737382647204722642221242016560150284971306332795814302516013694825567014780935790889657134926158161346901806965089556310121218491805847922720691871696316330044858020102860657858591269974637661741463934159569539554203314628026518951167938074573315759846086173702687867602943677780500244673391332431669880354073232388281847501051641331189537036488422690270478052742490603492082954755054003457160184072574536938145531175354210726557835615499874447480427323457880061873149341566046352979779455075359304795687209316724536547208381685855606043801977030764246083489876101345709394877002946175792061952549255757109038525171488525265671045349813419803390641529876343695420256080277614421914318921393908834543131769685101840103844472348948869520981943531906506555354617335814045544837884752526253949665869992058417652780125341033896469818642430034146791380619028059607854888010789705516946215228773090104467462497979992627120951684779568482583341402266477210843362437593741610536734041954738964197895425335036301861400951534766961476255651873823292468547356935802896011536791787303553159378363082248615177770541577576561759358512016692943111138863582159667618830326104164651714846979385422621687161400122378213779774131268977266712992025922017408770076956283473932201088159356286281928563571893384958850603853158179760679479840878360975960149733420572704603521790605647603285569276273495182203236144112584182426247712012035776388895974318232827871314608053533574494297621796789034568169889553518504478325616380709476951699086247100019748809205009521943632378719764870339223811540363475488626845956159755193765410115014067001226927474393888589943859730245414801061235908036274585288493563251585384383242493252666087588908318700709100237377106576985056433928854337658342596750653715005333514489908293887737352051459333049626531415141386124437935885070944688045486975358170212908490787347806814366323322819415827345671356443171537967818058195852464840084032909981943781718177302317003989733050495387356116261023999433259780126893432605584710278764901070923443884634011735556865903585244919370181041626208504299258697435817098133894045934471937493877624232409852832762266604942385129709453245586252103600829286649724174919141988966129558076770979594795306013119159011773943104209049079424448868513086844493705909026006120649425744710353547657859242708130410618546219881830090634588187038755856274911587375421064667951346487586771543838018521348281915812462599335160198935595167968932852205824799421034512715877163345222995418839680448835529753361286837225935390079201666941339091168758803988828869216002373257361588207163516271332810518187602104852180675526648673908900907195138058626735124312215691637902277328705410842037841525683288718046987952513073266340278519059417338920358540395677035611329354482585628287610610698229721420961993509331312171187891078766872044548876089410174798647137882462153955933333275562009439580434537919782280590395959927436913793778664940964048777841748336432684026282932406260081908081804390914556351936856063045089142289645219987798849347477729132797266027658401667890136490508741142126861969862044126965282981087045479861559545338021201155646979976785738920186243599326777689454060508218838227909833627167124490026761178498264377033002081844590009717235204331994708242098771514449751017055643029542821819670009202515615844174205933658148134902693111517093872260026458630561325605792560927332265579346280805683443921373688405650434307396574061017779370141424615493070741360805442100295600095663588977899267630517718781943706761498217564186590116160865408635391513039201316805769034172596453692350806417446562351523929050409479953184074862151210561833854566176652606393713658802521666223576132201941701372664966073252010771947931265282763302413805164907174565964853748354669194523580315301969160480994606814904037819829732360930087135760798621425422096419004367905479049930078372421581954535418371129368658430553842717628035279128821129308351575656599944741788438381565148434229858704245592434693295232821803508333726283791830216591836181554217157448465778420134329982594566884558266171979012180849480332448787258183774805522268151011371745368417870280274452442905474518234674919564188551244421337783521423865979925988203287085109338386829906571994614906290257427686038850511032638544540419184958866538545040571323629681069146814847869659166861842756798460041868762298055562963045953227923051616721591968675849523635298935788507746081537321454642984792310511676357749494622952569497660359473962430995343310404994209677883827002714478494069037073249106444151696053256560586778757417472110827435774315194060757983563629143326397812218946287447798119807225646714664054850131009656786314880090303749338875364183165134982546694673316118123364854397649325026179549357204305402182974871251107404011611405899911093062492312813116340549262571356721818628932786138833718028535056503591952741400869510926167541476792668032109237467087213606278332922386413619594121339278036118276324106004740971111048140003623342714514483334641675466354699731494756643423659493496845884551524150756376605086632827424794136062876041290644913828519456402643153225858624043141838669590633245063000392213192647625962691510904457695301444054618037857503036686212462278639752746667870121003392984873375014475600322100622358029343774955032037012738468163061026570300872275462966796880890587127676361066225722352229739206443093524327228100859973095132528630601105497915644791845004618046762408928925680912930592960642357021061524646205023248966593987324933967376952023991760898474571843531936646529125848064480196520162838795189499336759241485626136995945307287254532463291529110128763770605570609531377527751867923292134955245133089867969165129073841302167573238637575820080363575728002754490327953079900799442541108725693188014667935595834676432868876966610097395749967836593397846346959948950610490383647409504695226063858046758073069912290474089879166872117147527644711604401952718169508289733537148530928937046384420893299771125856840846608339934045689026787516008775461267988015465856522061210953490796707365539702576199431376639960606061106406959330828171876426043573425361756943784848495250108266488395159700490598380812105221111091943323951136051446459834210799058082093716464523127704023160072138543723461267260997870385657091998507595634613248460188409850194287687902268734556500519121546544063829253851276317663922050938345204300773017029940362615434001322763910912988327863920412300445551684054889809080779174636092439334912641164240093880746356607262336695842764583698268734815881961058571835767462009650526065929263548291499045768307210893245857073701660717398194485028842603963660746031184786225831056580870870305567595861341700745402965687634774176431051751036732869245558582082372038601781739405175130437994868822320044378043103170921034261674998000073016094814586374488778522273076330495383944345382770608760763542098445008306247630253572781032783461766970544287155315340016497076657195985041748199087201490875686037783591994719343352772947285537925787684832301101859365800717291186967617655053775030293033830706448912811412025506150896411007623824574488655182581058140345320124754723269087547507078577659732542844459353044992070014538748948226556442223696365544194225441338212225477497535494624827680533336983284156138692363443358553868471111430498248398991803165458638289353799130535222833430137953372954016257623228081138499491876144141322933767106563492528814528239506209022357876684650116660097382753660405446941653422239052108314585847035529352219928272760574821266065291385530345549744551470344939486863429459658431024190785923680224560763936784166270518555178702904073557304620639692453307795782245949710420188043000183881429008173039450507342787013124466860092778581811040911511729374873627887874907465285565434748886831064110051023020875107768918781525622735251550379532444857787277617001964853703555167655209119339343762866284619844026295252183678522367475108809781507098978413086245881522660963551401874495836926917799047120726494905737264286005211403581231076006699518536124862746756375896225299116496066876508261734178484789337295056739007878617925351440621045366250640463728815698232317500596261080921955211150859302955654967538862612972339914628358476048627627027309739202001432248707582337354915246085608210328882974183906478869923273691360048837436615223517058437705545210815513361262142911815615301758882573594892507108879262128641392443309383797333867806131795237315266773820858024701433527009243803266951742119507670884326346442749127558907746863582162166042741315170212458586056233631493164646913946562497471741958354218607748711057338458433689939645913740603382159352243594751626239188685307822821763983237306180204246560477527943104796189724299533029792497481684052893791044947004590864991872727345413508101983881864673609392571930511968645601855782450218231065889437986522432050677379966196955472440585922417953006820451795370043472451762893566770508490213107736625751697335527462302943031203596260953423574397249659211010657817826108745318874803187430823573699195156340957162700992444929749105489851519658664740148225106335367949737142510229341882585117371994499115097583746130105505064197721531929354875371191630262030328588658528480193509225875775597425276584011721342323648084027143356367542046375182552524944329657043861387865901965738802868401894087672816714137033661732650120578653915780703088714261519075001492576112927675193096728453971160213606303090542243966320674323582797889332324405779199278484633339777737655901870574806828678347965624146102899508487399692970750432753029972872297327934442988646412725348160603779707298299173029296308695801996312413304939350493325412355071054461182591141116454534710329881047844067780138077131465400099386306481266614330858206811395838319169545558259426895769841428893743467084107946318932539106963955780706021245974898293564613560788983472419979478564362042094613412387613198865352358312996862268948608408456655606876954501274486631405054735351746873009806322780468912246821460806727627708402402266155485024008952891657117617439020337584877842911289623247059191874691042005848326140677333751027195653994697162517248312230633919328707983800748485726516123434933273356664473358556430235280883924348278760886164943289399166399210488307847777048045728491456303353265070029588906265915498509407972767567129795010098229476228961891591441520032283878773485130979081019129267227103778898053964156362364169154985768408398468861684375407065121039062506128107663799047908879674778069738473170475253442156390387201238806323688037017949308954900776331523063548374256816653361606641980030188287123767481898330246836371488309259283375902278942588060087286038859168849730693948020511221766359138251524278670094406942355120201568377778851824670025651708509249623747726813694284350062938814429987905301056217375459182679973217735029368928065210025396268807498092643458011655715886700443503976505323478287327368840863540002740676783821963522226539290939807367391364082898722017776747168118195856133721583119054682936083236976113450281757830202934845982925000895682630271263295866292147653142233351793093387951357095346377183684092444422096319331295620305575517340067973740614162107923633423805646850092037167152642556371853889571416419772387422610596667396997173168169415435095283193556417705668622215217991151355639707143312893657553844648326201206424338016955862698561022460646069330793847858814367407000599769703649019273328826135329363112403650698652160638987250267238087403396744397830258296894256896741864336134979475245526291426522842419243083388103580053787023999542172113686550275341362211693140694669513186928102574795985605145005021715913317751609957865551981886193211282110709442287240442481153406055895958355815232012184605820563592699303478851132068626627588771446035996656108430725696500563064489187599466596772847171539573612108180841547273142661748933134174632662354222072600146012701206934639520564445543291662986660783089068118790090815295063626782075614388815781351134695366303878412092346942868730839320432333872775496805210302821544324723388845215343727250128589747691460808314404125868181540049187772287869801853454537006526655649170915429522756709222217474112062720656622989806032891672068743654948246108697367225547404812889242471854323605753411672850757552057131156697954584887398742228135887985840783135060548290551482785294891121905383195624228719484759407859398047901094194070671764439032730712135887385049993638838205501683402777496070276844880281912220636888636811043569529300652195528261526991271637277388418993287130563464688227398288763198645709836308917786487086676185485680047672552675414742851028145807403152992197814557756843681110185317498167016426647884090262682824448258027532094549915104518517716546311804904567985713257528117913656278158111288816562285876030875974963849435275676612168959261485030785362045274507752950631012480341804584059432926079854435620093708091821523920371790678121992280496069738238743312626730306795943960954957189577217915597300588693646845576676092450906088202212235719254536715191834872587423919410890444115959932760044506556206461164655665487594247369252336955993030355095817626176231849561906494839673002037763874369343999829430209147073618947932692762445186560239559053705128978163455423320114975994896278424327483788032701418676952621180975006405149755889650293004867605208010491537885413909424531691719987628941277221129464568294860281493181560249677887949813777216229359437811004448060797672429276249510784153446429150842764520002042769470698041775832209097020291657347251582904630910359037842977572651720877244740952267166306005469716387943171196873484688738186656751279298575016363411314627530499019135646823804329970695770150789337728658035712790913767420805655493624646

lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-null-max.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-null-max.input deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.input and /dev/null differ

lib/std/compress/deflate/testdata/huffman-null-max.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-pi.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-pi.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-pi.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-pi.input deleted-1

...	@@ -1 +0,0 @@
1	3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-pi.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-pi.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-1k.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-1k.input deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.input and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-limit.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-limit.input deleted-4

...	@@ -1,4 +0,0 @@
1	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2	��vH
3	��%�� ɷ��}��>��ls��m�IGH��1Y�4�[�� 0[\|]o#�
4	�-#��ul��pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O��{��k�*�w(��b� ��kQC9/��lu>�5�C.��u�

lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-max.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-max.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-rand-max.input deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-rand-max.input and /dev/null differ

lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-shifts.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-shifts.input deleted-2

...	@@ -1,2 +0,0 @@
1	101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2	232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-shifts.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text-shift.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text-shift.input deleted-14

...	@@ -1,14 +0,0 @@
1	//Copyright2009ThGoAuthor.Allrightrrvd.
2	//UofthiourccodigovrndbyBSD-tyl
3	//licnthtcnbfoundinthLICENSEfil.
4
5	pckgmin
6
7	import"o"
8
9	funcmin(){
10	vrb=mk([]byt,65535)
11	f,_:=o.Crt("huffmn-null-mx.in")
12	f.Writ(b)
13	}
14	ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text.input deleted-14

...	@@ -1,14 +0,0 @@
1	// zig v0.10.0
2	// create a file filled with 0x00
3	const std = @import("std");
4
5	pub fn main() !void {
6	var b = [1]u8{0} ** 65535;
7	const f = try std.fs.cwd().createFile(
8	"huffman-null-max.in",
9	.{ .read = true },
10	);
11	defer f.close();
12
13	_ = try f.writeAll(b[0..]);
14	}

lib/std/compress/deflate/testdata/huffman-text.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-zero.dyn.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/huffman-zero.golden deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-zero.golden and /dev/null differ

lib/std/compress/deflate/testdata/huffman-zero.input deleted-1

...	@@ -1 +0,0 @@
1	00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-zero.wb.expect deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-zero.wb.expect and /dev/null differ

lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput deleted

Binary files a/lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput and /dev/null differ

lib/std/compress/deflate/testdata/rfc1951.txt deleted-955

...	@@ -1,955 +0,0 @@
1
2
3
4
5
6
7	Network Working Group P. Deutsch
8	Request for Comments: 1951 Aladdin Enterprises
9	Category: Informational May 1996
10
11
12	DEFLATE Compressed Data Format Specification version 1.3
13
14	Status of This Memo
15
16	This memo provides information for the Internet community. This memo
17	does not specify an Internet standard of any kind. Distribution of
18	this memo is unlimited.
19
20	IESG Note:
21
22	The IESG takes no position on the validity of any Intellectual
23	Property Rights statements contained in this document.
24
25	Notices
26
27	Copyright (c) 1996 L. Peter Deutsch
28
29	Permission is granted to copy and distribute this document for any
30	purpose and without charge, including translations into other
31	languages and incorporation into compilations, provided that the
32	copyright notice and this notice are preserved, and that any
33	substantive changes or deletions from the original are clearly
34	marked.
35
36	A pointer to the latest version of this and related documentation in
37	HTML format can be found at the URL
38	<ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40	Abstract
41
42	This specification defines a lossless compressed data format that
43	compresses data using a combination of the LZ77 algorithm and Huffman
44	coding, with efficiency comparable to the best currently available
45	general-purpose compression methods. The data can be produced or
46	consumed, even for an arbitrarily long sequentially presented input
47	data stream, using only an a priori bounded amount of intermediate
48	storage. The format can be implemented readily in a manner not
49	covered by patents.
50
51
52
53
54
55
56
57
58	Deutsch Informational [Page 1]
59
60	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
61
62
63	Table of Contents
64
65	1. Introduction ................................................... 2
66	1.1. Purpose ................................................... 2
67	1.2. Intended audience ......................................... 3
68	1.3. Scope ..................................................... 3
69	1.4. Compliance ................................................ 3
70	1.5. Definitions of terms and conventions used ................ 3
71	1.6. Changes from previous versions ............................ 4
72	2. Compressed representation overview ............................. 4
73	3. Detailed specification ......................................... 5
74	3.1. Overall conventions ....................................... 5
75	3.1.1. Packing into bytes .................................. 5
76	3.2. Compressed block format ................................... 6
77	3.2.1. Synopsis of prefix and Huffman coding ............... 6
78	3.2.2. Use of Huffman coding in the "deflate" format ....... 7
79	3.2.3. Details of block format ............................. 9
80	3.2.4. Non-compressed blocks (BTYPE=00) ................... 11
81	3.2.5. Compressed blocks (length and distance codes) ...... 11
82	3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12
83	3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13
84	3.3. Compliance ............................................... 14
85	4. Compression algorithm details ................................. 14
86	5. References .................................................... 16
87	6. Security Considerations ....................................... 16
88	7. Source code ................................................... 16
89	8. Acknowledgements .............................................. 16
90	9. Author's Address .............................................. 17
91
92	1. Introduction
93
94	1.1. Purpose
95
96	The purpose of this specification is to define a lossless
97	compressed data format that:
98	* Is independent of CPU type, operating system, file system,
99	and character set, and hence can be used for interchange;
100	* Can be produced or consumed, even for an arbitrarily long
101	sequentially presented input data stream, using only an a
102	priori bounded amount of intermediate storage, and hence
103	can be used in data communications or similar structures
104	such as Unix filters;
105	* Compresses data with efficiency comparable to the best
106	currently available general-purpose compression methods,
107	and in particular considerably better than the "compress"
108	program;
109	* Can be implemented readily in a manner not covered by
110	patents, and hence can be practiced freely;
111
112
113
114	Deutsch Informational [Page 2]
115
116	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
117
118
119	* Is compatible with the file format produced by the current
120	widely used gzip utility, in that conforming decompressors
121	will be able to read data produced by the existing gzip
122	compressor.
123
124	The data format defined by this specification does not attempt to:
125
126	* Allow random access to compressed data;
127	* Compress specialized data (e.g., raster graphics) as well
128	as the best currently available specialized algorithms.
129
130	A simple counting argument shows that no lossless compression
131	algorithm can compress every possible input data set. For the
132	format defined here, the worst case expansion is 5 bytes per 32K-
133	byte block, i.e., a size increase of 0.015% for large data sets.
134	English text usually compresses by a factor of 2.5 to 3;
135	executable files usually compress somewhat less; graphical data
136	such as raster images may compress much more.
137
138	1.2. Intended audience
139
140	This specification is intended for use by implementors of software
141	to compress data into "deflate" format and/or decompress data from
142	"deflate" format.
143
144	The text of the specification assumes a basic background in
145	programming at the level of bits and other primitive data
146	representations. Familiarity with the technique of Huffman coding
147	is helpful but not required.
148
149	1.3. Scope
150
151	The specification specifies a method for representing a sequence
152	of bytes as a (usually shorter) sequence of bits, and a method for
153	packing the latter bit sequence into bytes.
154
155	1.4. Compliance
156
157	Unless otherwise indicated below, a compliant decompressor must be
158	able to accept and decompress any data set that conforms to all
159	the specifications presented here; a compliant compressor must
160	produce data sets that conform to all the specifications presented
161	here.
162
163	1.5. Definitions of terms and conventions used
164
165	Byte: 8 bits stored or transmitted as a unit (same as an octet).
166	For this specification, a byte is exactly 8 bits, even on machines
167
168
169
170	Deutsch Informational [Page 3]
171
172	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
173
174
175	which store a character on a number of bits different from eight.
176	See below, for the numbering of bits within a byte.
177
178	String: a sequence of arbitrary bytes.
179
180	1.6. Changes from previous versions
181
182	There have been no technical changes to the deflate format since
183	version 1.1 of this specification. In version 1.2, some
184	terminology was changed. Version 1.3 is a conversion of the
185	specification to RFC style.
186
187	2. Compressed representation overview
188
189	A compressed data set consists of a series of blocks, corresponding
190	to successive blocks of input data. The block sizes are arbitrary,
191	except that non-compressible blocks are limited to 65,535 bytes.
192
193	Each block is compressed using a combination of the LZ77 algorithm
194	and Huffman coding. The Huffman trees for each block are independent
195	of those for previous or subsequent blocks; the LZ77 algorithm may
196	use a reference to a duplicated string occurring in a previous block,
197	up to 32K input bytes before.
198
199	Each block consists of two parts: a pair of Huffman code trees that
200	describe the representation of the compressed data part, and a
201	compressed data part. (The Huffman trees themselves are compressed
202	using Huffman encoding.) The compressed data consists of a series of
203	elements of two types: literal bytes (of strings that have not been
204	detected as duplicated within the previous 32K input bytes), and
205	pointers to duplicated strings, where a pointer is represented as a
206	pair <length, backward distance>. The representation used in the
207	"deflate" format limits distances to 32K bytes and lengths to 258
208	bytes, but does not limit the size of a block, except for
209	uncompressible blocks, which are limited as noted above.
210
211	Each type of value (literals, distances, and lengths) in the
212	compressed data is represented using a Huffman code, using one code
213	tree for literals and lengths and a separate code tree for distances.
214	The code trees for each block appear in a compact form just before
215	the compressed data for that block.
216
217
218
219
220
221
222
223
224
225
226	Deutsch Informational [Page 4]
227
228	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
229
230
231	3. Detailed specification
232
233	3.1. Overall conventions In the diagrams below, a box like this:
234
235	+---+
236	\| \| <-- the vertical bars might be missing
237	+---+
238
239	represents one byte; a box like this:
240
241	+==============+
242	\| \|
243	+==============+
244
245	represents a variable number of bytes.
246
247	Bytes stored within a computer do not have a "bit order", since
248	they are always treated as a unit. However, a byte considered as
249	an integer between 0 and 255 does have a most- and least-
250	significant bit, and since we write numbers with the most-
251	significant digit on the left, we also write bytes with the most-
252	significant bit on the left. In the diagrams below, we number the
253	bits of a byte so that bit 0 is the least-significant bit, i.e.,
254	the bits are numbered:
255
256	+--------+
257	\|76543210\|
258	+--------+
259
260	Within a computer, a number may occupy multiple bytes. All
261	multi-byte numbers in the format described here are stored with
262	the least-significant byte first (at the lower memory address).
263	For example, the decimal number 520 is stored as:
264
265	0 1
266	+--------+--------+
267	\|00001000\|00000010\|
268	+--------+--------+
269	^ ^
270	\| \|
271	\| + more significant byte = 2 x 256
272	+ less significant byte = 8
273
274	3.1.1. Packing into bytes
275
276	This document does not address the issue of the order in which
277	bits of a byte are transmitted on a bit-sequential medium,
278	since the final data format described here is byte- rather than
279
280
281
282	Deutsch Informational [Page 5]
283
284	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
285
286
287	bit-oriented. However, we describe the compressed block format
288	in below, as a sequence of data elements of various bit
289	lengths, not a sequence of bytes. We must therefore specify
290	how to pack these data elements into bytes to form the final
291	compressed byte sequence:
292
293	* Data elements are packed into bytes in order of
294	increasing bit number within the byte, i.e., starting
295	with the least-significant bit of the byte.
296	* Data elements other than Huffman codes are packed
297	starting with the least-significant bit of the data
298	element.
299	* Huffman codes are packed starting with the most-
300	significant bit of the code.
301
302	In other words, if one were to print out the compressed data as
303	a sequence of bytes, starting with the first byte at the
304	right margin and proceeding to the left, with the most-
305	significant bit of each byte on the left as usual, one would be
306	able to parse the result from right to left, with fixed-width
307	elements in the correct MSB-to-LSB order and Huffman codes in
308	bit-reversed order (i.e., with the first bit of the code in the
309	relative LSB position).
310
311	3.2. Compressed block format
312
313	3.2.1. Synopsis of prefix and Huffman coding
314
315	Prefix coding represents symbols from an a priori known
316	alphabet by bit sequences (codes), one code for each symbol, in
317	a manner such that different symbols may be represented by bit
318	sequences of different lengths, but a parser can always parse
319	an encoded string unambiguously symbol-by-symbol.
320
321	We define a prefix code in terms of a binary tree in which the
322	two edges descending from each non-leaf node are labeled 0 and
323	1 and in which the leaf nodes correspond one-for-one with (are
324	labeled with) the symbols of the alphabet; then the code for a
325	symbol is the sequence of 0's and 1's on the edges leading from
326	the root to the leaf labeled with that symbol. For example:
327
328
329
330
331
332
333
334
335
336
337
338	Deutsch Informational [Page 6]
339
340	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
341
342
343	/\ Symbol Code
344	0 1 ------ ----
345	/ \ A 00
346	/\ B B 1
347	0 1 C 011
348	/ \ D 010
349	A /\
350	0 1
351	/ \
352	D C
353
354	A parser can decode the next symbol from an encoded input
355	stream by walking down the tree from the root, at each step
356	choosing the edge corresponding to the next input bit.
357
358	Given an alphabet with known symbol frequencies, the Huffman
359	algorithm allows the construction of an optimal prefix code
360	(one which represents strings with those symbol frequencies
361	using the fewest bits of any possible prefix codes for that
362	alphabet). Such a code is called a Huffman code. (See
363	reference [1] in Chapter 5, references for additional
364	information on Huffman codes.)
365
366	Note that in the "deflate" format, the Huffman codes for the
367	various alphabets must not exceed certain maximum code lengths.
368	This constraint complicates the algorithm for computing code
369	lengths from symbol frequencies. Again, see Chapter 5,
370	references for details.
371
372	3.2.2. Use of Huffman coding in the "deflate" format
373
374	The Huffman codes used for each alphabet in the "deflate"
375	format have two additional rules:
376
377	* All codes of a given bit length have lexicographically
378	consecutive values, in the same order as the symbols
379	they represent;
380
381	* Shorter codes lexicographically precede longer codes.
382
383
384
385
386
387
388
389
390
391
392
393
394	Deutsch Informational [Page 7]
395
396	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
397
398
399	We could recode the example above to follow this rule as
400	follows, assuming that the order of the alphabet is ABCD:
401
402	Symbol Code
403	------ ----
404	A 10
405	B 0
406	C 110
407	D 111
408
409	I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are
410	lexicographically consecutive.
411
412	Given this rule, we can define the Huffman code for an alphabet
413	just by giving the bit lengths of the codes for each symbol of
414	the alphabet in order; this is sufficient to determine the
415	actual codes. In our example, the code is completely defined
416	by the sequence of bit lengths (2, 1, 3, 3). The following
417	algorithm generates the codes as integers, intended to be read
418	from most- to least-significant bit. The code lengths are
419	initially in tree[I].Len; the codes are produced in
420	tree[I].Code.
421
422	1) Count the number of codes for each code length. Let
423	bl_count[N] be the number of codes of length N, N >= 1.
424
425	2) Find the numerical value of the smallest code for each
426	code length:
427
428	code = 0;
429	bl_count[0] = 0;
430	for (bits = 1; bits <= MAX_BITS; bits++) {
431	code = (code + bl_count[bits-1]) << 1;
432	next_code[bits] = code;
433	}
434
435	3) Assign numerical values to all codes, using consecutive
436	values for all codes of the same length with the base
437	values determined at step 2. Codes that are never used
438	(which have a bit length of zero) must not be assigned a
439	value.
440
441	for (n = 0; n <= max_code; n++) {
442	len = tree[n].Len;
443	if (len != 0) {
444	tree[n].Code = next_code[len];
445	next_code[len]++;
446	}
447
448
449
450	Deutsch Informational [Page 8]
451
452	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
453
454
455	}
456
457	Example:
458
459	Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3,
460	3, 2, 4, 4). After step 1, we have:
461
462	N bl_count[N]
463	- -----------
464	2 1
465	3 5
466	4 2
467
468	Step 2 computes the following next_code values:
469
470	N next_code[N]
471	- ------------
472	1 0
473	2 0
474	3 2
475	4 14
476
477	Step 3 produces the following code values:
478
479	Symbol Length Code
480	------ ------ ----
481	A 3 010
482	B 3 011
483	C 3 100
484	D 3 101
485	E 3 110
486	F 2 00
487	G 4 1110
488	H 4 1111
489
490	3.2.3. Details of block format
491
492	Each block of compressed data begins with 3 header bits
493	containing the following data:
494
495	first bit BFINAL
496	next 2 bits BTYPE
497
498	Note that the header bits do not necessarily begin on a byte
499	boundary, since a block does not necessarily occupy an integral
500	number of bytes.
501
502
503
504
505
506	Deutsch Informational [Page 9]
507
508	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
509
510
511	BFINAL is set if and only if this is the last block of the data
512	set.
513
514	BTYPE specifies how the data are compressed, as follows:
515
516	00 - no compression
517	01 - compressed with fixed Huffman codes
518	10 - compressed with dynamic Huffman codes
519	11 - reserved (error)
520
521	The only difference between the two compressed cases is how the
522	Huffman codes for the literal/length and distance alphabets are
523	defined.
524
525	In all cases, the decoding algorithm for the actual data is as
526	follows:
527
528	do
529	read block header from input stream.
530	if stored with no compression
531	skip any remaining bits in current partially
532	processed byte
533	read LEN and NLEN (see next section)
534	copy LEN bytes of data to output
535	otherwise
536	if compressed with dynamic Huffman codes
537	read representation of code trees (see
538	subsection below)
539	loop (until end of block code recognized)
540	decode literal/length value from input stream
541	if value < 256
542	copy value (literal byte) to output stream
543	otherwise
544	if value = end of block (256)
545	break from loop
546	otherwise (value = 257..285)
547	decode distance from input stream
548
549	move backwards distance bytes in the output
550	stream, and copy length bytes from this
551	position to the output stream.
552	end loop
553	while not last block
554
555	Note that a duplicated string reference may refer to a string
556	in a previous block; i.e., the backward distance may cross one
557	or more block boundaries. However a distance cannot refer past
558	the beginning of the output stream. (An application using a
559
560
561
562	Deutsch Informational [Page 10]
563
564	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
565
566
567	preset dictionary might discard part of the output stream; a
568	distance can refer to that part of the output stream anyway)
569	Note also that the referenced string may overlap the current
570	position; for example, if the last 2 bytes decoded have values
571	X and Y, a string reference with <length = 5, distance = 2>
572	adds X,Y,X,Y,X to the output stream.
573
574	We now specify each compression method in turn.
575
576	3.2.4. Non-compressed blocks (BTYPE=00)
577
578	Any bits of input up to the next byte boundary are ignored.
579	The rest of the block consists of the following information:
580
581	0 1 2 3 4...
582	+---+---+---+---+================================+
583	\| LEN \| NLEN \|... LEN bytes of literal data...\|
584	+---+---+---+---+================================+
585
586	LEN is the number of data bytes in the block. NLEN is the
587	one's complement of LEN.
588
589	3.2.5. Compressed blocks (length and distance codes)
590
591	As noted above, encoded data blocks in the "deflate" format
592	consist of sequences of symbols drawn from three conceptually
593	distinct alphabets: either literal bytes, from the alphabet of
594	byte values (0..255), or <length, backward distance> pairs,
595	where the length is drawn from (3..258) and the distance is
596	drawn from (1..32,768). In fact, the literal and length
597	alphabets are merged into a single alphabet (0..285), where
598	values 0..255 represent literal bytes, the value 256 indicates
599	end-of-block, and values 257..285 represent length codes
600	(possibly in conjunction with extra bits following the symbol
601	code) as follows:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618	Deutsch Informational [Page 11]
619
620	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
621
622
623	Extra Extra Extra
624	Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
625	---- ---- ------ ---- ---- ------- ---- ---- -------
626	257 0 3 267 1 15,16 277 4 67-82
627	258 0 4 268 1 17,18 278 4 83-98
628	259 0 5 269 2 19-22 279 4 99-114
629	260 0 6 270 2 23-26 280 4 115-130
630	261 0 7 271 2 27-30 281 5 131-162
631	262 0 8 272 2 31-34 282 5 163-194
632	263 0 9 273 3 35-42 283 5 195-226
633	264 0 10 274 3 43-50 284 5 227-257
634	265 1 11,12 275 3 51-58 285 0 258
635	266 1 13,14 276 3 59-66
636
637	The extra bits should be interpreted as a machine integer
638	stored with the most-significant bit first, e.g., bits 1110
639	represent the value 14.
640
641	Extra Extra Extra
642	Code Bits Dist Code Bits Dist Code Bits Distance
643	---- ---- ---- ---- ---- ------ ---- ---- --------
644	0 0 1 10 4 33-48 20 9 1025-1536
645	1 0 2 11 4 49-64 21 9 1537-2048
646	2 0 3 12 5 65-96 22 10 2049-3072
647	3 0 4 13 5 97-128 23 10 3073-4096
648	4 1 5,6 14 6 129-192 24 11 4097-6144
649	5 1 7,8 15 6 193-256 25 11 6145-8192
650	6 2 9-12 16 7 257-384 26 12 8193-12288
651	7 2 13-16 17 7 385-512 27 12 12289-16384
652	8 3 17-24 18 8 513-768 28 13 16385-24576
653	9 3 25-32 19 8 769-1024 29 13 24577-32768
654
655	3.2.6. Compression with fixed Huffman codes (BTYPE=01)
656
657	The Huffman codes for the two alphabets are fixed, and are not
658	represented explicitly in the data. The Huffman code lengths
659	for the literal/length alphabet are:
660
661	Lit Value Bits Codes
662	--------- ---- -----
663	0 - 143 8 00110000 through
664	10111111
665	144 - 255 9 110010000 through
666	111111111
667	256 - 279 7 0000000 through
668	0010111
669	280 - 287 8 11000000 through
670	11000111
671
672
673
674	Deutsch Informational [Page 12]
675
676	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
677
678
679	The code lengths are sufficient to generate the actual codes,
680	as described above; we show the codes in the table for added
681	clarity. Literal/length values 286-287 will never actually
682	occur in the compressed data, but participate in the code
683	construction.
684
685	Distance codes 0-31 are represented by (fixed-length) 5-bit
686	codes, with possible additional bits as shown in the table
687	shown in Paragraph 3.2.5, above. Note that distance codes 30-
688	31 will never actually occur in the compressed data.
689
690	3.2.7. Compression with dynamic Huffman codes (BTYPE=10)
691
692	The Huffman codes for the two alphabets appear in the block
693	immediately after the header bits and before the actual
694	compressed data, first the literal/length code and then the
695	distance code. Each code is defined by a sequence of code
696	lengths, as discussed in Paragraph 3.2.2, above. For even
697	greater compactness, the code length sequences themselves are
698	compressed using a Huffman code. The alphabet for code lengths
699	is as follows:
700
701	0 - 15: Represent code lengths of 0 - 15
702	16: Copy the previous code length 3 - 6 times.
703	The next 2 bits indicate repeat length
704	(0 = 3, ... , 3 = 6)
705	Example: Codes 8, 16 (+2 bits 11),
706	16 (+2 bits 10) will expand to
707	12 code lengths of 8 (1 + 6 + 5)
708	17: Repeat a code length of 0 for 3 - 10 times.
709	(3 bits of length)
710	18: Repeat a code length of 0 for 11 - 138 times
711	(7 bits of length)
712
713	A code length of 0 indicates that the corresponding symbol in
714	the literal/length or distance alphabet will not occur in the
715	block, and should not participate in the Huffman code
716	construction algorithm given earlier. If only one distance
717	code is used, it is encoded using one bit, not zero bits; in
718	this case there is a single code length of one, with one unused
719	code. One distance code of zero bits means that there are no
720	distance codes used at all (the data is all literals).
721
722	We can now define the format of the block:
723
724	5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286)
725	5 Bits: HDIST, # of Distance codes - 1 (1 - 32)
726	4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19)
727
728
729
730	Deutsch Informational [Page 13]
731
732	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
733
734
735	(HCLEN + 4) x 3 bits: code lengths for the code length
736	alphabet given just above, in the order: 16, 17, 18,
737	0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
738
739	These code lengths are interpreted as 3-bit integers
740	(0-7); as above, a code length of 0 means the
741	corresponding symbol (literal/length or distance code
742	length) is not used.
743
744	HLIT + 257 code lengths for the literal/length alphabet,
745	encoded using the code length Huffman code
746
747	HDIST + 1 code lengths for the distance alphabet,
748	encoded using the code length Huffman code
749
750	The actual compressed data of the block,
751	encoded using the literal/length and distance Huffman
752	codes
753
754	The literal/length symbol 256 (end of data),
755	encoded using the literal/length Huffman code
756
757	The code length repeat codes can cross from HLIT + 257 to the
758	HDIST + 1 code lengths. In other words, all code lengths form
759	a single sequence of HLIT + HDIST + 258 values.
760
761	3.3. Compliance
762
763	A compressor may limit further the ranges of values specified in
764	the previous section and still be compliant; for example, it may
765	limit the range of backward pointers to some value smaller than
766	32K. Similarly, a compressor may limit the size of blocks so that
767	a compressible block fits in memory.
768
769	A compliant decompressor must accept the full range of possible
770	values defined in the previous section, and must accept blocks of
771	arbitrary size.
772
773	4. Compression algorithm details
774
775	While it is the intent of this document to define the "deflate"
776	compressed data format without reference to any particular
777	compression algorithm, the format is related to the compressed
778	formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below);
779	since many variations of LZ77 are patented, it is strongly
780	recommended that the implementor of a compressor follow the general
781	algorithm presented here, which is known not to be patented per se.
782	The material in this section is not part of the definition of the
783
784
785
786	Deutsch Informational [Page 14]
787
788	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
789
790
791	specification per se, and a compressor need not follow it in order to
792	be compliant.
793
794	The compressor terminates a block when it determines that starting a
795	new block with fresh trees would be useful, or when the block size
796	fills up the compressor's block buffer.
797
798	The compressor uses a chained hash table to find duplicated strings,
799	using a hash function that operates on 3-byte sequences. At any
800	given point during compression, let XYZ be the next 3 input bytes to
801	be examined (not necessarily all different, of course). First, the
802	compressor examines the hash chain for XYZ. If the chain is empty,
803	the compressor simply writes out X as a literal byte and advances one
804	byte in the input. If the hash chain is not empty, indicating that
805	the sequence XYZ (or, if we are unlucky, some other 3 bytes with the
806	same hash function value) has occurred recently, the compressor
807	compares all strings on the XYZ hash chain with the actual input data
808	sequence starting at the current point, and selects the longest
809	match.
810
811	The compressor searches the hash chains starting with the most recent
812	strings, to favor small distances and thus take advantage of the
813	Huffman encoding. The hash chains are singly linked. There are no
814	deletions from the hash chains; the algorithm simply discards matches
815	that are too old. To avoid a worst-case situation, very long hash
816	chains are arbitrarily truncated at a certain length, determined by a
817	run-time parameter.
818
819	To improve overall compression, the compressor optionally defers the
820	selection of matches ("lazy matching"): after a match of length N has
821	been found, the compressor searches for a longer match starting at
822	the next input byte. If it finds a longer match, it truncates the
823	previous match to a length of one (thus producing a single literal
824	byte) and then emits the longer match. Otherwise, it emits the
825	original match, and, as described above, advances N bytes before
826	continuing.
827
828	Run-time parameters also control this "lazy match" procedure. If
829	compression ratio is most important, the compressor attempts a
830	complete second search regardless of the length of the first match.
831	In the normal case, if the current match is "long enough", the
832	compressor reduces the search for a longer match, thus speeding up
833	the process. If speed is most important, the compressor inserts new
834	strings in the hash table only when no match was found, or when the
835	match is not "too long". This degrades the compression ratio but
836	saves time since there are both fewer insertions and fewer searches.
837
838
839
840
841
842	Deutsch Informational [Page 15]
843
844	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
845
846
847	5. References
848
849	[1] Huffman, D. A., "A Method for the Construction of Minimum
850	Redundancy Codes", Proceedings of the Institute of Radio
851	Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101.
852
853	[2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data
854	Compression", IEEE Transactions on Information Theory, Vol. 23,
855	No. 3, pp. 337-343.
856
857	[3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources,
858	available in ftp://ftp.uu.net/pub/archiving/zip/doc/
859
860	[4] Gailly, J.-L., and Adler, M., GZIP documentation and sources,
861	available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/
862
863	[5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix
864	encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169.
865
866	[6] Hirschberg and Lelewer, "Efficient decoding of prefix codes,"
867	Comm. ACM, 33,4, April 1990, pp. 449-459.
868
869	6. Security Considerations
870
871	Any data compression method involves the reduction of redundancy in
872	the data. Consequently, any corruption of the data is likely to have
873	severe effects and be difficult to correct. Uncompressed text, on
874	the other hand, will probably still be readable despite the presence
875	of some corrupted bytes.
876
877	It is recommended that systems using this data format provide some
878	means of validating the integrity of the compressed data. See
879	reference [3], for example.
880
881	7. Source code
882
883	Source code for a C language implementation of a "deflate" compliant
884	compressor and decompressor is available within the zlib package at
885	ftp://ftp.uu.net/pub/archiving/zip/zlib/.
886
887	8. Acknowledgements
888
889	Trademarks cited in this document are the property of their
890	respective owners.
891
892	Phil Katz designed the deflate format. Jean-Loup Gailly and Mark
893	Adler wrote the related software described in this specification.
894	Glenn Randers-Pehrson converted this document to RFC and HTML format.
895
896
897
898	Deutsch Informational [Page 16]
899
900	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
901
902
903	9. Author's Address
904
905	L. Peter Deutsch
906	Aladdin Enterprises
907	203 Santa Margarita Ave.
908	Menlo Park, CA 94025
909
910	Phone: (415) 322-0103 (AM only)
911	FAX: (415) 322-1734
912	EMail: <ghost@aladdin.com>
913
914	Questions about the technical content of this specification can be
915	sent by email to:
916
917	Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
918	Mark Adler <madler@alumni.caltech.edu>
919
920	Editorial comments on this specification can be sent by email to:
921
922	L. Peter Deutsch <ghost@aladdin.com> and
923	Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954	Deutsch Informational [Page 17]
955

lib/std/compress/deflate/token.zig deleted-103

...	@@ -1,103 +0,0 @@
1	// 2 bits: type, can be 0 (literal), 1 (EOF), 2 (Match) or 3 (Unused).
2	// 8 bits: xlength (length - MIN_MATCH_LENGTH).
3	// 22 bits: xoffset (offset - MIN_OFFSET_SIZE), or literal.
4	const length_shift = 22;
5	const offset_mask = (1 << length_shift) - 1; // 4_194_303
6	const literal_type = 0 << 30; // 0
7	pub const match_type = 1 << 30; // 1_073_741_824
8
9	// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
10	// is length_codes[length - MIN_MATCH_LENGTH]
11	var length_codes = [_]u32{
12	0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
13	9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
14	13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
15	15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
16	17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
17	18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
18	19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
19	20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20	21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21	21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
22	22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
23	22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
24	23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
25	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
26	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
27	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
28	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
29	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
30	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
31	25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
32	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
33	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
34	26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
35	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
36	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
37	27, 27, 27, 27, 27, 28,
38	};
39
40	var offset_codes = [_]u32{
41	0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
42	8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
43	10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
44	11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
45	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
46	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
47	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
48	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
49	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
50	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
51	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
52	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
53	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
54	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
55	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
56	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
57	};
58
59	pub const Token = u32;
60
61	// Convert a literal into a literal token.
62	pub fn literalToken(lit: u32) Token {
63	return literal_type + lit;
64	}
65
66	// Convert a < xlength, xoffset > pair into a match token.
67	pub fn matchToken(xlength: u32, xoffset: u32) Token {
68	return match_type + (xlength << length_shift) + xoffset;
69	}
70
71	// Returns the literal of a literal token
72	pub fn literal(t: Token) u32 {
73	return @as(u32, @intCast(t - literal_type));
74	}
75
76	// Returns the extra offset of a match token
77	pub fn offset(t: Token) u32 {
78	return @as(u32, @intCast(t)) & offset_mask;
79	}
80
81	pub fn length(t: Token) u32 {
82	return @as(u32, @intCast((t - match_type) >> length_shift));
83	}
84
85	pub fn lengthCode(len: u32) u32 {
86	return length_codes[len];
87	}
88
89	// Returns the offset code corresponding to a specific offset
90	pub fn offsetCode(off: u32) u32 {
91	if (off < @as(u32, @intCast(offset_codes.len))) {
92	return offset_codes[off];
93	}
94	if (off >> 7 < @as(u32, @intCast(offset_codes.len))) {
95	return offset_codes[off >> 7] + 14;
96	}
97	return offset_codes[off >> 14] + 28;
98	}
99
100	test {
101	const std = @import("std");
102	try std.testing.expectEqual(@as(Token, 3_401_581_099), matchToken(555, 555));
103	}

lib/std/compress/flate.zig created+481

...	@@ -0,0 +1,481 @@
1	/// Deflate is a lossless data compression file format that uses a combination
2	/// of LZ77 and Huffman coding.
3	pub const deflate = @import("flate/deflate.zig");
4
5	/// Inflate is the decoding process that takes a Deflate bitstream for
6	/// decompression and correctly produces the original full-size data or file.
7	pub const inflate = @import("flate/inflate.zig");
8
9	/// Decompress compressed data from reader and write plain data to the writer.
10	pub fn decompress(reader: anytype, writer: anytype) !void {
11	try inflate.decompress(.raw, reader, writer);
12	}
13
14	/// Decompressor type
15	pub fn Decompressor(comptime ReaderType: type) type {
16	return inflate.Inflate(.raw, ReaderType);
17	}
18
19	/// Create Decompressor which will read compressed data from reader.
20	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
21	return inflate.decompressor(.raw, reader);
22	}
23
24	/// Compression level, trades between speed and compression size.
25	pub const Options = deflate.Options;
26
27	/// Compress plain data from reader and write compressed data to the writer.
28	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
29	try deflate.compress(.raw, reader, writer, options);
30	}
31
32	/// Compressor type
33	pub fn Compressor(comptime WriterType: type) type {
34	return deflate.Compressor(.raw, WriterType);
35	}
36
37	/// Create Compressor which outputs compressed data to the writer.
38	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
39	return try deflate.compressor(.raw, writer, options);
40	}
41
42	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
43	/// compression, less memory requirements but bigger compressed sizes.
44	pub const huffman = struct {
45	pub fn compress(reader: anytype, writer: anytype) !void {
46	try deflate.huffman.compress(.raw, reader, writer);
47	}
48
49	pub fn Compressor(comptime WriterType: type) type {
50	return deflate.huffman.Compressor(.raw, WriterType);
51	}
52
53	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
54	return deflate.huffman.compressor(.raw, writer);
55	}
56	};
57
58	// No compression store only. Compressed size is slightly bigger than plain.
59	pub const store = struct {
60	pub fn compress(reader: anytype, writer: anytype) !void {
61	try deflate.store.compress(.raw, reader, writer);
62	}
63
64	pub fn Compressor(comptime WriterType: type) type {
65	return deflate.store.Compressor(.raw, WriterType);
66	}
67
68	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
69	return deflate.store.compressor(.raw, writer);
70	}
71	};
72
73	/// Container defines header/footer arround deflate bit stream. Gzip and zlib
74	/// compression algorithms are containers arround deflate bit stream body.
75	const Container = @import("flate/container.zig").Container;
76	const std = @import("std");
77	const testing = std.testing;
78	const fixedBufferStream = std.io.fixedBufferStream;
79	const print = std.debug.print;
80	const builtin = @import("builtin");
81
82	test "flate" {
83	_ = @import("flate/deflate.zig");
84	_ = @import("flate/inflate.zig");
85	}
86
87	test "flate compress/decompress" {
88	if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest;
89
90	var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer
91	var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer
92
93	const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
94	const cases = [_]struct {
95	data: []const u8, // uncompressed content
96	// compressed data sizes per level 4-9
97	gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len,
98	huffman_only_size: usize = 0,
99	store_size: usize = 0,
100	}{
101	.{
102	.data = @embedFile("flate/testdata/rfc1951.txt"),
103	.gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 },
104	.huffman_only_size = 20287,
105	.store_size = 36967,
106	},
107	.{
108	.data = @embedFile("flate/testdata/fuzz/roundtrip1.input"),
109	.gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 },
110	.huffman_only_size = 393,
111	.store_size = 393,
112	},
113	.{
114	.data = @embedFile("flate/testdata/fuzz/roundtrip2.input"),
115	.gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 },
116	.huffman_only_size = 394,
117	.store_size = 394,
118	},
119	.{
120	.data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"),
121	.gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 },
122	.huffman_only_size = 498,
123	.store_size = 747,
124	},
125	};
126
127	for (cases, 0..) \|case, case_no\| { // for each case
128	const data = case.data;
129
130	for (levels, 0..) \|level, i\| { // for each compression level
131
132	inline for (Container.list) \|container\| { // for each wrapping
133	var compressed_size: usize = if (case.gzip_sizes[i] > 0)
134	case.gzip_sizes[i] - Container.gzip.size() + container.size()
135	else
136	0;
137
138	// compress original stream to compressed stream
139	{
140	var original = fixedBufferStream(data);
141	var compressed = fixedBufferStream(&cmp_buf);
142	try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level });
143	if (compressed_size == 0) {
144	if (container == .gzip)
145	print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
146	compressed_size = compressed.pos;
147	}
148	try testing.expectEqual(compressed_size, compressed.pos);
149	}
150	// decompress compressed stream to decompressed stream
151	{
152	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
153	var decompressed = fixedBufferStream(&dcm_buf);
154	try inflate.decompress(container, compressed.reader(), decompressed.writer());
155	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
156	}
157
158	// compressor writer interface
159	{
160	var compressed = fixedBufferStream(&cmp_buf);
161	var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level });
162	var cmp_wrt = cmp.writer();
163	try cmp_wrt.writeAll(data);
164	try cmp.finish();
165
166	try testing.expectEqual(compressed_size, compressed.pos);
167	}
168	// decompressor reader interface
169	{
170	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
171	var dcm = inflate.decompressor(container, compressed.reader());
172	var dcm_rdr = dcm.reader();
173	const n = try dcm_rdr.readAll(&dcm_buf);
174	try testing.expectEqual(data.len, n);
175	try testing.expectEqualSlices(u8, data, dcm_buf[0..n]);
176	}
177	}
178	}
179	// huffman only compression
180	{
181	inline for (Container.list) \|container\| { // for each wrapping
182	var compressed_size: usize = if (case.huffman_only_size > 0)
183	case.huffman_only_size - Container.gzip.size() + container.size()
184	else
185	0;
186
187	// compress original stream to compressed stream
188	{
189	var original = fixedBufferStream(data);
190	var compressed = fixedBufferStream(&cmp_buf);
191	var cmp = try deflate.huffman.compressor(container, compressed.writer());
192	try cmp.compress(original.reader());
193	try cmp.finish();
194	if (compressed_size == 0) {
195	if (container == .gzip)
196	print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos });
197	compressed_size = compressed.pos;
198	}
199	try testing.expectEqual(compressed_size, compressed.pos);
200	}
201	// decompress compressed stream to decompressed stream
202	{
203	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
204	var decompressed = fixedBufferStream(&dcm_buf);
205	try inflate.decompress(container, compressed.reader(), decompressed.writer());
206	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
207	}
208	}
209	}
210
211	// store only
212	{
213	inline for (Container.list) \|container\| { // for each wrapping
214	var compressed_size: usize = if (case.store_size > 0)
215	case.store_size - Container.gzip.size() + container.size()
216	else
217	0;
218
219	// compress original stream to compressed stream
220	{
221	var original = fixedBufferStream(data);
222	var compressed = fixedBufferStream(&cmp_buf);
223	var cmp = try deflate.store.compressor(container, compressed.writer());
224	try cmp.compress(original.reader());
225	try cmp.finish();
226	if (compressed_size == 0) {
227	if (container == .gzip)
228	print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos });
229	compressed_size = compressed.pos;
230	}
231
232	try testing.expectEqual(compressed_size, compressed.pos);
233	}
234	// decompress compressed stream to decompressed stream
235	{
236	var compressed = fixedBufferStream(cmp_buf[0..compressed_size]);
237	var decompressed = fixedBufferStream(&dcm_buf);
238	try inflate.decompress(container, compressed.reader(), decompressed.writer());
239	try testing.expectEqualSlices(u8, data, decompressed.getWritten());
240	}
241	}
242	}
243	}
244	}
245
246	fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void {
247	var in = fixedBufferStream(compressed);
248	var out = std.ArrayList(u8).init(testing.allocator);
249	defer out.deinit();
250
251	try inflate.decompress(container, in.reader(), out.writer());
252	try testing.expectEqualSlices(u8, expected_plain, out.items);
253	}
254
255	test "flate don't read past deflate stream's end" {
256	try testDecompress(.zlib, &[_]u8{
257	0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
258	0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
259	0x83, 0x95, 0x0b, 0xf5,
260	}, &[_]u8{
261	0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
262	0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
263	0x00, 0x00, 0xff, 0xff, 0xff,
264	});
265	}
266
267	test "flate zlib header" {
268	// Truncated header
269	try testing.expectError(
270	error.EndOfStream,
271	testDecompress(.zlib, &[_]u8{0x78}, ""),
272	);
273	// Wrong CM
274	try testing.expectError(
275	error.BadZlibHeader,
276	testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""),
277	);
278	// Wrong CINFO
279	try testing.expectError(
280	error.BadZlibHeader,
281	testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""),
282	);
283	// Wrong checksum
284	try testing.expectError(
285	error.WrongZlibChecksum,
286	testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
287	);
288	// Truncated checksum
289	try testing.expectError(
290	error.EndOfStream,
291	testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
292	);
293	}
294
295	test "flate gzip header" {
296	// Truncated header
297	try testing.expectError(
298	error.EndOfStream,
299	testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined),
300	);
301	// Wrong CM
302	try testing.expectError(
303	error.BadGzipHeader,
304	testDecompress(.gzip, &[_]u8{
305	0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
306	0x00, 0x03,
307	}, undefined),
308	);
309
310	// Wrong checksum
311	try testing.expectError(
312	error.WrongGzipChecksum,
313	testDecompress(.gzip, &[_]u8{
314	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
315	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
316	0x00, 0x00, 0x00, 0x00,
317	}, undefined),
318	);
319	// Truncated checksum
320	try testing.expectError(
321	error.EndOfStream,
322	testDecompress(.gzip, &[_]u8{
323	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
324	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
325	}, undefined),
326	);
327	// Wrong initial size
328	try testing.expectError(
329	error.WrongGzipSize,
330	testDecompress(.gzip, &[_]u8{
331	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
332	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
333	0x00, 0x00, 0x00, 0x01,
334	}, undefined),
335	);
336	// Truncated initial size field
337	try testing.expectError(
338	error.EndOfStream,
339	testDecompress(.gzip, &[_]u8{
340	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
341	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
342	0x00, 0x00, 0x00,
343	}, undefined),
344	);
345
346	try testDecompress(.gzip, &[_]u8{
347	// GZIP header
348	0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
349	// header.FHCRC (should cover entire header)
350	0x99, 0xd6,
351	// GZIP data
352	0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
353	}, "");
354	}
355
356	test "flate public interface" {
357	if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest;
358
359	const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a };
360
361	// deflate final stored block, header + plain (stored) data
362	const deflate_block = [_]u8{
363	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
364	} ++ plain_data;
365
366	// gzip header/footer + deflate block
367	const gzip_data =
368	[_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes)
369	deflate_block ++
370	[_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes)
371
372	// zlib header/footer + deflate block
373	const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)}
374	deflate_block ++
375	[_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum
376
377	const gzip = @import("gzip.zig");
378	const zlib = @import("zlib.zig");
379	const flate = @This();
380
381	try testInterface(gzip, &gzip_data, &plain_data);
382	try testInterface(zlib, &zlib_data, &plain_data);
383	try testInterface(flate, &deflate_block, &plain_data);
384	}
385
386	fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void {
387	var buffer1: [64]u8 = undefined;
388	var buffer2: [64]u8 = undefined;
389
390	var compressed = fixedBufferStream(&buffer1);
391	var plain = fixedBufferStream(&buffer2);
392
393	// decompress
394	{
395	var in = fixedBufferStream(gzip_data);
396	try pkg.decompress(in.reader(), plain.writer());
397	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
398	}
399	plain.reset();
400	compressed.reset();
401
402	// compress/decompress
403	{
404	var in = fixedBufferStream(plain_data);
405	try pkg.compress(in.reader(), compressed.writer(), .{});
406	compressed.reset();
407	try pkg.decompress(compressed.reader(), plain.writer());
408	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
409	}
410	plain.reset();
411	compressed.reset();
412
413	// compressor/decompressor
414	{
415	var in = fixedBufferStream(plain_data);
416	var cmp = try pkg.compressor(compressed.writer(), .{});
417	try cmp.compress(in.reader());
418	try cmp.finish();
419
420	compressed.reset();
421	var dcp = pkg.decompressor(compressed.reader());
422	try dcp.decompress(plain.writer());
423	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
424	}
425	plain.reset();
426	compressed.reset();
427
428	// huffman
429	{
430	// huffman compress/decompress
431	{
432	var in = fixedBufferStream(plain_data);
433	try pkg.huffman.compress(in.reader(), compressed.writer());
434	compressed.reset();
435	try pkg.decompress(compressed.reader(), plain.writer());
436	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
437	}
438	plain.reset();
439	compressed.reset();
440
441	// huffman compressor/decompressor
442	{
443	var in = fixedBufferStream(plain_data);
444	var cmp = try pkg.huffman.compressor(compressed.writer());
445	try cmp.compress(in.reader());
446	try cmp.finish();
447
448	compressed.reset();
449	try pkg.decompress(compressed.reader(), plain.writer());
450	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
451	}
452	}
453	plain.reset();
454	compressed.reset();
455
456	// store
457	{
458	// store compress/decompress
459	{
460	var in = fixedBufferStream(plain_data);
461	try pkg.store.compress(in.reader(), compressed.writer());
462	compressed.reset();
463	try pkg.decompress(compressed.reader(), plain.writer());
464	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
465	}
466	plain.reset();
467	compressed.reset();
468
469	// store compressor/decompressor
470	{
471	var in = fixedBufferStream(plain_data);
472	var cmp = try pkg.store.compressor(compressed.writer());
473	try cmp.compress(in.reader());
474	try cmp.finish();
475
476	compressed.reset();
477	try pkg.decompress(compressed.reader(), plain.writer());
478	try testing.expectEqualSlices(u8, plain_data, plain.getWritten());
479	}
480	}
481	}

lib/std/compress/flate/CircularBuffer.zig created+234

...	@@ -0,0 +1,234 @@
1	//! 64K buffer of uncompressed data created in inflate (decompression). Has enough
2	//! history to support writing match<length, distance>; copying length of bytes
3	//! from the position distance backward from current.
4	//!
5	//! Reads can return less than available bytes if they are spread across
6	//! different circles. So reads should repeat until get required number of bytes
7	//! or until returned slice is zero length.
8	//!
9	//! Note on deflate limits:
10	//! * non-compressible block is limited to 65,535 bytes.
11	//! * backward pointer is limited in distance to 32K bytes and in length to 258 bytes.
12	//!
13	//! Whole non-compressed block can be written without overlap. We always have
14	//! history of up to 64K, more then 32K needed.
15	//!
16	const std = @import("std");
17	const assert = std.debug.assert;
18	const testing = std.testing;
19
20	const consts = @import("consts.zig").match;
21
22	const mask = 0xffff; // 64K - 1
23	const buffer_len = mask + 1; // 64K buffer
24
25	const Self = @This();
26
27	buffer: [buffer_len]u8 = undefined,
28	wp: usize = 0, // write position
29	rp: usize = 0, // read position
30
31	fn writeAll(self: *Self, buf: []const u8) void {
32	for (buf) \|c\| self.write(c);
33	}
34
35	/// Write literal.
36	pub fn write(self: *Self, b: u8) void {
37	assert(self.wp - self.rp < mask);
38	self.buffer[self.wp & mask] = b;
39	self.wp += 1;
40	}
41
42	/// Write match (back-reference to the same data slice) starting at `distance`
43	/// back from current write position, and `length` of bytes.
44	pub fn writeMatch(self: *Self, length: u16, distance: u16) !void {
45	if (self.wp < distance or
46	length < consts.base_length or length > consts.max_length or
47	distance < consts.min_distance or distance > consts.max_distance)
48	{
49	return error.InvalidMatch;
50	}
51	assert(self.wp - self.rp < mask);
52
53	var from: usize = self.wp - distance;
54	const from_end: usize = from + length;
55	var to: usize = self.wp;
56	const to_end: usize = to + length;
57
58	self.wp += length;
59
60	// Fast path using memcpy
61	if (length <= distance and // no overlapping buffers
62	(from >> 16 == from_end >> 16) and // start and and at the same circle
63	(to >> 16 == to_end >> 16))
64	{
65	@memcpy(self.buffer[to & mask .. to_end & mask], self.buffer[from & mask .. from_end & mask]);
66	return;
67	}
68
69	// Slow byte by byte
70	while (to < to_end) {
71	self.buffer[to & mask] = self.buffer[from & mask];
72	to += 1;
73	from += 1;
74	}
75	}
76
77	/// Returns writable part of the internal buffer of size `n` at most. Advances
78	/// write pointer, assumes that returned buffer will be filled with data.
79	pub fn getWritable(self: *Self, n: usize) []u8 {
80	const wp = self.wp & mask;
81	const len = @min(n, buffer_len - wp);
82	self.wp += len;
83	return self.buffer[wp .. wp + len];
84	}
85
86	/// Read available data. Can return part of the available data if it is
87	/// spread across two circles. So read until this returns zero length.
88	pub fn read(self: *Self) []const u8 {
89	return self.readAtMost(buffer_len);
90	}
91
92	/// Read part of available data. Can return less than max even if there are
93	/// more than max decoded data.
94	pub fn readAtMost(self: *Self, limit: usize) []const u8 {
95	const rb = self.readBlock(if (limit == 0) buffer_len else limit);
96	defer self.rp += rb.len;
97	return self.buffer[rb.head..rb.tail];
98	}
99
100	const ReadBlock = struct {
101	head: usize,
102	tail: usize,
103	len: usize,
104	};
105
106	/// Returns position of continous read block data.
107	fn readBlock(self: *Self, max: usize) ReadBlock {
108	const r = self.rp & mask;
109	const w = self.wp & mask;
110	const n = @min(
111	max,
112	if (w >= r) w - r else buffer_len - r,
113	);
114	return .{
115	.head = r,
116	.tail = r + n,
117	.len = n,
118	};
119	}
120
121	/// Number of free bytes for write.
122	pub fn free(self: *Self) usize {
123	return buffer_len - (self.wp - self.rp);
124	}
125
126	/// Full if largest match can't fit. 258 is largest match length. That much
127	/// bytes can be produced in single decode step.
128	pub fn full(self: *Self) bool {
129	return self.free() < 258 + 1;
130	}
131
132	// example from: https://youtu.be/SJPvNi4HrWQ?t=3558
133	test "flate.CircularBuffer writeMatch" {
134	var cb: Self = .{};
135
136	cb.writeAll("a salad; ");
137	try cb.writeMatch(5, 9);
138	try cb.writeMatch(3, 3);
139
140	try testing.expectEqualStrings("a salad; a salsal", cb.read());
141	}
142
143	test "flate.CircularBuffer writeMatch overlap" {
144	var cb: Self = .{};
145
146	cb.writeAll("a b c ");
147	try cb.writeMatch(8, 4);
148	cb.write('d');
149
150	try testing.expectEqualStrings("a b c b c b c d", cb.read());
151	}
152
153	test "flate.CircularBuffer readAtMost" {
154	var cb: Self = .{};
155
156	cb.writeAll("0123456789");
157	try cb.writeMatch(50, 10);
158
159	try testing.expectEqualStrings("0123456789" ** 6, cb.buffer[cb.rp..cb.wp]);
160	for (0..6) \|i\| {
161	try testing.expectEqual(i * 10, cb.rp);
162	try testing.expectEqualStrings("0123456789", cb.readAtMost(10));
163	}
164	try testing.expectEqualStrings("", cb.readAtMost(10));
165	try testing.expectEqualStrings("", cb.read());
166	}
167
168	test "flate.CircularBuffer" {
169	var cb: Self = .{};
170
171	const data = "0123456789abcdef" ** (1024 / 16);
172	cb.writeAll(data);
173	try testing.expectEqual(@as(usize, 0), cb.rp);
174	try testing.expectEqual(@as(usize, 1024), cb.wp);
175	try testing.expectEqual(@as(usize, 1024 * 63), cb.free());
176
177	for (0..62 * 4) \|_\|
178	try cb.writeMatch(256, 1024); // write 62K
179
180	try testing.expectEqual(@as(usize, 0), cb.rp);
181	try testing.expectEqual(@as(usize, 63 * 1024), cb.wp);
182	try testing.expectEqual(@as(usize, 1024), cb.free());
183
184	cb.writeAll(data[0..200]);
185	_ = cb.readAtMost(1024); // make some space
186	cb.writeAll(data); // overflows write position
187	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
188	try testing.expectEqual(@as(usize, 1024), cb.rp);
189	try testing.expectEqual(@as(usize, 1024 - 200), cb.free());
190
191	const rb = cb.readBlock(Self.buffer_len);
192	try testing.expectEqual(@as(usize, 65536 - 1024), rb.len);
193	try testing.expectEqual(@as(usize, 1024), rb.head);
194	try testing.expectEqual(@as(usize, 65536), rb.tail);
195
196	try testing.expectEqual(@as(usize, 65536 - 1024), cb.read().len); // read to the end of the buffer
197	try testing.expectEqual(@as(usize, 200 + 65536), cb.wp);
198	try testing.expectEqual(@as(usize, 65536), cb.rp);
199	try testing.expectEqual(@as(usize, 65536 - 200), cb.free());
200
201	try testing.expectEqual(@as(usize, 200), cb.read().len); // read the rest
202	}
203
204	test "flate.CircularBuffer write overlap" {
205	var cb: Self = .{};
206	cb.wp = cb.buffer.len - 15;
207	cb.rp = cb.wp;
208
209	cb.writeAll("0123456789");
210	cb.writeAll("abcdefghij");
211
212	try testing.expectEqual(cb.buffer.len + 5, cb.wp);
213	try testing.expectEqual(cb.buffer.len - 15, cb.rp);
214
215	try testing.expectEqualStrings("0123456789abcde", cb.read());
216	try testing.expectEqualStrings("fghij", cb.read());
217
218	try testing.expect(cb.wp == cb.rp);
219	}
220
221	test "flate.CircularBuffer writeMatch/read overlap" {
222	var cb: Self = .{};
223	cb.wp = cb.buffer.len - 15;
224	cb.rp = cb.wp;
225
226	cb.writeAll("0123456789");
227	try cb.writeMatch(15, 5);
228
229	try testing.expectEqualStrings("012345678956789", cb.read());
230	try testing.expectEqualStrings("5678956789", cb.read());
231
232	try cb.writeMatch(20, 25);
233	try testing.expectEqualStrings("01234567895678956789", cb.read());
234	}

lib/std/compress/flate/Lookup.zig created+125

...	@@ -0,0 +1,125 @@
1	/// Lookup of the previous locations for the same 4 byte data. Works on hash of
2	/// 4 bytes data. Head contains position of the first match for each hash. Chain
3	/// points to the previous position of the same hash given the current location.
4	///
5	const std = @import("std");
6	const testing = std.testing;
7	const expect = testing.expect;
8	const consts = @import("consts.zig");
9
10	const Self = @This();
11
12	const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
13	const chain_len = 2 * consts.history.len;
14
15	// Maps hash => first position
16	head: [consts.lookup.len]u16 = [_]u16{0} ** consts.lookup.len,
17	// Maps position => previous positions for the same hash value
18	chain: [chain_len]u16 = [_]u16{0} ** (chain_len),
19
20	// Calculates hash of the 4 bytes from data.
21	// Inserts `pos` position of that hash in the lookup tables.
22	// Returns previous location with the same hash value.
23	pub fn add(self: *Self, data: []const u8, pos: u16) u16 {
24	if (data.len < 4) return 0;
25	const h = hash(data[0..4]);
26	return self.set(h, pos);
27	}
28
29	// Retruns previous location with the same hash value given the current
30	// position.
31	pub fn prev(self: *Self, pos: u16) u16 {
32	return self.chain[pos];
33	}
34
35	fn set(self: *Self, h: u32, pos: u16) u16 {
36	const p = self.head[h];
37	self.head[h] = pos;
38	self.chain[pos] = p;
39	return p;
40	}
41
42	// Slide all positions in head and chain for `n`
43	pub fn slide(self: *Self, n: u16) void {
44	for (&self.head) \|*v\| {
45	v.* -\|= n;
46	}
47	var i: usize = 0;
48	while (i < n) : (i += 1) {
49	self.chain[i] = self.chain[i + n] -\| n;
50	}
51	}
52
53	// Add `len` 4 bytes hashes from `data` into lookup.
54	// Position of the first byte is `pos`.
55	pub fn bulkAdd(self: *Self, data: []const u8, len: u16, pos: u16) void {
56	if (len == 0 or data.len < consts.match.min_length) {
57	return;
58	}
59	var hb =
60	@as(u32, data[3]) \|
61	@as(u32, data[2]) << 8 \|
62	@as(u32, data[1]) << 16 \|
63	@as(u32, data[0]) << 24;
64	_ = self.set(hashu(hb), pos);
65
66	var i = pos;
67	for (4..@min(len + 3, data.len)) \|j\| {
68	hb = (hb << 8) \| @as(u32, data[j]);
69	i += 1;
70	_ = self.set(hashu(hb), i);
71	}
72	}
73
74	// Calculates hash of the first 4 bytes of `b`.
75	fn hash(b: *const [4]u8) u32 {
76	return hashu(@as(u32, b[3]) \|
77	@as(u32, b[2]) << 8 \|
78	@as(u32, b[1]) << 16 \|
79	@as(u32, b[0]) << 24);
80	}
81
82	fn hashu(v: u32) u32 {
83	return @intCast((v *% prime4) >> consts.lookup.shift);
84	}
85
86	test "flate.Lookup add/prev" {
87	const data = [_]u8{
88	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
89	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
90	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
91	0x01, 0x02, 0x03,
92	};
93
94	var h: Self = .{};
95	for (data, 0..) \|_, i\| {
96	const p = h.add(data[i..], @intCast(i));
97	if (i >= 8 and i < 24) {
98	try expect(p == i - 8);
99	} else {
100	try expect(p == 0);
101	}
102	}
103
104	const v = Self.hash(data[2 .. 2 + 4]);
105	try expect(h.head[v] == 2 + 16);
106	try expect(h.chain[2 + 16] == 2 + 8);
107	try expect(h.chain[2 + 8] == 2);
108	}
109
110	test "flate.Lookup bulkAdd" {
111	const data = "Lorem ipsum dolor sit amet, consectetur adipiscing elit.";
112
113	// one by one
114	var h: Self = .{};
115	for (data, 0..) \|_, i\| {
116	_ = h.add(data[i..], @intCast(i));
117	}
118
119	// in bulk
120	var bh: Self = .{};
121	bh.bulkAdd(data, data.len, 0);
122
123	try testing.expectEqualSlices(u16, &h.head, &bh.head);
124	try testing.expectEqualSlices(u16, &h.chain, &bh.chain);
125	}

lib/std/compress/flate/SlidingWindow.zig created+160

...	@@ -0,0 +1,160 @@
1	//! Used in deflate (compression), holds uncompressed data form which Tokens are
2	//! produces. In combination with Lookup it is used to find matches in history data.
3	//!
4	const std = @import("std");
5	const consts = @import("consts.zig");
6
7	const expect = testing.expect;
8	const assert = std.debug.assert;
9	const testing = std.testing;
10
11	const hist_len = consts.history.len;
12	const buffer_len = 2 * hist_len;
13	const min_lookahead = consts.match.min_length + consts.match.max_length;
14	const max_rp = buffer_len - min_lookahead;
15
16	const Self = @This();
17
18	buffer: [buffer_len]u8 = undefined,
19	wp: usize = 0, // write position
20	rp: usize = 0, // read position
21	fp: isize = 0, // last flush position, tokens are build from fp..rp
22
23	/// Returns number of bytes written, or 0 if buffer is full and need to slide.
24	pub fn write(self: *Self, buf: []const u8) usize {
25	if (self.rp >= max_rp) return 0; // need to slide
26
27	const n = @min(buf.len, buffer_len - self.wp);
28	@memcpy(self.buffer[self.wp .. self.wp + n], buf[0..n]);
29	self.wp += n;
30	return n;
31	}
32
33	/// Slide buffer for hist_len.
34	/// Drops old history, preserves between hist_len and hist_len - min_lookahead.
35	/// Returns number of bytes removed.
36	pub fn slide(self: *Self) u16 {
37	assert(self.rp >= max_rp and self.wp >= self.rp);
38	const n = self.wp - hist_len;
39	@memcpy(self.buffer[0..n], self.buffer[hist_len..self.wp]);
40	self.rp -= hist_len;
41	self.wp -= hist_len;
42	self.fp -= hist_len;
43	return @intCast(n);
44	}
45
46	/// Data from the current position (read position). Those part of the buffer is
47	/// not converted to tokens yet.
48	fn lookahead(self: *Self) []const u8 {
49	assert(self.wp >= self.rp);
50	return self.buffer[self.rp..self.wp];
51	}
52
53	/// Returns part of the lookahead buffer. If should_flush is set no lookahead is
54	/// preserved otherwise preserves enough data for the longest match. Returns
55	/// null if there is not enough data.
56	pub fn activeLookahead(self: *Self, should_flush: bool) ?[]const u8 {
57	const min: usize = if (should_flush) 0 else min_lookahead;
58	const lh = self.lookahead();
59	return if (lh.len > min) lh else null;
60	}
61
62	/// Advances read position, shrinks lookahead.
63	pub fn advance(self: *Self, n: u16) void {
64	assert(self.wp >= self.rp + n);
65	self.rp += n;
66	}
67
68	/// Returns writable part of the buffer, where new uncompressed data can be
69	/// written.
70	pub fn writable(self: *Self) []u8 {
71	return self.buffer[self.wp..];
72	}
73
74	/// Notification of what part of writable buffer is filled with data.
75	pub fn written(self: *Self, n: usize) void {
76	self.wp += n;
77	}
78
79	/// Finds match length between previous and current position.
80	/// Used in hot path!
81	pub fn match(self: *Self, prev_pos: u16, curr_pos: u16, min_len: u16) u16 {
82	const max_len: usize = @min(self.wp - curr_pos, consts.match.max_length);
83	// lookahead buffers from previous and current positions
84	const prev_lh = self.buffer[prev_pos..][0..max_len];
85	const curr_lh = self.buffer[curr_pos..][0..max_len];
86
87	// If we alread have match (min_len > 0),
88	// test the first byte above previous len a[min_len] != b[min_len]
89	// and then all the bytes from that position to zero.
90	// That is likely positions to find difference than looping from first bytes.
91	var i: usize = min_len;
92	if (i > 0) {
93	if (max_len <= i) return 0;
94	while (true) {
95	if (prev_lh[i] != curr_lh[i]) return 0;
96	if (i == 0) break;
97	i -= 1;
98	}
99	i = min_len;
100	}
101	while (i < max_len) : (i += 1)
102	if (prev_lh[i] != curr_lh[i]) break;
103	return if (i >= consts.match.min_length) @intCast(i) else 0;
104	}
105
106	/// Current position of non-compressed data. Data before rp are already converted
107	/// to tokens.
108	pub fn pos(self: *Self) u16 {
109	return @intCast(self.rp);
110	}
111
112	/// Notification that token list is cleared.
113	pub fn flush(self: *Self) void {
114	self.fp = @intCast(self.rp);
115	}
116
117	/// Part of the buffer since last flush or null if there was slide in between (so
118	/// fp becomes negative).
119	pub fn tokensBuffer(self: *Self) ?[]const u8 {
120	assert(self.fp <= self.rp);
121	if (self.fp < 0) return null;
122	return self.buffer[@intCast(self.fp)..self.rp];
123	}
124
125	test "flate.SlidingWindow match" {
126	const data = "Blah blah blah blah blah!";
127	var win: Self = .{};
128	try expect(win.write(data) == data.len);
129	try expect(win.wp == data.len);
130	try expect(win.rp == 0);
131
132	// length between l symbols
133	try expect(win.match(1, 6, 0) == 18);
134	try expect(win.match(1, 11, 0) == 13);
135	try expect(win.match(1, 16, 0) == 8);
136	try expect(win.match(1, 21, 0) == 0);
137
138	// position 15 = "blah blah!"
139	// position 20 = "blah!"
140	try expect(win.match(15, 20, 0) == 4);
141	try expect(win.match(15, 20, 3) == 4);
142	try expect(win.match(15, 20, 4) == 0);
143	}
144
145	test "flate.SlidingWindow slide" {
146	var win: Self = .{};
147	win.wp = Self.buffer_len - 11;
148	win.rp = Self.buffer_len - 111;
149	win.buffer[win.rp] = 0xab;
150	try expect(win.lookahead().len == 100);
151	try expect(win.tokensBuffer().?.len == win.rp);
152
153	const n = win.slide();
154	try expect(n == 32757);
155	try expect(win.buffer[win.rp] == 0xab);
156	try expect(win.rp == Self.hist_len - 111);
157	try expect(win.wp == Self.hist_len - 11);
158	try expect(win.lookahead().len == 100);
159	try expect(win.tokensBuffer() == null);
160	}

lib/std/compress/flate/Token.zig created+327

...	@@ -0,0 +1,327 @@
1	//! Token cat be literal: single byte of data or match; reference to the slice of
2	//! data in the same stream represented with <length, distance>. Where length
3	//! can be 3 - 258 bytes, and distance 1 - 32768 bytes.
4	//!
5	const std = @import("std");
6	const assert = std.debug.assert;
7	const print = std.debug.print;
8	const expect = std.testing.expect;
9	const consts = @import("consts.zig").match;
10
11	const Token = @This();
12
13	pub const Kind = enum(u1) {
14	literal,
15	match,
16	};
17
18	// Distance range 1 - 32768, stored in dist as 0 - 32767 (fits u15)
19	dist: u15 = 0,
20	// Length range 3 - 258, stored in len_lit as 0 - 255 (fits u8)
21	len_lit: u8 = 0,
22	kind: Kind = .literal,
23
24	pub fn literal(t: Token) u8 {
25	return t.len_lit;
26	}
27
28	pub fn distance(t: Token) u16 {
29	return @as(u16, t.dist) + consts.min_distance;
30	}
31
32	pub fn length(t: Token) u16 {
33	return @as(u16, t.len_lit) + consts.base_length;
34	}
35
36	pub fn initLiteral(lit: u8) Token {
37	return .{ .kind = .literal, .len_lit = lit };
38	}
39
40	// distance range 1 - 32768, stored in dist as 0 - 32767 (u15)
41	// length range 3 - 258, stored in len_lit as 0 - 255 (u8)
42	pub fn initMatch(dist: u16, len: u16) Token {
43	assert(len >= consts.min_length and len <= consts.max_length);
44	assert(dist >= consts.min_distance and dist <= consts.max_distance);
45	return .{
46	.kind = .match,
47	.dist = @intCast(dist - consts.min_distance),
48	.len_lit = @intCast(len - consts.base_length),
49	};
50	}
51
52	pub fn eql(t: Token, o: Token) bool {
53	return t.kind == o.kind and
54	t.dist == o.dist and
55	t.len_lit == o.len_lit;
56	}
57
58	pub fn lengthCode(t: Token) u16 {
59	return match_lengths[match_lengths_index[t.len_lit]].code;
60	}
61
62	pub fn lengthEncoding(t: Token) MatchLength {
63	var c = match_lengths[match_lengths_index[t.len_lit]];
64	c.extra_length = t.len_lit - c.base_scaled;
65	return c;
66	}
67
68	// Returns the distance code corresponding to a specific distance.
69	// Distance code is in range: 0 - 29.
70	pub fn distanceCode(t: Token) u8 {
71	var dist: u16 = t.dist;
72	if (dist < match_distances_index.len) {
73	return match_distances_index[dist];
74	}
75	dist >>= 7;
76	if (dist < match_distances_index.len) {
77	return match_distances_index[dist] + 14;
78	}
79	dist >>= 7;
80	return match_distances_index[dist] + 28;
81	}
82
83	pub fn distanceEncoding(t: Token) MatchDistance {
84	var c = match_distances[t.distanceCode()];
85	c.extra_distance = t.dist - c.base_scaled;
86	return c;
87	}
88
89	pub fn lengthExtraBits(code: u32) u8 {
90	return match_lengths[code - length_codes_start].extra_bits;
91	}
92
93	pub fn matchLength(code: u8) MatchLength {
94	return match_lengths[code];
95	}
96
97	pub fn matchDistance(code: u8) MatchDistance {
98	return match_distances[code];
99	}
100
101	pub fn distanceExtraBits(code: u32) u8 {
102	return match_distances[code].extra_bits;
103	}
104
105	pub fn show(t: Token) void {
106	if (t.kind == .literal) {
107	print("L('{c}'), ", .{t.literal()});
108	} else {
109	print("M({d}, {d}), ", .{ t.distance(), t.length() });
110	}
111	}
112
113	// Retruns index in match_lengths table for each length in range 0-255.
114	const match_lengths_index = [_]u8{
115	0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
116	9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
117	13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
118	15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
119	17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
120	18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
121	19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
122	20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
123	21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
124	21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
125	22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
126	22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
127	23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
128	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
129	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
130	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
131	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
132	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
133	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
134	25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
135	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
136	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
137	26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
138	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
139	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
140	27, 27, 27, 27, 27, 28,
141	};
142
143	const MatchLength = struct {
144	code: u16,
145	base_scaled: u8, // base - 3, scaled to fit into u8 (0-255), same as lit_len field in Token.
146	base: u16, // 3-258
147	extra_length: u8 = 0,
148	extra_bits: u4,
149	};
150
151	// match_lengths represents table from rfc (https://datatracker.ietf.org/doc/html/rfc1951#page-12)
152	//
153	// Extra Extra Extra
154	// Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
155	// ---- ---- ------ ---- ---- ------- ---- ---- -------
156	// 257 0 3 267 1 15,16 277 4 67-82
157	// 258 0 4 268 1 17,18 278 4 83-98
158	// 259 0 5 269 2 19-22 279 4 99-114
159	// 260 0 6 270 2 23-26 280 4 115-130
160	// 261 0 7 271 2 27-30 281 5 131-162
161	// 262 0 8 272 2 31-34 282 5 163-194
162	// 263 0 9 273 3 35-42 283 5 195-226
163	// 264 0 10 274 3 43-50 284 5 227-257
164	// 265 1 11,12 275 3 51-58 285 0 258
165	// 266 1 13,14 276 3 59-66
166	//
167	pub const length_codes_start = 257;
168
169	const match_lengths = [_]MatchLength{
170	.{ .extra_bits = 0, .base_scaled = 0, .base = 3, .code = 257 },
171	.{ .extra_bits = 0, .base_scaled = 1, .base = 4, .code = 258 },
172	.{ .extra_bits = 0, .base_scaled = 2, .base = 5, .code = 259 },
173	.{ .extra_bits = 0, .base_scaled = 3, .base = 6, .code = 260 },
174	.{ .extra_bits = 0, .base_scaled = 4, .base = 7, .code = 261 },
175	.{ .extra_bits = 0, .base_scaled = 5, .base = 8, .code = 262 },
176	.{ .extra_bits = 0, .base_scaled = 6, .base = 9, .code = 263 },
177	.{ .extra_bits = 0, .base_scaled = 7, .base = 10, .code = 264 },
178	.{ .extra_bits = 1, .base_scaled = 8, .base = 11, .code = 265 },
179	.{ .extra_bits = 1, .base_scaled = 10, .base = 13, .code = 266 },
180	.{ .extra_bits = 1, .base_scaled = 12, .base = 15, .code = 267 },
181	.{ .extra_bits = 1, .base_scaled = 14, .base = 17, .code = 268 },
182	.{ .extra_bits = 2, .base_scaled = 16, .base = 19, .code = 269 },
183	.{ .extra_bits = 2, .base_scaled = 20, .base = 23, .code = 270 },
184	.{ .extra_bits = 2, .base_scaled = 24, .base = 27, .code = 271 },
185	.{ .extra_bits = 2, .base_scaled = 28, .base = 31, .code = 272 },
186	.{ .extra_bits = 3, .base_scaled = 32, .base = 35, .code = 273 },
187	.{ .extra_bits = 3, .base_scaled = 40, .base = 43, .code = 274 },
188	.{ .extra_bits = 3, .base_scaled = 48, .base = 51, .code = 275 },
189	.{ .extra_bits = 3, .base_scaled = 56, .base = 59, .code = 276 },
190	.{ .extra_bits = 4, .base_scaled = 64, .base = 67, .code = 277 },
191	.{ .extra_bits = 4, .base_scaled = 80, .base = 83, .code = 278 },
192	.{ .extra_bits = 4, .base_scaled = 96, .base = 99, .code = 279 },
193	.{ .extra_bits = 4, .base_scaled = 112, .base = 115, .code = 280 },
194	.{ .extra_bits = 5, .base_scaled = 128, .base = 131, .code = 281 },
195	.{ .extra_bits = 5, .base_scaled = 160, .base = 163, .code = 282 },
196	.{ .extra_bits = 5, .base_scaled = 192, .base = 195, .code = 283 },
197	.{ .extra_bits = 5, .base_scaled = 224, .base = 227, .code = 284 },
198	.{ .extra_bits = 0, .base_scaled = 255, .base = 258, .code = 285 },
199	};
200
201	// Used in distanceCode fn to get index in match_distance table for each distance in range 0-32767.
202	const match_distances_index = [_]u8{
203	0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
204	8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
205	10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
206	11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
207	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
208	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
209	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
210	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
211	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
212	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
213	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
214	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
215	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
216	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
217	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
218	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
219	};
220
221	const MatchDistance = struct {
222	base_scaled: u16, // base - 1, same as Token dist field
223	base: u16,
224	extra_distance: u16 = 0,
225	code: u8,
226	extra_bits: u4,
227	};
228
229	// match_distances represents table from rfc (https://datatracker.ietf.org/doc/html/rfc1951#page-12)
230	//
231	// Extra Extra Extra
232	// Code Bits Dist Code Bits Dist Code Bits Distance
233	// ---- ---- ---- ---- ---- ------ ---- ---- --------
234	// 0 0 1 10 4 33-48 20 9 1025-1536
235	// 1 0 2 11 4 49-64 21 9 1537-2048
236	// 2 0 3 12 5 65-96 22 10 2049-3072
237	// 3 0 4 13 5 97-128 23 10 3073-4096
238	// 4 1 5,6 14 6 129-192 24 11 4097-6144
239	// 5 1 7,8 15 6 193-256 25 11 6145-8192
240	// 6 2 9-12 16 7 257-384 26 12 8193-12288
241	// 7 2 13-16 17 7 385-512 27 12 12289-16384
242	// 8 3 17-24 18 8 513-768 28 13 16385-24576
243	// 9 3 25-32 19 8 769-1024 29 13 24577-32768
244	//
245	const match_distances = [_]MatchDistance{
246	.{ .extra_bits = 0, .base_scaled = 0x0000, .code = 0, .base = 1 },
247	.{ .extra_bits = 0, .base_scaled = 0x0001, .code = 1, .base = 2 },
248	.{ .extra_bits = 0, .base_scaled = 0x0002, .code = 2, .base = 3 },
249	.{ .extra_bits = 0, .base_scaled = 0x0003, .code = 3, .base = 4 },
250	.{ .extra_bits = 1, .base_scaled = 0x0004, .code = 4, .base = 5 },
251	.{ .extra_bits = 1, .base_scaled = 0x0006, .code = 5, .base = 7 },
252	.{ .extra_bits = 2, .base_scaled = 0x0008, .code = 6, .base = 9 },
253	.{ .extra_bits = 2, .base_scaled = 0x000c, .code = 7, .base = 13 },
254	.{ .extra_bits = 3, .base_scaled = 0x0010, .code = 8, .base = 17 },
255	.{ .extra_bits = 3, .base_scaled = 0x0018, .code = 9, .base = 25 },
256	.{ .extra_bits = 4, .base_scaled = 0x0020, .code = 10, .base = 33 },
257	.{ .extra_bits = 4, .base_scaled = 0x0030, .code = 11, .base = 49 },
258	.{ .extra_bits = 5, .base_scaled = 0x0040, .code = 12, .base = 65 },
259	.{ .extra_bits = 5, .base_scaled = 0x0060, .code = 13, .base = 97 },
260	.{ .extra_bits = 6, .base_scaled = 0x0080, .code = 14, .base = 129 },
261	.{ .extra_bits = 6, .base_scaled = 0x00c0, .code = 15, .base = 193 },
262	.{ .extra_bits = 7, .base_scaled = 0x0100, .code = 16, .base = 257 },
263	.{ .extra_bits = 7, .base_scaled = 0x0180, .code = 17, .base = 385 },
264	.{ .extra_bits = 8, .base_scaled = 0x0200, .code = 18, .base = 513 },
265	.{ .extra_bits = 8, .base_scaled = 0x0300, .code = 19, .base = 769 },
266	.{ .extra_bits = 9, .base_scaled = 0x0400, .code = 20, .base = 1025 },
267	.{ .extra_bits = 9, .base_scaled = 0x0600, .code = 21, .base = 1537 },
268	.{ .extra_bits = 10, .base_scaled = 0x0800, .code = 22, .base = 2049 },
269	.{ .extra_bits = 10, .base_scaled = 0x0c00, .code = 23, .base = 3073 },
270	.{ .extra_bits = 11, .base_scaled = 0x1000, .code = 24, .base = 4097 },
271	.{ .extra_bits = 11, .base_scaled = 0x1800, .code = 25, .base = 6145 },
272	.{ .extra_bits = 12, .base_scaled = 0x2000, .code = 26, .base = 8193 },
273	.{ .extra_bits = 12, .base_scaled = 0x3000, .code = 27, .base = 12289 },
274	.{ .extra_bits = 13, .base_scaled = 0x4000, .code = 28, .base = 16385 },
275	.{ .extra_bits = 13, .base_scaled = 0x6000, .code = 29, .base = 24577 },
276	};
277
278	test "flate.Token size" {
279	try expect(@sizeOf(Token) == 4);
280	}
281
282	// testing table https://datatracker.ietf.org/doc/html/rfc1951#page-12
283	test "flate.Token MatchLength" {
284	var c = Token.initMatch(1, 4).lengthEncoding();
285	try expect(c.code == 258);
286	try expect(c.extra_bits == 0);
287	try expect(c.extra_length == 0);
288
289	c = Token.initMatch(1, 11).lengthEncoding();
290	try expect(c.code == 265);
291	try expect(c.extra_bits == 1);
292	try expect(c.extra_length == 0);
293
294	c = Token.initMatch(1, 12).lengthEncoding();
295	try expect(c.code == 265);
296	try expect(c.extra_bits == 1);
297	try expect(c.extra_length == 1);
298
299	c = Token.initMatch(1, 130).lengthEncoding();
300	try expect(c.code == 280);
301	try expect(c.extra_bits == 4);
302	try expect(c.extra_length == 130 - 115);
303	}
304
305	test "flate.Token MatchDistance" {
306	var c = Token.initMatch(1, 4).distanceEncoding();
307	try expect(c.code == 0);
308	try expect(c.extra_bits == 0);
309	try expect(c.extra_distance == 0);
310
311	c = Token.initMatch(192, 4).distanceEncoding();
312	try expect(c.code == 14);
313	try expect(c.extra_bits == 6);
314	try expect(c.extra_distance == 192 - 129);
315	}
316
317	test "flate.Token match_lengths" {
318	for (match_lengths, 0..) \|ml, i\| {
319	try expect(@as(u16, ml.base_scaled) + 3 == ml.base);
320	try expect(i + 257 == ml.code);
321	}
322
323	for (match_distances, 0..) \|mo, i\| {
324	try expect(mo.base_scaled + 1 == mo.base);
325	try expect(i == mo.code);
326	}
327	}

lib/std/compress/flate/bit_reader.zig created+333

...	@@ -0,0 +1,333 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	pub fn bitReader(reader: anytype) BitReader(@TypeOf(reader)) {
6	return BitReader(@TypeOf(reader)).init(reader);
7	}
8
9	/// Bit reader used during inflate (decompression). Has internal buffer of 64
10	/// bits which shifts right after bits are consumed. Uses forward_reader to fill
11	/// that internal buffer when needed.
12	///
13	/// readF is the core function. Supports few different ways of getting bits
14	/// controlled by flags. In hot path we try to avoid checking whether we need to
15	/// fill buffer from forward_reader by calling fill in advance and readF with
16	/// buffered flag set.
17	///
18	pub fn BitReader(comptime ReaderType: type) type {
19	return struct {
20	// Underlying reader used for filling internal bits buffer
21	forward_reader: ReaderType = undefined,
22	// Internal buffer of 64 bits
23	bits: u64 = 0,
24	// Number of bits in the buffer
25	nbits: u32 = 0,
26
27	const Self = @This();
28
29	pub const Error = ReaderType.Error \|\| error{EndOfStream};
30
31	pub fn init(rdr: ReaderType) Self {
32	var self = Self{ .forward_reader = rdr };
33	self.fill(1) catch {};
34	return self;
35	}
36
37	/// Try to have `nice` bits are available in buffer. Reads from
38	/// forward reader if there is no `nice` bits in buffer. Returns error
39	/// if end of forward stream is reached and internal buffer is empty.
40	/// It will not error if less than `nice` bits are in buffer, only when
41	/// all bits are exhausted. During inflate we usually know what is the
42	/// maximum bits for the next step but usually that step will need less
43	/// bits to decode. So `nice` is not hard limit, it will just try to have
44	/// that number of bits available. If end of forward stream is reached
45	/// it may be some extra zero bits in buffer.
46	pub inline fn fill(self: *Self, nice: u6) !void {
47	if (self.nbits >= nice) {
48	return; // We have enought bits
49	}
50	// Read more bits from forward reader
51
52	// Number of empty bytes in bits, round nbits to whole bytes.
53	const empty_bytes =
54	@as(u8, if (self.nbits & 0x7 == 0) 8 else 7) - // 8 for 8, 16, 24..., 7 otherwise
55	(self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
56
57	var buf: [8]u8 = [_]u8{0} ** 8;
58	const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
59	if (bytes_read > 0) {
60	const u: u64 = std.mem.readInt(u64, buf[0..8], .little);
61	self.bits \|= u << @as(u6, @intCast(self.nbits));
62	self.nbits += 8 * @as(u8, @intCast(bytes_read));
63	return;
64	}
65
66	if (self.nbits == 0)
67	return error.EndOfStream;
68	}
69
70	/// Read exactly buf.len bytes into buf.
71	pub fn readAll(self: *Self, buf: []u8) !void {
72	assert(self.alignBits() == 0); // internal bits must be at byte boundary
73
74	// First read from internal bits buffer.
75	var n: usize = 0;
76	while (self.nbits > 0 and n < buf.len) {
77	buf[n] = try self.readF(u8, flag.buffered);
78	n += 1;
79	}
80	// Then use forward reader for all other bytes.
81	try self.forward_reader.readNoEof(buf[n..]);
82	}
83
84	pub const flag = struct {
85	pub const peek: u3 = 0b001; // dont advance internal buffer, just get bits, leave them in buffer
86	pub const buffered: u3 = 0b010; // assume that there is no need to fill, fill should be called before
87	pub const reverse: u3 = 0b100; // bit reverse readed bits
88	};
89
90	/// Alias for readF(U, 0).
91	pub fn read(self: *Self, comptime U: type) !U {
92	return self.readF(U, 0);
93	}
94
95	/// Alias for readF with flag.peak set.
96	pub inline fn peekF(self: *Self, comptime U: type, comptime how: u3) !U {
97	return self.readF(U, how \| flag.peek);
98	}
99
100	/// Read with flags provided.
101	pub fn readF(self: *Self, comptime U: type, comptime how: u3) !U {
102	const n: u6 = @bitSizeOf(U);
103	switch (how) {
104	0 => { // `normal` read
105	try self.fill(n); // ensure that there are n bits in the buffer
106	const u: U = @truncate(self.bits); // get n bits
107	try self.shift(n); // advance buffer for n
108	return u;
109	},
110	(flag.peek) => { // no shift, leave bits in the buffer
111	try self.fill(n);
112	return @truncate(self.bits);
113	},
114	flag.buffered => { // no fill, assume that buffer has enought bits
115	const u: U = @truncate(self.bits);
116	try self.shift(n);
117	return u;
118	},
119	(flag.reverse) => { // same as 0 with bit reverse
120	try self.fill(n);
121	const u: U = @truncate(self.bits);
122	try self.shift(n);
123	return @bitReverse(u);
124	},
125	(flag.peek \| flag.reverse) => {
126	try self.fill(n);
127	return @bitReverse(@as(U, @truncate(self.bits)));
128	},
129	(flag.buffered \| flag.reverse) => {
130	const u: U = @truncate(self.bits);
131	try self.shift(n);
132	return @bitReverse(u);
133	},
134	(flag.peek \| flag.buffered) => {
135	return @truncate(self.bits);
136	},
137	(flag.peek \| flag.buffered \| flag.reverse) => {
138	return @bitReverse(@as(U, @truncate(self.bits)));
139	},
140	}
141	}
142
143	/// Read n number of bits.
144	/// Only buffered flag can be used in how.
145	pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 {
146	switch (how) {
147	0 => {
148	try self.fill(n);
149	},
150	flag.buffered => {},
151	else => unreachable,
152	}
153	const mask: u16 = (@as(u16, 1) << n) - 1;
154	const u: u16 = @as(u16, @truncate(self.bits)) & mask;
155	try self.shift(n);
156	return u;
157	}
158
159	/// Advance buffer for n bits.
160	pub fn shift(self: *Self, n: u6) !void {
161	if (n > self.nbits) return error.EndOfStream;
162	self.bits >>= n;
163	self.nbits -= n;
164	}
165
166	/// Skip n bytes.
167	pub fn skipBytes(self: *Self, n: u16) !void {
168	for (0..n) \|_\| {
169	try self.fill(8);
170	try self.shift(8);
171	}
172	}
173
174	// Number of bits to align stream to the byte boundary.
175	fn alignBits(self: *Self) u3 {
176	return @intCast(self.nbits & 0x7);
177	}
178
179	/// Align stream to the byte boundary.
180	pub fn alignToByte(self: *Self) void {
181	const ab = self.alignBits();
182	if (ab > 0) self.shift(ab) catch unreachable;
183	}
184
185	/// Skip zero terminated string.
186	pub fn skipStringZ(self: *Self) !void {
187	while (true) {
188	if (try self.readF(u8, 0) == 0) break;
189	}
190	}
191
192	/// Read deflate fixed fixed code.
193	/// Reads first 7 bits, and then mybe 1 or 2 more to get full 7,8 or 9 bit code.
194	/// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
195	/// Lit Value Bits Codes
196	/// --------- ---- -----
197	/// 0 - 143 8 00110000 through
198	/// 10111111
199	/// 144 - 255 9 110010000 through
200	/// 111111111
201	/// 256 - 279 7 0000000 through
202	/// 0010111
203	/// 280 - 287 8 11000000 through
204	/// 11000111
205	pub fn readFixedCode(self: *Self) !u16 {
206	try self.fill(7 + 2);
207	const code7 = try self.readF(u7, flag.buffered \| flag.reverse);
208	if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
209	return @as(u16, code7) + 256;
210	} else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
211	return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, flag.buffered)) - 0b0011_0000;
212	} else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
213	return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, flag.buffered) + 280;
214	} else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
215	return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, flag.buffered \| flag.reverse)) + 144;
216	}
217	}
218	};
219	}
220
221	test "flate.BitReader" {
222	var fbs = std.io.fixedBufferStream(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
223	var br = bitReader(fbs.reader());
224	const F = BitReader(@TypeOf(fbs.reader())).flag;
225
226	try testing.expectEqual(@as(u8, 48), br.nbits);
227	try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
228
229	try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
230	try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
231	try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
232	try testing.expectEqual(@as(u3, 5), br.alignBits());
233
234	try testing.expect(try br.readF(u8, F.peek) == 0b0001_1110);
235	try testing.expect(try br.readF(u9, F.peek) == 0b1_0001_1110);
236	try br.shift(9);
237	try testing.expectEqual(@as(u8, 36), br.nbits);
238	try testing.expectEqual(@as(u3, 4), br.alignBits());
239
240	try testing.expect(try br.readF(u4, 0) == 0b0100);
241	try testing.expectEqual(@as(u8, 32), br.nbits);
242	try testing.expectEqual(@as(u3, 0), br.alignBits());
243
244	try br.shift(1);
245	try testing.expectEqual(@as(u3, 7), br.alignBits());
246	try br.shift(1);
247	try testing.expectEqual(@as(u3, 6), br.alignBits());
248	br.alignToByte();
249	try testing.expectEqual(@as(u3, 0), br.alignBits());
250
251	try testing.expectEqual(@as(u64, 0xc9), br.bits);
252	try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
253	try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
254	}
255
256	test "flate.BitReader read block type 1 data" {
257	const data = [_]u8{
258	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
259	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
260	0x0c, 0x01, 0x02, 0x03, //
261	0xaa, 0xbb, 0xcc, 0xdd,
262	};
263	var fbs = std.io.fixedBufferStream(&data);
264	var br = bitReader(fbs.reader());
265	const F = BitReader(@TypeOf(fbs.reader())).flag;
266
267	try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
268	try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
269
270	for ("Hello world\n") \|c\| {
271	try testing.expectEqual(@as(u8, c), try br.readF(u8, F.reverse) - 0x30);
272	}
273	try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
274	br.alignToByte();
275	try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
276	try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
277	try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
278	}
279
280	test "flate.BitReader init" {
281	const data = [_]u8{
282	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
283	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
284	};
285	var fbs = std.io.fixedBufferStream(&data);
286	var br = bitReader(fbs.reader());
287
288	try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
289	try br.shift(8);
290	try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
291	try br.fill(60); // fill with 1 byte
292	try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
293	try br.shift(8 * 4 + 4);
294	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
295
296	try br.fill(60); // fill with 4 bytes (shift by 4)
297	try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
298	try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
299
300	try br.shift(@intCast(br.nbits)); // clear buffer
301	try br.fill(8); // refill with the rest of the bytes
302	try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
303	}
304
305	test "flate.BitReader readAll" {
306	const data = [_]u8{
307	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
308	0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
309	};
310	var fbs = std.io.fixedBufferStream(&data);
311	var br = bitReader(fbs.reader());
312
313	try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
314
315	var out: [16]u8 = undefined;
316	try br.readAll(out[0..]);
317	try testing.expect(br.nbits == 0);
318	try testing.expect(br.bits == 0);
319
320	try testing.expectEqualSlices(u8, data[0..16], &out);
321	}
322
323	test "flate.BitReader readFixedCode" {
324	const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
325
326	var fbs = std.io.fixedBufferStream(&fixed_codes);
327	var rdr = bitReader(fbs.reader());
328
329	for (0..286) \|c\| {
330	try testing.expectEqual(c, try rdr.readFixedCode());
331	}
332	try testing.expect(rdr.nbits == 0);
333	}

lib/std/compress/flate/bit_writer.zig created+99

...	@@ -0,0 +1,99 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3
4	/// Bit writer for use in deflate (compression).
5	///
6	/// Has internal bits buffer of 64 bits and internal bytes buffer of 248 bytes.
7	/// When we accumulate 48 bits 6 bytes are moved to the bytes buffer. When we
8	/// accumulate 240 bytes they are flushed to the underlying inner_writer.
9	///
10	pub fn BitWriter(comptime WriterType: type) type {
11	// buffer_flush_size indicates the buffer size
12	// after which bytes are flushed to the writer.
13	// Should preferably be a multiple of 6, since
14	// we accumulate 6 bytes between writes to the buffer.
15	const buffer_flush_size = 240;
16
17	// buffer_size is the actual output byte buffer size.
18	// It must have additional headroom for a flush
19	// which can contain up to 8 bytes.
20	const buffer_size = buffer_flush_size + 8;
21
22	return struct {
23	inner_writer: WriterType,
24
25	// Data waiting to be written is bytes[0 .. nbytes]
26	// and then the low nbits of bits. Data is always written
27	// sequentially into the bytes array.
28	bits: u64 = 0,
29	nbits: u32 = 0, // number of bits
30	bytes: [buffer_size]u8 = undefined,
31	nbytes: u32 = 0, // number of bytes
32
33	const Self = @This();
34
35	pub const Error = WriterType.Error \|\| error{UnfinishedBits};
36
37	pub fn init(writer: WriterType) Self {
38	return .{ .inner_writer = writer };
39	}
40
41	pub fn setWriter(self: *Self, new_writer: WriterType) void {
42	//assert(self.bits == 0 and self.nbits == 0 and self.nbytes == 0);
43	self.inner_writer = new_writer;
44	}
45
46	pub fn flush(self: *Self) Error!void {
47	var n = self.nbytes;
48	while (self.nbits != 0) {
49	self.bytes[n] = @as(u8, @truncate(self.bits));
50	self.bits >>= 8;
51	if (self.nbits > 8) { // Avoid underflow
52	self.nbits -= 8;
53	} else {
54	self.nbits = 0;
55	}
56	n += 1;
57	}
58	self.bits = 0;
59	_ = try self.inner_writer.write(self.bytes[0..n]);
60	self.nbytes = 0;
61	}
62
63	pub fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
64	self.bits \|= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
65	self.nbits += nb;
66	if (self.nbits < 48)
67	return;
68
69	var n = self.nbytes;
70	std.mem.writeInt(u64, self.bytes[n..][0..8], self.bits, .little);
71	n += 6;
72	if (n >= buffer_flush_size) {
73	_ = try self.inner_writer.write(self.bytes[0..n]);
74	n = 0;
75	}
76	self.nbytes = n;
77	self.bits >>= 48;
78	self.nbits -= 48;
79	}
80
81	pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
82	var n = self.nbytes;
83	if (self.nbits & 7 != 0) {
84	return error.UnfinishedBits;
85	}
86	while (self.nbits != 0) {
87	self.bytes[n] = @as(u8, @truncate(self.bits));
88	self.bits >>= 8;
89	self.nbits -= 8;
90	n += 1;
91	}
92	if (n != 0) {
93	_ = try self.inner_writer.write(self.bytes[0..n]);
94	}
95	self.nbytes = 0;
96	_ = try self.inner_writer.write(bytes);
97	}
98	};
99	}

lib/std/compress/flate/block_writer.zig created+706

...	@@ -0,0 +1,706 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4
5	const hc = @import("huffman_encoder.zig");
6	const consts = @import("consts.zig").huffman;
7	const Token = @import("Token.zig");
8	const BitWriter = @import("bit_writer.zig").BitWriter;
9
10	pub fn blockWriter(writer: anytype) BlockWriter(@TypeOf(writer)) {
11	return BlockWriter(@TypeOf(writer)).init(writer);
12	}
13
14	/// Accepts list of tokens, decides what is best block type to write. What block
15	/// type will provide best compression. Writes header and body of the block.
16	///
17	pub fn BlockWriter(comptime WriterType: type) type {
18	const BitWriterType = BitWriter(WriterType);
19	return struct {
20	const codegen_order = consts.codegen_order;
21	const end_code_mark = 255;
22	const Self = @This();
23
24	pub const Error = BitWriterType.Error;
25	bit_writer: BitWriterType,
26
27	codegen_freq: [consts.codegen_code_count]u16 = undefined,
28	literal_freq: [consts.max_num_lit]u16 = undefined,
29	distance_freq: [consts.distance_code_count]u16 = undefined,
30	codegen: [consts.max_num_lit + consts.distance_code_count + 1]u8 = undefined,
31	literal_encoding: hc.LiteralEncoder = .{},
32	distance_encoding: hc.DistanceEncoder = .{},
33	codegen_encoding: hc.CodegenEncoder = .{},
34	fixed_literal_encoding: hc.LiteralEncoder,
35	fixed_distance_encoding: hc.DistanceEncoder,
36	huff_distance: hc.DistanceEncoder,
37
38	pub fn init(writer: WriterType) Self {
39	return .{
40	.bit_writer = BitWriterType.init(writer),
41	.fixed_literal_encoding = hc.fixedLiteralEncoder(),
42	.fixed_distance_encoding = hc.fixedDistanceEncoder(),
43	.huff_distance = hc.huffmanDistanceEncoder(),
44	};
45	}
46
47	/// Flush intrenal bit buffer to the writer.
48	/// Should be called only when bit stream is at byte boundary.
49	///
50	/// That is after final block; when last byte could be incomplete or
51	/// after stored block; which is aligned to the byte bounday (it has x
52	/// padding bits after first 3 bits).
53	pub fn flush(self: *Self) Error!void {
54	try self.bit_writer.flush();
55	}
56
57	pub fn setWriter(self: *Self, new_writer: WriterType) void {
58	self.bit_writer.setWriter(new_writer);
59	}
60
61	fn writeCode(self: *Self, c: hc.HuffCode) Error!void {
62	try self.bit_writer.writeBits(c.code, c.len);
63	}
64
65	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
66	// the literal and distance lengths arrays (which are concatenated into a single
67	// array). This method generates that run-length encoding.
68	//
69	// The result is written into the codegen array, and the frequencies
70	// of each code is written into the codegen_freq array.
71	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
72	// information. Code bad_code is an end marker
73	//
74	// num_literals: The number of literals in literal_encoding
75	// num_distances: The number of distances in distance_encoding
76	// lit_enc: The literal encoder to use
77	// dist_enc: The distance encoder to use
78	fn generateCodegen(
79	self: *Self,
80	num_literals: u32,
81	num_distances: u32,
82	lit_enc: *hc.LiteralEncoder,
83	dist_enc: *hc.DistanceEncoder,
84	) void {
85	for (self.codegen_freq, 0..) \|_, i\| {
86	self.codegen_freq[i] = 0;
87	}
88
89	// Note that we are using codegen both as a temporary variable for holding
90	// a copy of the frequencies, and as the place where we put the result.
91	// This is fine because the output is always shorter than the input used
92	// so far.
93	var codegen = &self.codegen; // cache
94	// Copy the concatenated code sizes to codegen. Put a marker at the end.
95	var cgnl = codegen[0..num_literals];
96	for (cgnl, 0..) \|_, i\| {
97	cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
98	}
99
100	cgnl = codegen[num_literals .. num_literals + num_distances];
101	for (cgnl, 0..) \|_, i\| {
102	cgnl[i] = @as(u8, @intCast(dist_enc.codes[i].len));
103	}
104	codegen[num_literals + num_distances] = end_code_mark;
105
106	var size = codegen[0];
107	var count: i32 = 1;
108	var out_index: u32 = 0;
109	var in_index: u32 = 1;
110	while (size != end_code_mark) : (in_index += 1) {
111	// INVARIANT: We have seen "count" copies of size that have not yet
112	// had output generated for them.
113	const next_size = codegen[in_index];
114	if (next_size == size) {
115	count += 1;
116	continue;
117	}
118	// We need to generate codegen indicating "count" of size.
119	if (size != 0) {
120	codegen[out_index] = size;
121	out_index += 1;
122	self.codegen_freq[size] += 1;
123	count -= 1;
124	while (count >= 3) {
125	var n: i32 = 6;
126	if (n > count) {
127	n = count;
128	}
129	codegen[out_index] = 16;
130	out_index += 1;
131	codegen[out_index] = @as(u8, @intCast(n - 3));
132	out_index += 1;
133	self.codegen_freq[16] += 1;
134	count -= n;
135	}
136	} else {
137	while (count >= 11) {
138	var n: i32 = 138;
139	if (n > count) {
140	n = count;
141	}
142	codegen[out_index] = 18;
143	out_index += 1;
144	codegen[out_index] = @as(u8, @intCast(n - 11));
145	out_index += 1;
146	self.codegen_freq[18] += 1;
147	count -= n;
148	}
149	if (count >= 3) {
150	// 3 <= count <= 10
151	codegen[out_index] = 17;
152	out_index += 1;
153	codegen[out_index] = @as(u8, @intCast(count - 3));
154	out_index += 1;
155	self.codegen_freq[17] += 1;
156	count = 0;
157	}
158	}
159	count -= 1;
160	while (count >= 0) : (count -= 1) {
161	codegen[out_index] = size;
162	out_index += 1;
163	self.codegen_freq[size] += 1;
164	}
165	// Set up invariant for next time through the loop.
166	size = next_size;
167	count = 1;
168	}
169	// Marker indicating the end of the codegen.
170	codegen[out_index] = end_code_mark;
171	}
172
173	const DynamicSize = struct {
174	size: u32,
175	num_codegens: u32,
176	};
177
178	// dynamicSize returns the size of dynamically encoded data in bits.
179	fn dynamicSize(
180	self: *Self,
181	lit_enc: *hc.LiteralEncoder, // literal encoder
182	dist_enc: *hc.DistanceEncoder, // distance encoder
183	extra_bits: u32,
184	) DynamicSize {
185	var num_codegens = self.codegen_freq.len;
186	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
187	num_codegens -= 1;
188	}
189	const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
190	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
191	self.codegen_freq[16] * 2 +
192	self.codegen_freq[17] * 3 +
193	self.codegen_freq[18] * 7;
194	const size = header +
195	lit_enc.bitLength(&self.literal_freq) +
196	dist_enc.bitLength(&self.distance_freq) +
197	extra_bits;
198
199	return DynamicSize{
200	.size = @as(u32, @intCast(size)),
201	.num_codegens = @as(u32, @intCast(num_codegens)),
202	};
203	}
204
205	// fixedSize returns the size of dynamically encoded data in bits.
206	fn fixedSize(self: *Self, extra_bits: u32) u32 {
207	return 3 +
208	self.fixed_literal_encoding.bitLength(&self.literal_freq) +
209	self.fixed_distance_encoding.bitLength(&self.distance_freq) +
210	extra_bits;
211	}
212
213	const StoredSize = struct {
214	size: u32,
215	storable: bool,
216	};
217
218	// storedSizeFits calculates the stored size, including header.
219	// The function returns the size in bits and whether the block
220	// fits inside a single block.
221	fn storedSizeFits(in: ?[]const u8) StoredSize {
222	if (in == null) {
223	return .{ .size = 0, .storable = false };
224	}
225	if (in.?.len <= consts.max_store_block_size) {
226	return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
227	}
228	return .{ .size = 0, .storable = false };
229	}
230
231	// Write the header of a dynamic Huffman block to the output stream.
232	//
233	// num_literals: The number of literals specified in codegen
234	// num_distances: The number of distances specified in codegen
235	// num_codegens: The number of codegens used in codegen
236	// eof: Is it the end-of-file? (end of stream)
237	fn dynamicHeader(
238	self: *Self,
239	num_literals: u32,
240	num_distances: u32,
241	num_codegens: u32,
242	eof: bool,
243	) Error!void {
244	const first_bits: u32 = if (eof) 5 else 4;
245	try self.bit_writer.writeBits(first_bits, 3);
246	try self.bit_writer.writeBits(num_literals - 257, 5);
247	try self.bit_writer.writeBits(num_distances - 1, 5);
248	try self.bit_writer.writeBits(num_codegens - 4, 4);
249
250	var i: u32 = 0;
251	while (i < num_codegens) : (i += 1) {
252	const value = self.codegen_encoding.codes[codegen_order[i]].len;
253	try self.bit_writer.writeBits(value, 3);
254	}
255
256	i = 0;
257	while (true) {
258	const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
259	i += 1;
260	if (code_word == end_code_mark) {
261	break;
262	}
263	try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
264
265	switch (code_word) {
266	16 => {
267	try self.bit_writer.writeBits(self.codegen[i], 2);
268	i += 1;
269	},
270	17 => {
271	try self.bit_writer.writeBits(self.codegen[i], 3);
272	i += 1;
273	},
274	18 => {
275	try self.bit_writer.writeBits(self.codegen[i], 7);
276	i += 1;
277	},
278	else => {},
279	}
280	}
281	}
282
283	fn storedHeader(self: *Self, length: usize, eof: bool) Error!void {
284	assert(length <= 65535);
285	const flag: u32 = if (eof) 1 else 0;
286	try self.bit_writer.writeBits(flag, 3);
287	try self.flush();
288	const l: u16 = @intCast(length);
289	try self.bit_writer.writeBits(l, 16);
290	try self.bit_writer.writeBits(~l, 16);
291	}
292
293	fn fixedHeader(self: *Self, eof: bool) Error!void {
294	// Indicate that we are a fixed Huffman block
295	var value: u32 = 2;
296	if (eof) {
297	value = 3;
298	}
299	try self.bit_writer.writeBits(value, 3);
300	}
301
302	// Write a block of tokens with the smallest encoding. Will choose block type.
303	// The original input can be supplied, and if the huffman encoded data
304	// is larger than the original bytes, the data will be written as a
305	// stored block.
306	// If the input is null, the tokens will always be Huffman encoded.
307	pub fn write(self: *Self, tokens: []const Token, eof: bool, input: ?[]const u8) Error!void {
308	const lit_and_dist = self.indexTokens(tokens);
309	const num_literals = lit_and_dist.num_literals;
310	const num_distances = lit_and_dist.num_distances;
311
312	var extra_bits: u32 = 0;
313	const ret = storedSizeFits(input);
314	const stored_size = ret.size;
315	const storable = ret.storable;
316
317	if (storable) {
318	// We only bother calculating the costs of the extra bits required by
319	// the length of distance fields (which will be the same for both fixed
320	// and dynamic encoding), if we need to compare those two encodings
321	// against stored encoding.
322	var length_code: u16 = Token.length_codes_start + 8;
323	while (length_code < num_literals) : (length_code += 1) {
324	// First eight length codes have extra size = 0.
325	extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
326	@as(u32, @intCast(Token.lengthExtraBits(length_code)));
327	}
328	var distance_code: u16 = 4;
329	while (distance_code < num_distances) : (distance_code += 1) {
330	// First four distance codes have extra size = 0.
331	extra_bits += @as(u32, @intCast(self.distance_freq[distance_code])) *
332	@as(u32, @intCast(Token.distanceExtraBits(distance_code)));
333	}
334	}
335
336	// Figure out smallest code.
337	// Fixed Huffman baseline.
338	var literal_encoding = &self.fixed_literal_encoding;
339	var distance_encoding = &self.fixed_distance_encoding;
340	var size = self.fixedSize(extra_bits);
341
342	// Dynamic Huffman?
343	var num_codegens: u32 = 0;
344
345	// Generate codegen and codegenFrequencies, which indicates how to encode
346	// the literal_encoding and the distance_encoding.
347	self.generateCodegen(
348	num_literals,
349	num_distances,
350	&self.literal_encoding,
351	&self.distance_encoding,
352	);
353	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
354	const dynamic_size = self.dynamicSize(
355	&self.literal_encoding,
356	&self.distance_encoding,
357	extra_bits,
358	);
359	const dyn_size = dynamic_size.size;
360	num_codegens = dynamic_size.num_codegens;
361
362	if (dyn_size < size) {
363	size = dyn_size;
364	literal_encoding = &self.literal_encoding;
365	distance_encoding = &self.distance_encoding;
366	}
367
368	// Stored bytes?
369	if (storable and stored_size < size) {
370	try self.storedBlock(input.?, eof);
371	return;
372	}
373
374	// Huffman.
375	if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
376	try self.fixedHeader(eof);
377	} else {
378	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
379	}
380
381	// Write the tokens.
382	try self.writeTokens(tokens, &literal_encoding.codes, &distance_encoding.codes);
383	}
384
385	pub fn storedBlock(self: *Self, input: []const u8, eof: bool) Error!void {
386	try self.storedHeader(input.len, eof);
387	try self.bit_writer.writeBytes(input);
388	}
389
390	// writeBlockDynamic encodes a block using a dynamic Huffman table.
391	// This should be used if the symbols used have a disproportionate
392	// histogram distribution.
393	// If input is supplied and the compression savings are below 1/16th of the
394	// input size the block is stored.
395	fn dynamicBlock(
396	self: *Self,
397	tokens: []const Token,
398	eof: bool,
399	input: ?[]const u8,
400	) Error!void {
401	const total_tokens = self.indexTokens(tokens);
402	const num_literals = total_tokens.num_literals;
403	const num_distances = total_tokens.num_distances;
404
405	// Generate codegen and codegenFrequencies, which indicates how to encode
406	// the literal_encoding and the distance_encoding.
407	self.generateCodegen(
408	num_literals,
409	num_distances,
410	&self.literal_encoding,
411	&self.distance_encoding,
412	);
413	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
414	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.distance_encoding, 0);
415	const size = dynamic_size.size;
416	const num_codegens = dynamic_size.num_codegens;
417
418	// Store bytes, if we don't get a reasonable improvement.
419
420	const stored_size = storedSizeFits(input);
421	const ssize = stored_size.size;
422	const storable = stored_size.storable;
423	if (storable and ssize < (size + (size >> 4))) {
424	try self.storedBlock(input.?, eof);
425	return;
426	}
427
428	// Write Huffman table.
429	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
430
431	// Write the tokens.
432	try self.writeTokens(tokens, &self.literal_encoding.codes, &self.distance_encoding.codes);
433	}
434
435	const TotalIndexedTokens = struct {
436	num_literals: u32,
437	num_distances: u32,
438	};
439
440	// Indexes a slice of tokens followed by an end_block_marker, and updates
441	// literal_freq and distance_freq, and generates literal_encoding
442	// and distance_encoding.
443	// The number of literal and distance tokens is returned.
444	fn indexTokens(self: *Self, tokens: []const Token) TotalIndexedTokens {
445	var num_literals: u32 = 0;
446	var num_distances: u32 = 0;
447
448	for (self.literal_freq, 0..) \|_, i\| {
449	self.literal_freq[i] = 0;
450	}
451	for (self.distance_freq, 0..) \|_, i\| {
452	self.distance_freq[i] = 0;
453	}
454
455	for (tokens) \|t\| {
456	if (t.kind == Token.Kind.literal) {
457	self.literal_freq[t.literal()] += 1;
458	continue;
459	}
460	self.literal_freq[t.lengthCode()] += 1;
461	self.distance_freq[t.distanceCode()] += 1;
462	}
463	// add end_block_marker token at the end
464	self.literal_freq[consts.end_block_marker] += 1;
465
466	// get the number of literals
467	num_literals = @as(u32, @intCast(self.literal_freq.len));
468	while (self.literal_freq[num_literals - 1] == 0) {
469	num_literals -= 1;
470	}
471	// get the number of distances
472	num_distances = @as(u32, @intCast(self.distance_freq.len));
473	while (num_distances > 0 and self.distance_freq[num_distances - 1] == 0) {
474	num_distances -= 1;
475	}
476	if (num_distances == 0) {
477	// We haven't found a single match. If we want to go with the dynamic encoding,
478	// we should count at least one distance to be sure that the distance huffman tree could be encoded.
479	self.distance_freq[0] = 1;
480	num_distances = 1;
481	}
482	self.literal_encoding.generate(&self.literal_freq, 15);
483	self.distance_encoding.generate(&self.distance_freq, 15);
484	return TotalIndexedTokens{
485	.num_literals = num_literals,
486	.num_distances = num_distances,
487	};
488	}
489
490	// Writes a slice of tokens to the output followed by and end_block_marker.
491	// codes for literal and distance encoding must be supplied.
492	fn writeTokens(
493	self: *Self,
494	tokens: []const Token,
495	le_codes: []hc.HuffCode,
496	oe_codes: []hc.HuffCode,
497	) Error!void {
498	for (tokens) \|t\| {
499	if (t.kind == Token.Kind.literal) {
500	try self.writeCode(le_codes[t.literal()]);
501	continue;
502	}
503
504	// Write the length
505	const le = t.lengthEncoding();
506	try self.writeCode(le_codes[le.code]);
507	if (le.extra_bits > 0) {
508	try self.bit_writer.writeBits(le.extra_length, le.extra_bits);
509	}
510
511	// Write the distance
512	const oe = t.distanceEncoding();
513	try self.writeCode(oe_codes[oe.code]);
514	if (oe.extra_bits > 0) {
515	try self.bit_writer.writeBits(oe.extra_distance, oe.extra_bits);
516	}
517	}
518	// add end_block_marker at the end
519	try self.writeCode(le_codes[consts.end_block_marker]);
520	}
521
522	// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
523	// if the results only gains very little from compression.
524	pub fn huffmanBlock(self: *Self, input: []const u8, eof: bool) Error!void {
525	// Add everything as literals
526	histogram(input, &self.literal_freq);
527
528	self.literal_freq[consts.end_block_marker] = 1;
529
530	const num_literals = consts.end_block_marker + 1;
531	self.distance_freq[0] = 1;
532	const num_distances = 1;
533
534	self.literal_encoding.generate(&self.literal_freq, 15);
535
536	// Figure out smallest code.
537	// Always use dynamic Huffman or Store
538	var num_codegens: u32 = 0;
539
540	// Generate codegen and codegenFrequencies, which indicates how to encode
541	// the literal_encoding and the distance_encoding.
542	self.generateCodegen(
543	num_literals,
544	num_distances,
545	&self.literal_encoding,
546	&self.huff_distance,
547	);
548	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
549	const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_distance, 0);
550	const size = dynamic_size.size;
551	num_codegens = dynamic_size.num_codegens;
552
553	// Store bytes, if we don't get a reasonable improvement.
554	const stored_size_ret = storedSizeFits(input);
555	const ssize = stored_size_ret.size;
556	const storable = stored_size_ret.storable;
557
558	if (storable and ssize < (size + (size >> 4))) {
559	try self.storedBlock(input, eof);
560	return;
561	}
562
563	// Huffman.
564	try self.dynamicHeader(num_literals, num_distances, num_codegens, eof);
565	const encoding = self.literal_encoding.codes[0..257];
566
567	for (input) \|t\| {
568	const c = encoding[t];
569	try self.bit_writer.writeBits(c.code, c.len);
570	}
571	try self.writeCode(encoding[consts.end_block_marker]);
572	}
573
574	// histogram accumulates a histogram of b in h.
575	fn histogram(b: []const u8, h: *[286]u16) void {
576	// Clear histogram
577	for (h, 0..) \|_, i\| {
578	h[i] = 0;
579	}
580
581	var lh = h.*[0..256];
582	for (b) \|t\| {
583	lh[t] += 1;
584	}
585	}
586	};
587	}
588
589	// tests
590	const expect = std.testing.expect;
591	const fmt = std.fmt;
592	const testing = std.testing;
593	const ArrayList = std.ArrayList;
594
595	const TestCase = @import("testdata/block_writer.zig").TestCase;
596	const testCases = @import("testdata/block_writer.zig").testCases;
597
598	// tests if the writeBlock encoding has changed.
599	test "flate.BlockWriter write" {
600	inline for (0..testCases.len) \|i\| {
601	try testBlock(testCases[i], .write_block);
602	}
603	}
604
605	// tests if the writeBlockDynamic encoding has changed.
606	test "flate.BlockWriter dynamicBlock" {
607	inline for (0..testCases.len) \|i\| {
608	try testBlock(testCases[i], .write_dyn_block);
609	}
610	}
611
612	test "flate.BlockWriter huffmanBlock" {
613	inline for (0..testCases.len) \|i\| {
614	try testBlock(testCases[i], .write_huffman_block);
615	}
616	try testBlock(.{
617	.tokens = &[_]Token{},
618	.input = "huffman-rand-max.input",
619	.want = "huffman-rand-max.{s}.expect",
620	}, .write_huffman_block);
621	}
622
623	const TestFn = enum {
624	write_block,
625	write_dyn_block, // write dynamic block
626	write_huffman_block,
627
628	fn to_s(self: TestFn) []const u8 {
629	return switch (self) {
630	.write_block => "wb",
631	.write_dyn_block => "dyn",
632	.write_huffman_block => "huff",
633	};
634	}
635
636	fn write(
637	comptime self: TestFn,
638	bw: anytype,
639	tok: []const Token,
640	input: ?[]const u8,
641	final: bool,
642	) !void {
643	switch (self) {
644	.write_block => try bw.write(tok, final, input),
645	.write_dyn_block => try bw.dynamicBlock(tok, final, input),
646	.write_huffman_block => try bw.huffmanBlock(input.?, final),
647	}
648	try bw.flush();
649	}
650	};
651
652	// testBlock tests a block against its references
653	//
654	// size
655	// 64K [file-name].input - input non compressed file
656	// 8.1K [file-name].golden -
657	// 78 [file-name].dyn.expect - output with writeBlockDynamic
658	// 78 [file-name].wb.expect - output with writeBlock
659	// 8.1K [file-name].huff.expect - output with writeBlockHuff
660	// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
661	// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
662	//
663	// wb - writeBlock
664	// dyn - writeBlockDynamic
665	// huff - writeBlockHuff
666	//
667	fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
668	if (tc.input.len != 0 and tc.want.len != 0) {
669	const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
670	const input = @embedFile("testdata/block_writer/" ++ tc.input);
671	const want = @embedFile("testdata/block_writer/" ++ want_name);
672	try testWriteBlock(tfn, input, want, tc.tokens);
673	}
674
675	if (tfn == .write_huffman_block) {
676	return;
677	}
678
679	const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
680	const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
681	try testWriteBlock(tfn, null, want, tc.tokens);
682	}
683
684	// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
685	fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
686	var buf = ArrayList(u8).init(testing.allocator);
687	var bw = blockWriter(buf.writer());
688	try tfn.write(&bw, tokens, input, false);
689	var got = buf.items;
690	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
691	try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
692	//
693	// Test if the writer produces the same output after reset.
694	buf.deinit();
695	buf = ArrayList(u8).init(testing.allocator);
696	defer buf.deinit();
697	bw.setWriter(buf.writer());
698
699	try tfn.write(&bw, tokens, input, true);
700	try bw.flush();
701	got = buf.items;
702
703	try expect(got[0] & 1 == 1); // bfinal is set
704	buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
705	try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
706	}

lib/std/compress/flate/consts.zig created+49

...	@@ -0,0 +1,49 @@
1	pub const deflate = struct {
2	// Number of tokens to accumlate in deflate before starting block encoding.
3	//
4	// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
5	// 8 and max 9 that gives 14 or 15 bits.
6	pub const tokens = 1 << 15;
7	};
8
9	pub const match = struct {
10	pub const base_length = 3; // smallest match length per the RFC section 3.2.5
11	pub const min_length = 4; // min length used in this algorithm
12	pub const max_length = 258;
13
14	pub const min_distance = 1;
15	pub const max_distance = 32768;
16	};
17
18	pub const history = struct {
19	pub const len = match.max_distance;
20	};
21
22	pub const lookup = struct {
23	pub const bits = 15;
24	pub const len = 1 << bits;
25	pub const shift = 32 - bits;
26	};
27
28	pub const huffman = struct {
29	// The odd order in which the codegen code sizes are written.
30	pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
31	// The number of codegen codes.
32	pub const codegen_code_count = 19;
33
34	// The largest distance code.
35	pub const distance_code_count = 30;
36
37	// Maximum number of literals.
38	pub const max_num_lit = 286;
39
40	// Max number of frequencies used for a Huffman Code
41	// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
42	// The largest of these is max_num_lit.
43	pub const max_num_frequencies = max_num_lit;
44
45	// Biggest block size for uncompressed block.
46	pub const max_store_block_size = 65535;
47	// The special code used to mark the end of a block.
48	pub const end_block_marker = 256;
49	};

lib/std/compress/flate/container.zig created+207

...	@@ -0,0 +1,207 @@
1	//! Container of the deflate bit stream body. Container adds header before
2	//! deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
3	//! no footer, raw bit stream).
4	//!
5	//! Zlib format is defined in rfc 1950. Header has 2 bytes and footer 4 bytes
6	//! addler 32 checksum.
7	//!
8	//! Gzip format is defined in rfc 1952. Header has 10+ bytes and footer 4 bytes
9	//! crc32 checksum and 4 bytes of uncompressed data length.
10	//!
11	//!
12	//! rfc 1950: https://datatracker.ietf.org/doc/html/rfc1950#page-4
13	//! rfc 1952: https://datatracker.ietf.org/doc/html/rfc1952#page-5
14	//!
15
16	const std = @import("std");
17
18	pub const Container = enum {
19	raw, // no header or footer
20	gzip, // gzip header and footer
21	zlib, // zlib header and footer
22
23	pub fn size(w: Container) usize {
24	return headerSize(w) + footerSize(w);
25	}
26
27	pub fn headerSize(w: Container) usize {
28	return switch (w) {
29	.gzip => 10,
30	.zlib => 2,
31	.raw => 0,
32	};
33	}
34
35	pub fn footerSize(w: Container) usize {
36	return switch (w) {
37	.gzip => 8,
38	.zlib => 4,
39	.raw => 0,
40	};
41	}
42
43	pub const list = [_]Container{ .raw, .gzip, .zlib };
44
45	pub const Error = error{
46	BadGzipHeader,
47	BadZlibHeader,
48	WrongGzipChecksum,
49	WrongGzipSize,
50	WrongZlibChecksum,
51	};
52
53	pub fn writeHeader(comptime wrap: Container, writer: anytype) !void {
54	switch (wrap) {
55	.gzip => {
56	// GZIP 10 byte header (https://datatracker.ietf.org/doc/html/rfc1952#page-5):
57	// - ID1 (IDentification 1), always 0x1f
58	// - ID2 (IDentification 2), always 0x8b
59	// - CM (Compression Method), always 8 = deflate
60	// - FLG (Flags), all set to 0
61	// - 4 bytes, MTIME (Modification time), not used, all set to zero
62	// - XFL (eXtra FLags), all set to zero
63	// - OS (Operating System), 03 = Unix
64	const gzipHeader = [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 };
65	try writer.writeAll(&gzipHeader);
66	},
67	.zlib => {
68	// ZLIB has a two-byte header (https://datatracker.ietf.org/doc/html/rfc1950#page-4):
69	// 1st byte:
70	// - First four bits is the CINFO (compression info), which is 7 for the default deflate window size.
71	// - The next four bits is the CM (compression method), which is 8 for deflate.
72	// 2nd byte:
73	// - Two bits is the FLEVEL (compression level). Values are: 0=fastest, 1=fast, 2=default, 3=best.
74	// - The next bit, FDICT, is set if a dictionary is given.
75	// - The final five FCHECK bits form a mod-31 checksum.
76	//
77	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
78	const zlibHeader = [_]u8{ 0x78, 0b10_0_11100 };
79	try writer.writeAll(&zlibHeader);
80	},
81	.raw => {},
82	}
83	}
84
85	pub fn writeFooter(comptime wrap: Container, hasher: *Hasher(wrap), writer: anytype) !void {
86	var bits: [4]u8 = undefined;
87	switch (wrap) {
88	.gzip => {
89	// GZIP 8 bytes footer
90	// - 4 bytes, CRC32 (CRC-32)
91	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
92	std.mem.writeInt(u32, &bits, hasher.chksum(), .little);
93	try writer.writeAll(&bits);
94
95	std.mem.writeInt(u32, &bits, hasher.bytesRead(), .little);
96	try writer.writeAll(&bits);
97	},
98	.zlib => {
99	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
100	// 4 bytes of ADLER32 (Adler-32 checksum)
101	// Checksum value of the uncompressed data (excluding any
102	// dictionary data) computed according to Adler-32
103	// algorithm.
104	std.mem.writeInt(u32, &bits, hasher.chksum(), .big);
105	try writer.writeAll(&bits);
106	},
107	.raw => {},
108	}
109	}
110
111	pub fn parseHeader(comptime wrap: Container, reader: anytype) !void {
112	switch (wrap) {
113	.gzip => try parseGzipHeader(reader),
114	.zlib => try parseZlibHeader(reader),
115	.raw => {},
116	}
117	}
118
119	fn parseGzipHeader(reader: anytype) !void {
120	const magic1 = try reader.read(u8);
121	const magic2 = try reader.read(u8);
122	const method = try reader.read(u8);
123	const flags = try reader.read(u8);
124	try reader.skipBytes(6); // mtime(4), xflags, os
125	if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
126	return error.BadGzipHeader;
127	// Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
128	if (flags != 0) {
129	if (flags & 0b0000_0100 != 0) { // FEXTRA
130	const extra_len = try reader.read(u16);
131	try reader.skipBytes(extra_len);
132	}
133	if (flags & 0b0000_1000 != 0) { // FNAME
134	try reader.skipStringZ();
135	}
136	if (flags & 0b0001_0000 != 0) { // FCOMMENT
137	try reader.skipStringZ();
138	}
139	if (flags & 0b0000_0010 != 0) { // FHCRC
140	try reader.skipBytes(2);
141	}
142	}
143	}
144
145	fn parseZlibHeader(reader: anytype) !void {
146	const cm = try reader.read(u4);
147	const cinfo = try reader.read(u4);
148	_ = try reader.read(u8);
149	if (cm != 8 or cinfo > 7) {
150	return error.BadZlibHeader;
151	}
152	}
153
154	pub fn parseFooter(comptime wrap: Container, hasher: *Hasher(wrap), reader: anytype) !void {
155	switch (wrap) {
156	.gzip => {
157	if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
158	if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
159	},
160	.zlib => {
161	const chksum: u32 = @byteSwap(hasher.chksum());
162	if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
163	},
164	.raw => {},
165	}
166	}
167
168	pub fn Hasher(comptime wrap: Container) type {
169	const HasherType = switch (wrap) {
170	.gzip => std.hash.Crc32,
171	.zlib => std.hash.Adler32,
172	.raw => struct {
173	pub fn init() @This() {
174	return .{};
175	}
176	},
177	};
178
179	return struct {
180	hasher: HasherType = HasherType.init(),
181	bytes: usize = 0,
182
183	const Self = @This();
184
185	pub fn update(self: *Self, buf: []const u8) void {
186	switch (wrap) {
187	.raw => {},
188	else => {
189	self.hasher.update(buf);
190	self.bytes += buf.len;
191	},
192	}
193	}
194
195	pub fn chksum(self: *Self) u32 {
196	switch (wrap) {
197	.raw => return 0,
198	else => return self.hasher.final(),
199	}
200	}
201
202	pub fn bytesRead(self: *Self) u32 {
203	return @truncate(self.bytes);
204	}
205	};
206	}
207	};

lib/std/compress/flate/deflate.zig created+748

...	@@ -0,0 +1,748 @@
1	const std = @import("std");
2	const io = std.io;
3	const assert = std.debug.assert;
4	const testing = std.testing;
5	const expect = testing.expect;
6	const print = std.debug.print;
7
8	const Token = @import("Token.zig");
9	const consts = @import("consts.zig");
10	const BlockWriter = @import("block_writer.zig").BlockWriter;
11	const Container = @import("container.zig").Container;
12	const SlidingWindow = @import("SlidingWindow.zig");
13	const Lookup = @import("Lookup.zig");
14
15	pub const Options = struct {
16	level: Level = .default,
17	};
18
19	/// Trades between speed and compression size.
20	/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
21	/// levels 1-3 are using different algorithm to perform faster but with less
22	/// compression. That is not implemented here.
23	pub const Level = enum(u4) {
24	// zig fmt: off
25	fast = 0xb, level_4 = 4,
26	level_5 = 5,
27	default = 0xc, level_6 = 6,
28	level_7 = 7,
29	level_8 = 8,
30	best = 0xd, level_9 = 9,
31	// zig fmt: on
32	};
33
34	/// Algorithm knobs for each level.
35	const LevelArgs = struct {
36	good: u16, // Do less lookups if we already have match of this length.
37	nice: u16, // Stop looking for better match if we found match with at least this length.
38	lazy: u16, // Don't do lazy match find if got match with at least this length.
39	chain: u16, // How many lookups for previous match to perform.
40
41	pub fn get(level: Level) LevelArgs {
42	// zig fmt: off
43	return switch (level) {
44	.fast, .level_4 => .{ .good = 4, .lazy = 4, .nice = 16, .chain = 16 },
45	.level_5 => .{ .good = 8, .lazy = 16, .nice = 32, .chain = 32 },
46	.default, .level_6 => .{ .good = 8, .lazy = 16, .nice = 128, .chain = 128 },
47	.level_7 => .{ .good = 8, .lazy = 32, .nice = 128, .chain = 256 },
48	.level_8 => .{ .good = 32, .lazy = 128, .nice = 258, .chain = 1024 },
49	.best, .level_9 => .{ .good = 32, .lazy = 258, .nice = 258, .chain = 4096 },
50	};
51	// zig fmt: on
52	}
53	};
54
55	/// Compress plain data from reader into compressed stream written to writer.
56	pub fn compress(comptime container: Container, reader: anytype, writer: anytype, options: Options) !void {
57	var c = try compressor(container, writer, options);
58	try c.compress(reader);
59	try c.finish();
60	}
61
62	/// Create compressor for writer type.
63	pub fn compressor(comptime container: Container, writer: anytype, options: Options) !Compressor(
64	container,
65	@TypeOf(writer),
66	) {
67	return try Compressor(container, @TypeOf(writer)).init(writer, options);
68	}
69
70	/// Compressor type.
71	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
72	const TokenWriterType = BlockWriter(WriterType);
73	return Deflate(container, WriterType, TokenWriterType);
74	}
75
76	/// Default compression algorithm. Has two steps: tokenization and token
77	/// encoding.
78	///
79	/// Tokenization takes uncompressed input stream and produces list of tokens.
80	/// Each token can be literal (byte of data) or match (backrefernce to previous
81	/// data with length and distance). Tokenization accumulators 32K tokens, when
82	/// full or `flush` is called tokens are passed to the `block_writer`. Level
83	/// defines how hard (how slow) it tries to find match.
84	///
85	/// Block writer will decide which type of deflate block to write (stored, fixed,
86	/// dynamic) and encode tokens to the output byte stream. Client has to call
87	/// `finish` to write block with the final bit set.
88	///
89	/// Container defines type of header and footer which can be gzip, zlib or raw.
90	/// They all share same deflate body. Raw has no header or footer just deflate
91	/// body.
92	///
93	/// Compression algorithm explained in rfc-1951 (slightly edited for this case):
94	///
95	/// The compressor uses a chained hash table `lookup` to find duplicated
96	/// strings, using a hash function that operates on 4-byte sequences. At any
97	/// given point during compression, let XYZW be the next 4 input bytes
98	/// (lookahead) to be examined (not necessarily all different, of course).
99	/// First, the compressor examines the hash chain for XYZW. If the chain is
100	/// empty, the compressor simply writes out X as a literal byte and advances
101	/// one byte in the input. If the hash chain is not empty, indicating that the
102	/// sequence XYZW (or, if we are unlucky, some other 4 bytes with the same
103	/// hash function value) has occurred recently, the compressor compares all
104	/// strings on the XYZW hash chain with the actual input data sequence
105	/// starting at the current point, and selects the longest match.
106	///
107	/// To improve overall compression, the compressor defers the selection of
108	/// matches ("lazy matching"): after a match of length N has been found, the
109	/// compressor searches for a longer match starting at the next input byte. If
110	/// it finds a longer match, it truncates the previous match to a length of
111	/// one (thus producing a single literal byte) and then emits the longer
112	/// match. Otherwise, it emits the original match, and, as described above,
113	/// advances N bytes before continuing.
114	///
115	///
116	/// Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
117	///
118	/// Deflate function accepts BlockWriterType so we can change that in test to test
119	/// just tokenization part.
120	///
121	fn Deflate(comptime container: Container, comptime WriterType: type, comptime BlockWriterType: type) type {
122	return struct {
123	lookup: Lookup = .{},
124	win: SlidingWindow = .{},
125	tokens: Tokens = .{},
126	wrt: WriterType,
127	block_writer: BlockWriterType,
128	level: LevelArgs,
129	hasher: container.Hasher() = .{},
130
131	// Match and literal at the previous position.
132	// Used for lazy match finding in processWindow.
133	prev_match: ?Token = null,
134	prev_literal: ?u8 = null,
135
136	const Self = @This();
137
138	pub fn init(wrt: WriterType, options: Options) !Self {
139	const self = Self{
140	.wrt = wrt,
141	.block_writer = BlockWriterType.init(wrt),
142	.level = LevelArgs.get(options.level),
143	};
144	try container.writeHeader(self.wrt);
145	return self;
146	}
147
148	const FlushOption = enum { none, flush, final };
149
150	// Process data in window and create tokens. If token buffer is full
151	// flush tokens to the token writer. In the case of `flush` or `final`
152	// option it will process all data from the window. In the `none` case
153	// it will preserve some data for the next match.
154	fn tokenize(self: *Self, flush_opt: FlushOption) !void {
155	// flush - process all data from window
156	const should_flush = (flush_opt != .none);
157
158	// While there is data in active lookahead buffer.
159	while (self.win.activeLookahead(should_flush)) \|lh\| {
160	var step: u16 = 1; // 1 in the case of literal, match length otherwise
161	const pos: u16 = self.win.pos();
162	const literal = lh[0]; // literal at current position
163	const min_len: u16 = if (self.prev_match) \|m\| m.length() else 0;
164
165	// Try to find match at least min_len long.
166	if (self.findMatch(pos, lh, min_len)) \|match\| {
167	// Found better match than previous.
168	try self.addPrevLiteral();
169
170	// Is found match length good enough?
171	if (match.length() >= self.level.lazy) {
172	// Don't try to lazy find better match, use this.
173	step = try self.addMatch(match);
174	} else {
175	// Store this match.
176	self.prev_literal = literal;
177	self.prev_match = match;
178	}
179	} else {
180	// There is no better match at current pos then it was previous.
181	// Write previous match or literal.
182	if (self.prev_match) \|m\| {
183	// Write match from previous position.
184	step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
185	} else {
186	// No match at previous postition.
187	// Write previous literal if any, and remember this literal.
188	try self.addPrevLiteral();
189	self.prev_literal = literal;
190	}
191	}
192	// Advance window and add hashes.
193	self.windowAdvance(step, lh, pos);
194	}
195
196	if (should_flush) {
197	// In the case of flushing, last few lookahead buffers were smaller then min match len.
198	// So only last literal can be unwritten.
199	assert(self.prev_match == null);
200	try self.addPrevLiteral();
201	self.prev_literal = null;
202
203	try self.flushTokens(flush_opt);
204	}
205	}
206
207	fn windowAdvance(self: *Self, step: u16, lh: []const u8, pos: u16) void {
208	// current position is already added in findMatch
209	self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
210	self.win.advance(step);
211	}
212
213	// Add previous literal (if any) to the tokens list.
214	fn addPrevLiteral(self: *Self) !void {
215	if (self.prev_literal) \|l\| try self.addToken(Token.initLiteral(l));
216	}
217
218	// Add match to the tokens list, reset prev pointers.
219	// Returns length of the added match.
220	fn addMatch(self: *Self, m: Token) !u16 {
221	try self.addToken(m);
222	self.prev_literal = null;
223	self.prev_match = null;
224	return m.length();
225	}
226
227	fn addToken(self: *Self, token: Token) !void {
228	self.tokens.add(token);
229	if (self.tokens.full()) try self.flushTokens(.none);
230	}
231
232	// Finds largest match in the history window with the data at current pos.
233	fn findMatch(self: *Self, pos: u16, lh: []const u8, min_len: u16) ?Token {
234	var len: u16 = min_len;
235	// Previous location with the same hash (same 4 bytes).
236	var prev_pos = self.lookup.add(lh, pos);
237	// Last found match.
238	var match: ?Token = null;
239
240	// How much back-references to try, performance knob.
241	var chain: usize = self.level.chain;
242	if (len >= self.level.good) {
243	// If we've got a match that's good enough, only look in 1/4 the chain.
244	chain >>= 2;
245	}
246
247	// Hot path loop!
248	while (prev_pos > 0 and chain > 0) : (chain -= 1) {
249	const distance = pos - prev_pos;
250	if (distance > consts.match.max_distance)
251	break;
252
253	const new_len = self.win.match(prev_pos, pos, len);
254	if (new_len > len) {
255	match = Token.initMatch(@intCast(distance), new_len);
256	if (new_len >= self.level.nice) {
257	// The match is good enough that we don't try to find a better one.
258	return match;
259	}
260	len = new_len;
261	}
262	prev_pos = self.lookup.prev(prev_pos);
263	}
264
265	return match;
266	}
267
268	fn flushTokens(self: *Self, flush_opt: FlushOption) !void {
269	// Pass tokens to the token writer
270	try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
271	// Stored block ensures byte aligment.
272	// It has 3 bits (final, block_type) and then padding until byte boundary.
273	// After that everyting is aligned to the boundary in the stored block.
274	// Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
275	// Last 4 bytes are byte aligned.
276	if (flush_opt == .flush) {
277	try self.block_writer.storedBlock("", false);
278	}
279	if (flush_opt != .none) {
280	// Safe to call only when byte aligned or it is OK to add
281	// padding bits (on last byte of the final block).
282	try self.block_writer.flush();
283	}
284	// Reset internal tokens store.
285	self.tokens.reset();
286	// Notify win that tokens are flushed.
287	self.win.flush();
288	}
289
290	// Slide win and if needed lookup tables.
291	fn slide(self: *Self) void {
292	const n = self.win.slide();
293	self.lookup.slide(n);
294	}
295
296	/// Compresses as much data as possible, stops when the reader becomes
297	/// empty. It will introduce some output latency (reading input without
298	/// producing all output) because some data are still in internal
299	/// buffers.
300	///
301	/// It is up to the caller to call flush (if needed) or finish (required)
302	/// when is need to output any pending data or complete stream.
303	///
304	pub fn compress(self: *Self, reader: anytype) !void {
305	while (true) {
306	// Fill window from reader
307	const buf = self.win.writable();
308	if (buf.len == 0) {
309	try self.tokenize(.none);
310	self.slide();
311	continue;
312	}
313	const n = try reader.readAll(buf);
314	self.hasher.update(buf[0..n]);
315	self.win.written(n);
316	// Process window
317	try self.tokenize(.none);
318	// Exit when no more data in reader
319	if (n < buf.len) break;
320	}
321	}
322
323	/// Flushes internal buffers to the output writer. Outputs empty stored
324	/// block to sync bit stream to the byte boundary, so that the
325	/// decompressor can get all input data available so far.
326	///
327	/// It is useful mainly in compressed network protocols, to ensure that
328	/// deflate bit stream can be used as byte stream. May degrade
329	/// compression so it should be used only when necessary.
330	///
331	/// Completes the current deflate block and follows it with an empty
332	/// stored block that is three zero bits plus filler bits to the next
333	/// byte, followed by four bytes (00 00 ff ff).
334	///
335	pub fn flush(self: *Self) !void {
336	try self.tokenize(.flush);
337	}
338
339	/// Completes deflate bit stream by writing any pending data as deflate
340	/// final deflate block. HAS to be called once all data are written to
341	/// the compressor as a signal that next block has to have final bit
342	/// set.
343	///
344	pub fn finish(self: *Self) !void {
345	try self.tokenize(.final);
346	try container.writeFooter(&self.hasher, self.wrt);
347	}
348
349	/// Use another writer while preserving history. Most probably flush
350	/// should be called on old writer before setting new.
351	pub fn setWriter(self: *Self, new_writer: WriterType) void {
352	self.block_writer.setWriter(new_writer);
353	self.wrt = new_writer;
354	}
355
356	// Writer interface
357
358	pub const Writer = io.Writer(*Self, Error, write);
359	pub const Error = BlockWriterType.Error;
360
361	/// Write `input` of uncompressed data.
362	/// See compress.
363	pub fn write(self: *Self, input: []const u8) !usize {
364	var fbs = io.fixedBufferStream(input);
365	try self.compress(fbs.reader());
366	return input.len;
367	}
368
369	pub fn writer(self: *Self) Writer {
370	return .{ .context = self };
371	}
372	};
373	}
374
375	// Tokens store
376	const Tokens = struct {
377	list: [consts.deflate.tokens]Token = undefined,
378	pos: usize = 0,
379
380	fn add(self: *Tokens, t: Token) void {
381	self.list[self.pos] = t;
382	self.pos += 1;
383	}
384
385	fn full(self: *Tokens) bool {
386	return self.pos == self.list.len;
387	}
388
389	fn reset(self: *Tokens) void {
390	self.pos = 0;
391	}
392
393	fn tokens(self: *Tokens) []const Token {
394	return self.list[0..self.pos];
395	}
396	};
397
398	/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
399	/// only performs Huffman entropy encoding. Results in faster compression, much
400	/// less memory requirements during compression but bigger compressed sizes.
401	pub const huffman = struct {
402	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
403	var c = try huffman.compressor(container, writer);
404	try c.compress(reader);
405	try c.finish();
406	}
407
408	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
409	return SimpleCompressor(.huffman, container, WriterType);
410	}
411
412	pub fn compressor(comptime container: Container, writer: anytype) !huffman.Compressor(container, @TypeOf(writer)) {
413	return try huffman.Compressor(container, @TypeOf(writer)).init(writer);
414	}
415	};
416
417	/// Creates store blocks only. Data are not compressed only packed into deflate
418	/// store blocks. That adds 9 bytes of header for each block. Max stored block
419	/// size is 64K. Block is emitted when flush is called on on finish.
420	pub const store = struct {
421	pub fn compress(comptime container: Container, reader: anytype, writer: anytype) !void {
422	var c = try store.compressor(container, writer);
423	try c.compress(reader);
424	try c.finish();
425	}
426
427	pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
428	return SimpleCompressor(.store, container, WriterType);
429	}
430
431	pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
432	return try store.Compressor(container, @TypeOf(writer)).init(writer);
433	}
434	};
435
436	const SimpleCompressorKind = enum {
437	huffman,
438	store,
439	};
440
441	fn simpleCompressor(
442	comptime kind: SimpleCompressorKind,
443	comptime container: Container,
444	writer: anytype,
445	) !SimpleCompressor(kind, container, @TypeOf(writer)) {
446	return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
447	}
448
449	fn SimpleCompressor(
450	comptime kind: SimpleCompressorKind,
451	comptime container: Container,
452	comptime WriterType: type,
453	) type {
454	const BlockWriterType = BlockWriter(WriterType);
455	return struct {
456	buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
457	wp: usize = 0,
458
459	wrt: WriterType,
460	block_writer: BlockWriterType,
461	hasher: container.Hasher() = .{},
462
463	const Self = @This();
464
465	pub fn init(wrt: WriterType) !Self {
466	const self = Self{
467	.wrt = wrt,
468	.block_writer = BlockWriterType.init(wrt),
469	};
470	try container.writeHeader(self.wrt);
471	return self;
472	}
473
474	pub fn flush(self: *Self) !void {
475	try self.flushBuffer(false);
476	try self.block_writer.storedBlock("", false);
477	try self.block_writer.flush();
478	}
479
480	pub fn finish(self: *Self) !void {
481	try self.flushBuffer(true);
482	try self.block_writer.flush();
483	try container.writeFooter(&self.hasher, self.wrt);
484	}
485
486	fn flushBuffer(self: *Self, final: bool) !void {
487	const buf = self.buffer[0..self.wp];
488	switch (kind) {
489	.huffman => try self.block_writer.huffmanBlock(buf, final),
490	.store => try self.block_writer.storedBlock(buf, final),
491	}
492	self.wp = 0;
493	}
494
495	// Writes all data from the input reader of uncompressed data.
496	// It is up to the caller to call flush or finish if there is need to
497	// output compressed blocks.
498	pub fn compress(self: *Self, reader: anytype) !void {
499	while (true) {
500	// read from rdr into buffer
501	const buf = self.buffer[self.wp..];
502	if (buf.len == 0) {
503	try self.flushBuffer(false);
504	continue;
505	}
506	const n = try reader.readAll(buf);
507	self.hasher.update(buf[0..n]);
508	self.wp += n;
509	if (n < buf.len) break; // no more data in reader
510	}
511	}
512
513	// Writer interface
514
515	pub const Writer = io.Writer(*Self, Error, write);
516	pub const Error = BlockWriterType.Error;
517
518	// Write `input` of uncompressed data.
519	pub fn write(self: *Self, input: []const u8) !usize {
520	var fbs = io.fixedBufferStream(input);
521	try self.compress(fbs.reader());
522	return input.len;
523	}
524
525	pub fn writer(self: *Self) Writer {
526	return .{ .context = self };
527	}
528	};
529	}
530
531	const builtin = @import("builtin");
532
533	test "flate.Deflate tokenization" {
534	if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest;
535
536	const L = Token.initLiteral;
537	const M = Token.initMatch;
538
539	const cases = [_]struct {
540	data: []const u8,
541	tokens: []const Token,
542	}{
543	.{
544	.data = "Blah blah blah blah blah!",
545	.tokens = &[_]Token{ L('B'), L('l'), L('a'), L('h'), L(' '), L('b'), M(5, 18), L('!') },
546	},
547	.{
548	.data = "ABCDEABCD ABCDEABCD",
549	.tokens = &[_]Token{
550	L('A'), L('B'), L('C'), L('D'), L('E'), L('A'), L('B'), L('C'), L('D'), L(' '),
551	L('A'), M(10, 8),
552	},
553	},
554	};
555
556	for (cases) \|c\| {
557	inline for (Container.list) \|container\| { // for each wrapping
558
559	var cw = io.countingWriter(io.null_writer);
560	const cww = cw.writer();
561	var df = try Deflate(container, @TypeOf(cww), TestTokenWriter).init(cww, .{});
562
563	_ = try df.write(c.data);
564	try df.flush();
565
566	// df.token_writer.show();
567	try expect(df.block_writer.pos == c.tokens.len); // number of tokens written
568	try testing.expectEqualSlices(Token, df.block_writer.get(), c.tokens); // tokens match
569
570	try testing.expectEqual(container.headerSize(), cw.bytes_written);
571	try df.finish();
572	try testing.expectEqual(container.size(), cw.bytes_written);
573	}
574	}
575	}
576
577	// Tests that tokens writen are equal to expected token list.
578	const TestTokenWriter = struct {
579	const Self = @This();
580
581	pos: usize = 0,
582	actual: [128]Token = undefined,
583
584	pub fn init(_: anytype) Self {
585	return .{};
586	}
587	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
588	for (tokens) \|t\| {
589	self.actual[self.pos] = t;
590	self.pos += 1;
591	}
592	}
593
594	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
595
596	pub fn get(self: *Self) []Token {
597	return self.actual[0..self.pos];
598	}
599
600	pub fn show(self: *Self) void {
601	print("\n", .{});
602	for (self.get()) \|t\| {
603	t.show();
604	}
605	}
606
607	pub fn flush(_: *Self) !void {}
608	};
609
610	test "flate deflate file tokenization" {
611	if (builtin.target.cpu.arch == .wasm32) return error.SkipZigTest;
612
613	const levels = [_]Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 };
614	const cases = [_]struct {
615	data: []const u8, // uncompressed content
616	// expected number of tokens producet in deflate tokenization
617	tokens_count: [levels.len]usize = .{0} ** levels.len,
618	}{
619	.{
620	.data = @embedFile("testdata/rfc1951.txt"),
621	.tokens_count = .{ 7675, 7672, 7599, 7594, 7598, 7599 },
622	},
623
624	.{
625	.data = @embedFile("testdata/block_writer/huffman-null-max.input"),
626	.tokens_count = .{ 257, 257, 257, 257, 257, 257 },
627	},
628	.{
629	.data = @embedFile("testdata/block_writer/huffman-pi.input"),
630	.tokens_count = .{ 2570, 2564, 2564, 2564, 2564, 2564 },
631	},
632	.{
633	.data = @embedFile("testdata/block_writer/huffman-text.input"),
634	.tokens_count = .{ 235, 234, 234, 234, 234, 234 },
635	},
636	.{
637	.data = @embedFile("testdata/fuzz/roundtrip1.input"),
638	.tokens_count = .{ 333, 331, 331, 331, 331, 331 },
639	},
640	.{
641	.data = @embedFile("testdata/fuzz/roundtrip2.input"),
642	.tokens_count = .{ 334, 334, 334, 334, 334, 334 },
643	},
644	};
645
646	for (cases) \|case\| { // for each case
647	const data = case.data;
648
649	for (levels, 0..) \|level, i\| { // for each compression level
650	var original = io.fixedBufferStream(data);
651
652	// buffer for decompressed data
653	var al = std.ArrayList(u8).init(testing.allocator);
654	defer al.deinit();
655	const writer = al.writer();
656
657	// create compressor
658	const WriterType = @TypeOf(writer);
659	const TokenWriter = TokenDecoder(@TypeOf(writer));
660	var cmp = try Deflate(.raw, WriterType, TokenWriter).init(writer, .{ .level = level });
661
662	// Stream uncompressed `orignal` data to the compressor. It will
663	// produce tokens list and pass that list to the TokenDecoder. This
664	// TokenDecoder uses CircularBuffer from inflate to convert list of
665	// tokens back to the uncompressed stream.
666	try cmp.compress(original.reader());
667	try cmp.flush();
668	const expected_count = case.tokens_count[i];
669	const actual = cmp.block_writer.tokens_count;
670	if (expected_count == 0) {
671	print("actual token count {d}\n", .{actual});
672	} else {
673	try testing.expectEqual(expected_count, actual);
674	}
675
676	try testing.expectEqual(data.len, al.items.len);
677	try testing.expectEqualSlices(u8, data, al.items);
678	}
679	}
680	}
681
682	fn TokenDecoder(comptime WriterType: type) type {
683	return struct {
684	const CircularBuffer = @import("CircularBuffer.zig");
685	hist: CircularBuffer = .{},
686	wrt: WriterType,
687	tokens_count: usize = 0,
688
689	const Self = @This();
690
691	pub fn init(wrt: WriterType) Self {
692	return .{ .wrt = wrt };
693	}
694
695	pub fn write(self: *Self, tokens: []const Token, _: bool, _: ?[]const u8) !void {
696	self.tokens_count += tokens.len;
697	for (tokens) \|t\| {
698	switch (t.kind) {
699	.literal => self.hist.write(t.literal()),
700	.match => try self.hist.writeMatch(t.length(), t.distance()),
701	}
702	if (self.hist.free() < 285) try self.flushWin();
703	}
704	try self.flushWin();
705	}
706
707	pub fn storedBlock(_: *Self, _: []const u8, _: bool) !void {}
708
709	fn flushWin(self: *Self) !void {
710	while (true) {
711	const buf = self.hist.read();
712	if (buf.len == 0) break;
713	try self.wrt.writeAll(buf);
714	}
715	}
716
717	pub fn flush(_: *Self) !void {}
718	};
719	}
720
721	test "flate.Deflate store simple compressor" {
722	const data = "Hello world!";
723	const expected = [_]u8{
724	0x1, // block type 0, final bit set
725	0xc, 0x0, // len = 12
726	0xf3, 0xff, // ~len
727	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
728	//0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
729	};
730
731	var fbs = std.io.fixedBufferStream(data);
732	var al = std.ArrayList(u8).init(testing.allocator);
733	defer al.deinit();
734
735	var cmp = try store.compressor(.raw, al.writer());
736	try cmp.compress(fbs.reader());
737	try cmp.finish();
738	try testing.expectEqualSlices(u8, &expected, al.items);
739
740	fbs.reset();
741	try al.resize(0);
742
743	// huffman only compresoor will also emit store block for this small sample
744	var hc = try huffman.compressor(.raw, al.writer());
745	try hc.compress(fbs.reader());
746	try hc.finish();
747	try testing.expectEqualSlices(u8, &expected, al.items);
748	}

lib/std/compress/flate/huffman_decoder.zig created+308

...	@@ -0,0 +1,308 @@
1	const std = @import("std");
2	const testing = std.testing;
3
4	pub const Symbol = packed struct {
5	pub const Kind = enum(u2) {
6	literal,
7	end_of_block,
8	match,
9	};
10
11	symbol: u8 = 0, // symbol from alphabet
12	code_bits: u4 = 0, // number of bits in code 0-15
13	kind: Kind = .literal,
14
15	code: u16 = 0, // huffman code of the symbol
16	next: u16 = 0, // pointer to the next symbol in linked list
17	// it is safe to use 0 as null pointer, when sorted 0 has shortest code and fits into lookup
18
19	// Sorting less than function.
20	pub fn asc(_: void, a: Symbol, b: Symbol) bool {
21	if (a.code_bits == b.code_bits) {
22	if (a.kind == b.kind) {
23	return a.symbol < b.symbol;
24	}
25	return @intFromEnum(a.kind) < @intFromEnum(b.kind);
26	}
27	return a.code_bits < b.code_bits;
28	}
29	};
30
31	pub const LiteralDecoder = HuffmanDecoder(286, 15, 9);
32	pub const DistanceDecoder = HuffmanDecoder(30, 15, 9);
33	pub const CodegenDecoder = HuffmanDecoder(19, 7, 7);
34
35	pub const Error = error{
36	InvalidCode,
37	OversubscribedHuffmanTree,
38	IncompleteHuffmanTree,
39	MissingEndOfBlockCode,
40	};
41
42	/// Creates huffman tree codes from list of code lengths (in `build`).
43	///
44	/// `find` then finds symbol for code bits. Code can be any length between 1 and
45	/// 15 bits. When calling `find` we don't know how many bits will be used to
46	/// find symbol. When symbol is returned it has code_bits field which defines
47	/// how much we should advance in bit stream.
48	///
49	/// Lookup table is used to map 15 bit int to symbol. Same symbol is written
50	/// many times in this table; 32K places for 286 (at most) symbols.
51	/// Small lookup table is optimization for faster search.
52	/// It is variation of the algorithm explained in [zlib](https://github.com/madler/zlib/blob/643e17b7498d12ab8d15565662880579692f769d/doc/algorithm.txt#L92)
53	/// with difference that we here use statically allocated arrays.
54	///
55	fn HuffmanDecoder(
56	comptime alphabet_size: u16,
57	comptime max_code_bits: u4,
58	comptime lookup_bits: u4,
59	) type {
60	const lookup_shift = max_code_bits - lookup_bits;
61
62	return struct {
63	// all symbols in alaphabet, sorted by code_len, symbol
64	symbols: [alphabet_size]Symbol = undefined,
65	// lookup table code -> symbol
66	lookup: [1 << lookup_bits]Symbol = undefined,
67
68	const Self = @This();
69
70	/// Generates symbols and lookup tables from list of code lens for each symbol.
71	pub fn generate(self: *Self, lens: []const u4) !void {
72	try checkCompletnes(lens);
73
74	// init alphabet with code_bits
75	for (self.symbols, 0..) \|_, i\| {
76	const cb: u4 = if (i < lens.len) lens[i] else 0;
77	self.symbols[i] = if (i < 256)
78	.{ .kind = .literal, .symbol = @intCast(i), .code_bits = cb }
79	else if (i == 256)
80	.{ .kind = .end_of_block, .symbol = 0xff, .code_bits = cb }
81	else
82	.{ .kind = .match, .symbol = @intCast(i - 257), .code_bits = cb };
83	}
84	std.sort.heap(Symbol, &self.symbols, {}, Symbol.asc);
85
86	// reset lookup table
87	for (0..self.lookup.len) \|i\| {
88	self.lookup[i] = .{};
89	}
90
91	// assign code to symbols
92	// reference: https://youtu.be/9_YEGLe33NA?list=PLU4IQLU9e_OrY8oASHx0u3IXAL9TOdidm&t=2639
93	var code: u16 = 0;
94	var idx: u16 = 0;
95	for (&self.symbols, 0..) \|*sym, pos\| {
96	//print("sym: {}\n", .{sym});
97	if (sym.code_bits == 0) continue; // skip unused
98	sym.code = code;
99
100	const next_code = code + (@as(u16, 1) << (max_code_bits - sym.code_bits));
101	const next_idx = next_code >> lookup_shift;
102
103	if (next_idx > self.lookup.len or idx >= self.lookup.len) break;
104	if (sym.code_bits <= lookup_bits) {
105	// fill small lookup table
106	for (idx..next_idx) \|j\|
107	self.lookup[j] = sym.*;
108	} else {
109	// insert into linked table starting at root
110	const root = &self.lookup[idx];
111	const root_next = root.next;
112	root.next = @intCast(pos);
113	sym.next = root_next;
114	}
115
116	idx = next_idx;
117	code = next_code;
118	}
119	//print("decoder generate, code: {d}, idx: {d}\n", .{ code, idx });
120	}
121
122	/// Given the list of code lengths check that it represents a canonical
123	/// Huffman code for n symbols.
124	///
125	/// Reference: https://github.com/madler/zlib/blob/5c42a230b7b468dff011f444161c0145b5efae59/contrib/puff/puff.c#L340
126	fn checkCompletnes(lens: []const u4) !void {
127	if (alphabet_size == 286)
128	if (lens[256] == 0) return error.MissingEndOfBlockCode;
129
130	var count = [_]u16{0} ** (@as(usize, max_code_bits) + 1);
131	var max: usize = 0;
132	for (lens) \|n\| {
133	if (n == 0) continue;
134	if (n > max) max = n;
135	count[n] += 1;
136	}
137	if (max == 0) // emtpy tree
138	return;
139
140	// check for an over-subscribed or incomplete set of lengths
141	var left: usize = 1; // one possible code of zero length
142	for (1..count.len) \|len\| {
143	left <<= 1; // one more bit, double codes left
144	if (count[len] > left)
145	return error.OversubscribedHuffmanTree;
146	left -= count[len]; // deduct count from possible codes
147	}
148	if (left > 0) { // left > 0 means incomplete
149	// incomplete code ok only for single length 1 code
150	if (max_code_bits > 7 and max == count[0] + count[1]) return;
151	return error.IncompleteHuffmanTree;
152	}
153	}
154
155	/// Finds symbol for lookup table code.
156	pub fn find(self: *Self, code: u16) !Symbol {
157	// try to find in lookup table
158	const idx = code >> lookup_shift;
159	const sym = self.lookup[idx];
160	if (sym.code_bits != 0) return sym;
161	// if not use linked list of symbols with same prefix
162	return self.findLinked(code, sym.next);
163	}
164
165	inline fn findLinked(self: *Self, code: u16, start: u16) !Symbol {
166	var pos = start;
167	while (pos > 0) {
168	const sym = self.symbols[pos];
169	const shift = max_code_bits - sym.code_bits;
170	// compare code_bits number of upper bits
171	if ((code ^ sym.code) >> shift == 0) return sym;
172	pos = sym.next;
173	}
174	return error.InvalidCode;
175	}
176	};
177	}
178
179	test "flate.HuffmanDecoder init/find" {
180	// example data from: https://youtu.be/SJPvNi4HrWQ?t=8423
181	const code_lens = [_]u4{ 4, 3, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3, 2 };
182	var h: CodegenDecoder = .{};
183	try h.generate(&code_lens);
184
185	const expected = [_]struct {
186	sym: Symbol,
187	code: u16,
188	}{
189	.{
190	.code = 0b00_00000,
191	.sym = .{ .symbol = 3, .code_bits = 2 },
192	},
193	.{
194	.code = 0b01_00000,
195	.sym = .{ .symbol = 18, .code_bits = 2 },
196	},
197	.{
198	.code = 0b100_0000,
199	.sym = .{ .symbol = 1, .code_bits = 3 },
200	},
201	.{
202	.code = 0b101_0000,
203	.sym = .{ .symbol = 4, .code_bits = 3 },
204	},
205	.{
206	.code = 0b110_0000,
207	.sym = .{ .symbol = 17, .code_bits = 3 },
208	},
209	.{
210	.code = 0b1110_000,
211	.sym = .{ .symbol = 0, .code_bits = 4 },
212	},
213	.{
214	.code = 0b1111_000,
215	.sym = .{ .symbol = 16, .code_bits = 4 },
216	},
217	};
218
219	// unused symbols
220	for (0..12) \|i\| {
221	try testing.expectEqual(0, h.symbols[i].code_bits);
222	}
223	// used, from index 12
224	for (expected, 12..) \|e, i\| {
225	try testing.expectEqual(e.sym.symbol, h.symbols[i].symbol);
226	try testing.expectEqual(e.sym.code_bits, h.symbols[i].code_bits);
227	const sym_from_code = try h.find(e.code);
228	try testing.expectEqual(e.sym.symbol, sym_from_code.symbol);
229	}
230
231	// All possible codes for each symbol.
232	// Lookup table has 126 elements, to cover all possible 7 bit codes.
233	for (0b0000_000..0b0100_000) \|c\| // 0..32 (32)
234	try testing.expectEqual(3, (try h.find(@intCast(c))).symbol);
235
236	for (0b0100_000..0b1000_000) \|c\| // 32..64 (32)
237	try testing.expectEqual(18, (try h.find(@intCast(c))).symbol);
238
239	for (0b1000_000..0b1010_000) \|c\| // 64..80 (16)
240	try testing.expectEqual(1, (try h.find(@intCast(c))).symbol);
241
242	for (0b1010_000..0b1100_000) \|c\| // 80..96 (16)
243	try testing.expectEqual(4, (try h.find(@intCast(c))).symbol);
244
245	for (0b1100_000..0b1110_000) \|c\| // 96..112 (16)
246	try testing.expectEqual(17, (try h.find(@intCast(c))).symbol);
247
248	for (0b1110_000..0b1111_000) \|c\| // 112..120 (8)
249	try testing.expectEqual(0, (try h.find(@intCast(c))).symbol);
250
251	for (0b1111_000..0b1_0000_000) \|c\| // 120...128 (8)
252	try testing.expectEqual(16, (try h.find(@intCast(c))).symbol);
253	}
254
255	const print = std.debug.print;
256	const assert = std.debug.assert;
257	const expect = std.testing.expect;
258
259	test "flate.HuffmanDecoder encode/decode literals" {
260	const LiteralEncoder = @import("huffman_encoder.zig").LiteralEncoder;
261
262	for (1..286) \|j\| { // for all different number of codes
263	var enc: LiteralEncoder = .{};
264	// create freqencies
265	var freq = [_]u16{0} ** 286;
266	freq[256] = 1; // ensure we have end of block code
267	for (&freq, 1..) \|*f, i\| {
268	if (i % j == 0)
269	f.* = @intCast(i);
270	}
271
272	// encoder from freqencies
273	enc.generate(&freq, 15);
274
275	// get code_lens from encoder
276	var code_lens = [_]u4{0} ** 286;
277	for (code_lens, 0..) \|_, i\| {
278	code_lens[i] = @intCast(enc.codes[i].len);
279	}
280	// generate decoder from code lens
281	var dec: LiteralDecoder = .{};
282	try dec.generate(&code_lens);
283
284	// expect decoder code to match original encoder code
285	for (dec.symbols) \|s\| {
286	if (s.code_bits == 0) continue;
287	const c_code: u16 = @bitReverse(@as(u15, @intCast(s.code)));
288	const symbol: u16 = switch (s.kind) {
289	.literal => s.symbol,
290	.end_of_block => 256,
291	.match => @as(u16, s.symbol) + 257,
292	};
293
294	const c = enc.codes[symbol];
295	try expect(c.code == c_code);
296	}
297
298	// find each symbol by code
299	for (enc.codes) \|c\| {
300	if (c.len == 0) continue;
301
302	const s_code: u15 = @bitReverse(@as(u15, @intCast(c.code)));
303	const s = try dec.find(s_code);
304	try expect(s.code == s_code);
305	try expect(s.code_bits == c.len);
306	}
307	}
308	}

lib/std/compress/flate/huffman_encoder.zig created+536

...	@@ -0,0 +1,536 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5	const sort = std.sort;
6	const testing = std.testing;
7
8	const consts = @import("consts.zig").huffman;
9
10	const LiteralNode = struct {
11	literal: u16,
12	freq: u16,
13	};
14
15	// Describes the state of the constructed tree for a given depth.
16	const LevelInfo = struct {
17	// Our level. for better printing
18	level: u32,
19
20	// The frequency of the last node at this level
21	last_freq: u32,
22
23	// The frequency of the next character to add to this level
24	next_char_freq: u32,
25
26	// The frequency of the next pair (from level below) to add to this level.
27	// Only valid if the "needed" value of the next lower level is 0.
28	next_pair_freq: u32,
29
30	// The number of chains remaining to generate for this level before moving
31	// up to the next level
32	needed: u32,
33	};
34
35	// hcode is a huffman code with a bit code and bit length.
36	pub const HuffCode = struct {
37	code: u16 = 0,
38	len: u16 = 0,
39
40	// set sets the code and length of an hcode.
41	fn set(self: *HuffCode, code: u16, length: u16) void {
42	self.len = length;
43	self.code = code;
44	}
45	};
46
47	pub fn HuffmanEncoder(comptime size: usize) type {
48	return struct {
49	codes: [size]HuffCode = undefined,
50	// Reusable buffer with the longest possible frequency table.
51	freq_cache: [consts.max_num_frequencies + 1]LiteralNode = undefined,
52	bit_count: [17]u32 = undefined,
53	lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
54	lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
55
56	const Self = @This();
57
58	// Update this Huffman Code object to be the minimum code for the specified frequency count.
59	//
60	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
61	// max_bits The maximum number of bits to use for any literal.
62	pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
63	var list = self.freq_cache[0 .. freq.len + 1];
64	// Number of non-zero literals
65	var count: u32 = 0;
66	// Set list to be the set of all non-zero literals and their frequencies
67	for (freq, 0..) \|f, i\| {
68	if (f != 0) {
69	list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
70	count += 1;
71	} else {
72	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
73	self.codes[i].len = 0;
74	}
75	}
76	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
77
78	list = list[0..count];
79	if (count <= 2) {
80	// Handle the small cases here, because they are awkward for the general case code. With
81	// two or fewer literals, everything has bit length 1.
82	for (list, 0..) \|node, i\| {
83	// "list" is in order of increasing literal value.
84	self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
85	}
86	return;
87	}
88	self.lfs = list;
89	mem.sort(LiteralNode, self.lfs, {}, byFreq);
90
91	// Get the number of literals for each bit count
92	const bit_count = self.bitCounts(list, max_bits);
93	// And do the assignment
94	self.assignEncodingAndSize(bit_count, list);
95	}
96
97	pub fn bitLength(self: *Self, freq: []u16) u32 {
98	var total: u32 = 0;
99	for (freq, 0..) \|f, i\| {
100	if (f != 0) {
101	total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
102	}
103	}
104	return total;
105	}
106
107	// Return the number of literals assigned to each bit size in the Huffman encoding
108	//
109	// This method is only called when list.len >= 3
110	// The cases of 0, 1, and 2 literals are handled by special case code.
111	//
112	// list: An array of the literals with non-zero frequencies
113	// and their associated frequencies. The array is in order of increasing
114	// frequency, and has as its last element a special element with frequency
115	// std.math.maxInt(i32)
116	//
117	// max_bits: The maximum number of bits that should be used to encode any literal.
118	// Must be less than 16.
119	//
120	// Returns an integer array in which array[i] indicates the number of literals
121	// that should be encoded in i bits.
122	fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
123	var max_bits = max_bits_to_use;
124	const n = list.len;
125	const max_bits_limit = 16;
126
127	assert(max_bits < max_bits_limit);
128
129	// The tree can't have greater depth than n - 1, no matter what. This
130	// saves a little bit of work in some small cases
131	max_bits = @min(max_bits, n - 1);
132
133	// Create information about each of the levels.
134	// A bogus "Level 0" whose sole purpose is so that
135	// level1.prev.needed == 0. This makes level1.next_pair_freq
136	// be a legitimate value that never gets chosen.
137	var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
138	// leaf_counts[i] counts the number of literals at the left
139	// of ancestors of the rightmost node at level i.
140	// leaf_counts[i][j] is the number of literals at the left
141	// of the level j ancestor.
142	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
143
144	{
145	var level = @as(u32, 1);
146	while (level <= max_bits) : (level += 1) {
147	// For every level, the first two items are the first two characters.
148	// We initialize the levels as if we had already figured this out.
149	levels[level] = LevelInfo{
150	.level = level,
151	.last_freq = list[1].freq,
152	.next_char_freq = list[2].freq,
153	.next_pair_freq = list[0].freq + list[1].freq,
154	.needed = 0,
155	};
156	leaf_counts[level][level] = 2;
157	if (level == 1) {
158	levels[level].next_pair_freq = math.maxInt(i32);
159	}
160	}
161	}
162
163	// We need a total of 2*n - 2 items at top level and have already generated 2.
164	levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
165
166	{
167	var level = max_bits;
168	while (true) {
169	var l = &levels[level];
170	if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
171	// We've run out of both leafs and pairs.
172	// End all calculations for this level.
173	// To make sure we never come back to this level or any lower level,
174	// set next_pair_freq impossibly large.
175	l.needed = 0;
176	levels[level + 1].next_pair_freq = math.maxInt(i32);
177	level += 1;
178	continue;
179	}
180
181	const prev_freq = l.last_freq;
182	if (l.next_char_freq < l.next_pair_freq) {
183	// The next item on this row is a leaf node.
184	const next = leaf_counts[level][level] + 1;
185	l.last_freq = l.next_char_freq;
186	// Lower leaf_counts are the same of the previous node.
187	leaf_counts[level][level] = next;
188	if (next >= list.len) {
189	l.next_char_freq = maxNode().freq;
190	} else {
191	l.next_char_freq = list[next].freq;
192	}
193	} else {
194	// The next item on this row is a pair from the previous row.
195	// next_pair_freq isn't valid until we generate two
196	// more values in the level below
197	l.last_freq = l.next_pair_freq;
198	// Take leaf counts from the lower level, except counts[level] remains the same.
199	@memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
200	levels[l.level - 1].needed = 2;
201	}
202
203	l.needed -= 1;
204	if (l.needed == 0) {
205	// We've done everything we need to do for this level.
206	// Continue calculating one level up. Fill in next_pair_freq
207	// of that level with the sum of the two nodes we've just calculated on
208	// this level.
209	if (l.level == max_bits) {
210	// All done!
211	break;
212	}
213	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
214	level += 1;
215	} else {
216	// If we stole from below, move down temporarily to replenish it.
217	while (levels[level - 1].needed > 0) {
218	level -= 1;
219	if (level == 0) {
220	break;
221	}
222	}
223	}
224	}
225	}
226
227	// Somethings is wrong if at the end, the top level is null or hasn't used
228	// all of the leaves.
229	assert(leaf_counts[max_bits][max_bits] == n);
230
231	var bit_count = self.bit_count[0 .. max_bits + 1];
232	var bits: u32 = 1;
233	const counts = &leaf_counts[max_bits];
234	{
235	var level = max_bits;
236	while (level > 0) : (level -= 1) {
237	// counts[level] gives the number of literals requiring at least "bits"
238	// bits to encode.
239	bit_count[bits] = counts[level] - counts[level - 1];
240	bits += 1;
241	if (level == 0) {
242	break;
243	}
244	}
245	}
246	return bit_count;
247	}
248
249	// Look at the leaves and assign them a bit count and an encoding as specified
250	// in RFC 1951 3.2.2
251	fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
252	var code = @as(u16, 0);
253	var list = list_arg;
254
255	for (bit_count, 0..) \|bits, n\| {
256	code <<= 1;
257	if (n == 0 or bits == 0) {
258	continue;
259	}
260	// The literals list[list.len-bits] .. list[list.len-bits]
261	// are encoded using "bits" bits, and get the values
262	// code, code + 1, .... The code values are
263	// assigned in literal order (not frequency order).
264	const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
265
266	self.lns = chunk;
267	mem.sort(LiteralNode, self.lns, {}, byLiteral);
268
269	for (chunk) \|node\| {
270	self.codes[node.literal] = HuffCode{
271	.code = bitReverse(u16, code, @as(u5, @intCast(n))),
272	.len = @as(u16, @intCast(n)),
273	};
274	code += 1;
275	}
276	list = list[0 .. list.len - @as(u32, @intCast(bits))];
277	}
278	}
279	};
280	}
281
282	fn maxNode() LiteralNode {
283	return LiteralNode{
284	.literal = math.maxInt(u16),
285	.freq = math.maxInt(u16),
286	};
287	}
288
289	pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
290	return .{};
291	}
292
293	pub const LiteralEncoder = HuffmanEncoder(consts.max_num_frequencies);
294	pub const DistanceEncoder = HuffmanEncoder(consts.distance_code_count);
295	pub const CodegenEncoder = HuffmanEncoder(19);
296
297	// Generates a HuffmanCode corresponding to the fixed literal table
298	pub fn fixedLiteralEncoder() LiteralEncoder {
299	var h: LiteralEncoder = undefined;
300	var ch: u16 = 0;
301
302	while (ch < consts.max_num_frequencies) : (ch += 1) {
303	var bits: u16 = undefined;
304	var size: u16 = undefined;
305	switch (ch) {
306	0...143 => {
307	// size 8, 000110000 .. 10111111
308	bits = ch + 48;
309	size = 8;
310	},
311	144...255 => {
312	// size 9, 110010000 .. 111111111
313	bits = ch + 400 - 144;
314	size = 9;
315	},
316	256...279 => {
317	// size 7, 0000000 .. 0010111
318	bits = ch - 256;
319	size = 7;
320	},
321	else => {
322	// size 8, 11000000 .. 11000111
323	bits = ch + 192 - 280;
324	size = 8;
325	},
326	}
327	h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
328	}
329	return h;
330	}
331
332	pub fn fixedDistanceEncoder() DistanceEncoder {
333	var h: DistanceEncoder = undefined;
334	for (h.codes, 0..) \|_, ch\| {
335	h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
336	}
337	return h;
338	}
339
340	pub fn huffmanDistanceEncoder() DistanceEncoder {
341	var distance_freq = [1]u16{0} ** consts.distance_code_count;
342	distance_freq[0] = 1;
343	// huff_distance is a static distance encoder used for huffman only encoding.
344	// It can be reused since we will not be encoding distance values.
345	var h: DistanceEncoder = .{};
346	h.generate(distance_freq[0..], 15);
347	return h;
348	}
349
350	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
351	_ = context;
352	return a.literal < b.literal;
353	}
354
355	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
356	_ = context;
357	if (a.freq == b.freq) {
358	return a.literal < b.literal;
359	}
360	return a.freq < b.freq;
361	}
362
363	test "flate.HuffmanEncoder generate a Huffman code from an array of frequencies" {
364	var freqs: [19]u16 = [_]u16{
365	8, // 0
366	1, // 1
367	1, // 2
368	2, // 3
369	5, // 4
370	10, // 5
371	9, // 6
372	1, // 7
373	0, // 8
374	0, // 9
375	0, // 10
376	0, // 11
377	0, // 12
378	0, // 13
379	0, // 14
380	0, // 15
381	1, // 16
382	3, // 17
383	5, // 18
384	};
385
386	var enc = huffmanEncoder(19);
387	enc.generate(freqs[0..], 7);
388
389	try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391	try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392	try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393	try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394	try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395	try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396	try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397	try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398	try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399	try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400	try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401	try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402	try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403	try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404	try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405	try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406	try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407	try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408	try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409	try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411	try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412	try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413	try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414	try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415	try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416	try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417	try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418	try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419	try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420	try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421	try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422	}
423
424	test "flate.HuffmanEncoder generate a Huffman code for the fixed literal table specific to Deflate" {
425	const enc = fixedLiteralEncoder();
426	for (enc.codes) \|c\| {
427	switch (c.len) {
428	7 => {
429	const v = @bitReverse(@as(u7, @intCast(c.code)));
430	try testing.expect(v <= 0b0010111);
431	},
432	8 => {
433	const v = @bitReverse(@as(u8, @intCast(c.code)));
434	try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
435	(v >= 0b11000000 and v <= 11000111));
436	},
437	9 => {
438	const v = @bitReverse(@as(u9, @intCast(c.code)));
439	try testing.expect(v >= 0b110010000 and v <= 0b111111111);
440	},
441	else => unreachable,
442	}
443	}
444	}
445
446	test "flate.HuffmanEncoder generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
447	const enc = fixedDistanceEncoder();
448	for (enc.codes) \|c\| {
449	const v = @bitReverse(@as(u5, @intCast(c.code)));
450	try testing.expect(v <= 29);
451	try testing.expect(c.len == 5);
452	}
453	}
454
455	// Reverse bit-by-bit a N-bit code.
456	fn bitReverse(comptime T: type, value: T, n: usize) T {
457	const r = @bitReverse(value);
458	return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).Int.bits - n));
459	}
460
461	test "flate bitReverse" {
462	const ReverseBitsTest = struct {
463	in: u16,
464	bit_count: u5,
465	out: u16,
466	};
467
468	const reverse_bits_tests = [_]ReverseBitsTest{
469	.{ .in = 1, .bit_count = 1, .out = 1 },
470	.{ .in = 1, .bit_count = 2, .out = 2 },
471	.{ .in = 1, .bit_count = 3, .out = 4 },
472	.{ .in = 1, .bit_count = 4, .out = 8 },
473	.{ .in = 1, .bit_count = 5, .out = 16 },
474	.{ .in = 17, .bit_count = 5, .out = 17 },
475	.{ .in = 257, .bit_count = 9, .out = 257 },
476	.{ .in = 29, .bit_count = 5, .out = 23 },
477	};
478
479	for (reverse_bits_tests) \|h\| {
480	const v = bitReverse(u16, h.in, h.bit_count);
481	try std.testing.expectEqual(h.out, v);
482	}
483	}
484
485	test "flate.HuffmanEncoder fixedLiteralEncoder codes" {
486	var al = std.ArrayList(u8).init(testing.allocator);
487	defer al.deinit();
488	var bw = std.io.bitWriter(.little, al.writer());
489
490	const f = fixedLiteralEncoder();
491	for (f.codes) \|c\| {
492	try bw.writeBits(c.code, c.len);
493	}
494	try testing.expectEqualSlices(u8, &fixed_codes, al.items);
495	}
496
497	pub const fixed_codes = [_]u8{
498	0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
499	0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
500	0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
501	0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
502	0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
503	0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
504	0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
505	0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
506	0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
507	0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
508	0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
509	0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
510	0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
511	0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
512	0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
513	0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
514	0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
515	0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
516	0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
517	0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
518	0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
519	0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
520	0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
521	0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
522	0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
523	0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
524	0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
525	0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
526	0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
527	0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
528	0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
529	0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
530	0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
531	0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
532	0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
533	0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
534	0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
535	0b10100011,
536	};

lib/std/compress/flate/inflate.zig created+529

...	@@ -0,0 +1,529 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const testing = std.testing;
4
5	const hfd = @import("huffman_decoder.zig");
6	const BitReader = @import("bit_reader.zig").BitReader;
7	const CircularBuffer = @import("CircularBuffer.zig");
8	const Container = @import("container.zig").Container;
9	const Token = @import("Token.zig");
10	const codegen_order = @import("consts.zig").huffman.codegen_order;
11
12	/// Decompresses deflate bit stream `reader` and writes uncompressed data to the
13	/// `writer` stream.
14	pub fn decompress(comptime container: Container, reader: anytype, writer: anytype) !void {
15	var d = decompressor(container, reader);
16	try d.decompress(writer);
17	}
18
19	/// Inflate decompressor for the reader type.
20	pub fn decompressor(comptime container: Container, reader: anytype) Inflate(container, @TypeOf(reader)) {
21	return Inflate(container, @TypeOf(reader)).init(reader);
22	}
23
24	/// Inflate decompresses deflate bit stream. Reads compressed data from reader
25	/// provided in init. Decompressed data are stored in internal hist buffer and
26	/// can be accesses iterable `next` or reader interface.
27	///
28	/// Container defines header/footer wrapper around deflate bit stream. Can be
29	/// gzip or zlib.
30	///
31	/// Deflate bit stream consists of multiple blocks. Block can be one of three types:
32	/// * stored, non compressed, max 64k in size
33	/// * fixed, huffman codes are predefined
34	/// * dynamic, huffman code tables are encoded at the block start
35	///
36	/// `step` function runs decoder until internal `hist` buffer is full. Client
37	/// than needs to read that data in order to proceed with decoding.
38	///
39	/// Allocates 74.5K of internal buffers, most important are:
40	/// * 64K for history (CircularBuffer)
41	/// * ~10K huffman decoders (Literal and DistanceDecoder)
42	///
43	pub fn Inflate(comptime container: Container, comptime ReaderType: type) type {
44	return struct {
45	const BitReaderType = BitReader(ReaderType);
46	const F = BitReaderType.flag;
47
48	bits: BitReaderType = .{},
49	hist: CircularBuffer = .{},
50	// Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.
51	hasher: container.Hasher() = .{},
52
53	// dynamic block huffman code decoders
54	lit_dec: hfd.LiteralDecoder = .{}, // literals
55	dst_dec: hfd.DistanceDecoder = .{}, // distances
56
57	// current read state
58	bfinal: u1 = 0,
59	block_type: u2 = 0b11,
60	state: ReadState = .protocol_header,
61
62	const ReadState = enum {
63	protocol_header,
64	block_header,
65	block,
66	protocol_footer,
67	end,
68	};
69
70	const Self = @This();
71
72	pub const Error = BitReaderType.Error \|\| Container.Error \|\| hfd.Error \|\| error{
73	InvalidCode,
74	InvalidMatch,
75	InvalidBlockType,
76	WrongStoredBlockNlen,
77	InvalidDynamicBlockHeader,
78	};
79
80	pub fn init(rt: ReaderType) Self {
81	return .{ .bits = BitReaderType.init(rt) };
82	}
83
84	fn blockHeader(self: *Self) !void {
85	self.bfinal = try self.bits.read(u1);
86	self.block_type = try self.bits.read(u2);
87	}
88
89	fn storedBlock(self: *Self) !bool {
90	self.bits.alignToByte(); // skip padding until byte boundary
91	// everyting after this is byte aligned in stored block
92	var len = try self.bits.read(u16);
93	const nlen = try self.bits.read(u16);
94	if (len != ~nlen) return error.WrongStoredBlockNlen;
95
96	while (len > 0) {
97	const buf = self.hist.getWritable(len);
98	try self.bits.readAll(buf);
99	len -= @intCast(buf.len);
100	}
101	return true;
102	}
103
104	fn fixedBlock(self: *Self) !bool {
105	while (!self.hist.full()) {
106	const code = try self.bits.readFixedCode();
107	switch (code) {
108	0...255 => self.hist.write(@intCast(code)),
109	256 => return true, // end of block
110	257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
111	else => return error.InvalidCode,
112	}
113	}
114	return false;
115	}
116
117	// Handles fixed block non literal (length) code.
118	// Length code is followed by 5 bits of distance code.
119	fn fixedDistanceCode(self: *Self, code: u8) !void {
120	try self.bits.fill(5 + 5 + 13);
121	const length = try self.decodeLength(code);
122	const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered \| F.reverse));
123	try self.hist.writeMatch(length, distance);
124	}
125
126	inline fn decodeLength(self: *Self, code: u8) !u16 {
127	if (code > 28) return error.InvalidCode;
128	const ml = Token.matchLength(code);
129	return if (ml.extra_bits == 0) // 0 - 5 extra bits
130	ml.base
131	else
132	ml.base + try self.bits.readN(ml.extra_bits, F.buffered);
133	}
134
135	fn decodeDistance(self: *Self, code: u8) !u16 {
136	if (code > 29) return error.InvalidCode;
137	const md = Token.matchDistance(code);
138	return if (md.extra_bits == 0) // 0 - 13 extra bits
139	md.base
140	else
141	md.base + try self.bits.readN(md.extra_bits, F.buffered);
142	}
143
144	fn dynamicBlockHeader(self: *Self) !void {
145	const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
146	const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
147	const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lenths are encoded
148
149	if (hlit > 286 or hdist > 30)
150	return error.InvalidDynamicBlockHeader;
151
152	// lengths for code lengths
153	var cl_lens = [_]u4{0} ** 19;
154	for (0..hclen) \|i\| {
155	cl_lens[codegen_order[i]] = try self.bits.read(u3);
156	}
157	var cl_dec: hfd.CodegenDecoder = .{};
158	try cl_dec.generate(&cl_lens);
159
160	// literal code lengths
161	var lit_lens = [_]u4{0} ** (286);
162	var pos: usize = 0;
163	while (pos < hlit) {
164	const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
165	try self.bits.shift(sym.code_bits);
166	pos += try self.dynamicCodeLength(sym.symbol, &lit_lens, pos);
167	}
168	if (pos > hlit)
169	return error.InvalidDynamicBlockHeader;
170
171	// distance code lenths
172	var dst_lens = [_]u4{0} ** (30);
173	pos = 0;
174	while (pos < hdist) {
175	const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));
176	try self.bits.shift(sym.code_bits);
177	pos += try self.dynamicCodeLength(sym.symbol, &dst_lens, pos);
178	}
179	if (pos > hdist)
180	return error.InvalidDynamicBlockHeader;
181
182	try self.lit_dec.generate(&lit_lens);
183	try self.dst_dec.generate(&dst_lens);
184	}
185
186	// Decode code length symbol to code length. Writes decoded length into
187	// lens slice starting at position pos. Returns number of positions
188	// advanced.
189	fn dynamicCodeLength(self: *Self, code: u16, lens: []u4, pos: usize) !usize {
190	if (pos >= lens.len)
191	return error.InvalidDynamicBlockHeader;
192
193	switch (code) {
194	0...15 => {
195	// Represent code lengths of 0 - 15
196	lens[pos] = @intCast(code);
197	return 1;
198	},
199	16 => {
200	// Copy the previous code length 3 - 6 times.
201	// The next 2 bits indicate repeat length
202	const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
203	if (pos == 0 or pos + n > lens.len)
204	return error.InvalidDynamicBlockHeader;
205	for (0..n) \|i\| {
206	lens[pos + i] = lens[pos + i - 1];
207	}
208	return n;
209	},
210	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
211	17 => return @as(u8, try self.bits.read(u3)) + 3,
212	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
213	18 => return @as(u8, try self.bits.read(u7)) + 11,
214	else => return error.InvalidDynamicBlockHeader,
215	}
216	}
217
218	// In larger archives most blocks are usually dynamic, so decompression
219	// performance depends on this function.
220	fn dynamicBlock(self: *Self) !bool {
221	// Hot path loop!
222	while (!self.hist.full()) {
223	try self.bits.fill(15); // optimization so other bit reads can be buffered (avoiding one `if` in hot path)
224	const sym = try self.decodeSymbol(&self.lit_dec);
225
226	switch (sym.kind) {
227	.literal => self.hist.write(sym.symbol),
228	.match => { // Decode match backreference <length, distance>
229	try self.bits.fill(5 + 15 + 13); // so we can use buffered reads
230	const length = try self.decodeLength(sym.symbol);
231	const dsm = try self.decodeSymbol(&self.dst_dec);
232	const distance = try self.decodeDistance(dsm.symbol);
233	try self.hist.writeMatch(length, distance);
234	},
235	.end_of_block => return true,
236	}
237	}
238	return false;
239	}
240
241	// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
242	// decoder to find symbol for that code. We then know how many bits is
243	// used. Shift bit reader for that much bits, those bits are used. And
244	// return symbol.
245	fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
246	const sym = try decoder.find(try self.bits.peekF(u15, F.buffered \| F.reverse));
247	try self.bits.shift(sym.code_bits);
248	return sym;
249	}
250
251	fn step(self: *Self) !void {
252	switch (self.state) {
253	.protocol_header => {
254	try container.parseHeader(&self.bits);
255	self.state = .block_header;
256	},
257	.block_header => {
258	try self.blockHeader();
259	self.state = .block;
260	if (self.block_type == 2) try self.dynamicBlockHeader();
261	},
262	.block => {
263	const done = switch (self.block_type) {
264	0 => try self.storedBlock(),
265	1 => try self.fixedBlock(),
266	2 => try self.dynamicBlock(),
267	else => return error.InvalidBlockType,
268	};
269	if (done) {
270	self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
271	}
272	},
273	.protocol_footer => {
274	self.bits.alignToByte();
275	try container.parseFooter(&self.hasher, &self.bits);
276	self.state = .end;
277	},
278	.end => {},
279	}
280	}
281
282	/// Replaces the inner reader with new reader.
283	pub fn setReader(self: *Self, new_reader: ReaderType) void {
284	self.bits.forward_reader = new_reader;
285	if (self.state == .end or self.state == .protocol_footer) {
286	self.state = .protocol_header;
287	}
288	}
289
290	// Reads all compressed data from the internal reader and outputs plain
291	// (uncompressed) data to the provided writer.
292	pub fn decompress(self: *Self, writer: anytype) !void {
293	while (try self.next()) \|buf\| {
294	try writer.writeAll(buf);
295	}
296	}
297
298	// Iterator interface
299
300	/// Can be used in iterator like loop without memcpy to another buffer:
301	/// while (try inflate.next()) \|buf\| { ... }
302	pub fn next(self: *Self) Error!?[]const u8 {
303	const out = try self.get(0);
304	if (out.len == 0) return null;
305	return out;
306	}
307
308	/// Returns decompressed data from internal sliding window buffer.
309	/// Returned buffer can be any length between 0 and `limit` bytes. 0
310	/// returned bytes means end of stream reached. With limit=0 returns as
311	/// much data it can. It newer will be more than 65536 bytes, which is
312	/// size of internal buffer.
313	pub fn get(self: *Self, limit: usize) Error![]const u8 {
314	while (true) {
315	const out = self.hist.readAtMost(limit);
316	if (out.len > 0) {
317	self.hasher.update(out);
318	return out;
319	}
320	if (self.state == .end) return out;
321	try self.step();
322	}
323	}
324
325	// Reader interface
326
327	pub const Reader = std.io.Reader(*Self, Error, read);
328
329	/// Returns the number of bytes read. It may be less than buffer.len.
330	/// If the number of bytes read is 0, it means end of stream.
331	/// End of stream is not an error condition.
332	pub fn read(self: *Self, buffer: []u8) Error!usize {
333	const out = try self.get(buffer.len);
334	@memcpy(buffer[0..out.len], out);
335	return out.len;
336	}
337
338	pub fn reader(self: *Self) Reader {
339	return .{ .context = self };
340	}
341	};
342	}
343
344	test "flate.Inflate decompress" {
345	const cases = [_]struct {
346	in: []const u8,
347	out: []const u8,
348	}{
349	// non compressed block (type 0)
350	.{
351	.in = &[_]u8{
352	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
353	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
354	},
355	.out = "Hello world\n",
356	},
357	// fixed code block (type 1)
358	.{
359	.in = &[_]u8{
360	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
361	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
362	},
363	.out = "Hello world\n",
364	},
365	// dynamic block (type 2)
366	.{
367	.in = &[_]u8{
368	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
369	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
370	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
371	},
372	.out = "ABCDEABCD ABCDEABCD",
373	},
374	};
375	for (cases) \|c\| {
376	var fb = std.io.fixedBufferStream(c.in);
377	var al = std.ArrayList(u8).init(testing.allocator);
378	defer al.deinit();
379
380	try decompress(.raw, fb.reader(), al.writer());
381	try testing.expectEqualStrings(c.out, al.items);
382	}
383	}
384
385	test "flate.Inflate gzip decompress" {
386	const cases = [_]struct {
387	in: []const u8,
388	out: []const u8,
389	}{
390	// non compressed block (type 0)
391	.{
392	.in = &[_]u8{
393	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
394	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
395	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
396	0xd5, 0xe0, 0x39, 0xb7, // gzip footer: checksum
397	0x0c, 0x00, 0x00, 0x00, // gzip footer: size
398	},
399	.out = "Hello world\n",
400	},
401	// fixed code block (type 1)
402	.{
403	.in = &[_]u8{
404	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x03, // gzip header (10 bytes)
405	0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
406	0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
407	0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
408	},
409	.out = "Hello world\n",
410	},
411	// dynamic block (type 2)
412	.{
413	.in = &[_]u8{
414	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, // gzip header (10 bytes)
415	0x3d, 0xc6, 0x39, 0x11, 0x00, 0x00, 0x0c, 0x02, // deflate data block type 2
416	0x30, 0x2b, 0xb5, 0x52, 0x1e, 0xff, 0x96, 0x38,
417	0x16, 0x96, 0x5c, 0x1e, 0x94, 0xcb, 0x6d, 0x01,
418	0x17, 0x1c, 0x39, 0xb4, 0x13, 0x00, 0x00, 0x00, // gzip footer (chksum, len)
419	},
420	.out = "ABCDEABCD ABCDEABCD",
421	},
422	// gzip header with name
423	.{
424	.in = &[_]u8{
425	0x1f, 0x8b, 0x08, 0x08, 0xe5, 0x70, 0xb1, 0x65, 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e,
426	0x74, 0x78, 0x74, 0x00, 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1,
427	0x02, 0x00, 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00,
428	},
429	.out = "Hello world\n",
430	},
431	};
432	for (cases) \|c\| {
433	var fb = std.io.fixedBufferStream(c.in);
434	var al = std.ArrayList(u8).init(testing.allocator);
435	defer al.deinit();
436
437	try decompress(.gzip, fb.reader(), al.writer());
438	try testing.expectEqualStrings(c.out, al.items);
439	}
440	}
441
442	test "flate.Inflate zlib decompress" {
443	const cases = [_]struct {
444	in: []const u8,
445	out: []const u8,
446	}{
447	// non compressed block (type 0)
448	.{
449	.in = &[_]u8{
450	0x78, 0b10_0_11100, // zlib header (2 bytes)
451	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
452	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
453	0x1c, 0xf2, 0x04, 0x47, // zlib footer: checksum
454	},
455	.out = "Hello world\n",
456	},
457	};
458	for (cases) \|c\| {
459	var fb = std.io.fixedBufferStream(c.in);
460	var al = std.ArrayList(u8).init(testing.allocator);
461	defer al.deinit();
462
463	try decompress(.zlib, fb.reader(), al.writer());
464	try testing.expectEqualStrings(c.out, al.items);
465	}
466	}
467
468	test "flate.Inflate fuzzing tests" {
469	const cases = [_]struct {
470	input: []const u8,
471	out: []const u8 = "",
472	err: ?anyerror = null,
473	}{
474	.{ .input = "deflate-stream", .out = @embedFile("testdata/fuzz/deflate-stream.expect") }, // 0
475	.{ .input = "empty-distance-alphabet01" },
476	.{ .input = "empty-distance-alphabet02" },
477	.{ .input = "end-of-stream", .err = error.EndOfStream },
478	.{ .input = "invalid-distance", .err = error.InvalidMatch },
479	.{ .input = "invalid-tree01", .err = error.IncompleteHuffmanTree }, // 5
480	.{ .input = "invalid-tree02", .err = error.IncompleteHuffmanTree },
481	.{ .input = "invalid-tree03", .err = error.IncompleteHuffmanTree },
482	.{ .input = "lengths-overflow", .err = error.InvalidDynamicBlockHeader },
483	.{ .input = "out-of-codes", .err = error.InvalidCode },
484	.{ .input = "puff01", .err = error.WrongStoredBlockNlen }, // 10
485	.{ .input = "puff02", .err = error.EndOfStream },
486	.{ .input = "puff03", .out = &[_]u8{0xa} },
487	.{ .input = "puff04", .err = error.InvalidCode },
488	.{ .input = "puff05", .err = error.EndOfStream },
489	.{ .input = "puff06", .err = error.EndOfStream },
490	.{ .input = "puff08", .err = error.InvalidCode },
491	.{ .input = "puff09", .out = "P" },
492	.{ .input = "puff10", .err = error.InvalidCode },
493	.{ .input = "puff11", .err = error.InvalidMatch },
494	.{ .input = "puff12", .err = error.InvalidDynamicBlockHeader }, // 20
495	.{ .input = "puff13", .err = error.IncompleteHuffmanTree },
496	.{ .input = "puff14", .err = error.EndOfStream },
497	.{ .input = "puff15", .err = error.IncompleteHuffmanTree },
498	.{ .input = "puff16", .err = error.InvalidDynamicBlockHeader },
499	.{ .input = "puff17", .err = error.InvalidDynamicBlockHeader }, // 25
500	.{ .input = "fuzz1", .err = error.InvalidDynamicBlockHeader },
501	.{ .input = "fuzz2", .err = error.InvalidDynamicBlockHeader },
502	.{ .input = "fuzz3", .err = error.InvalidMatch },
503	.{ .input = "fuzz4", .err = error.OversubscribedHuffmanTree },
504	.{ .input = "puff18", .err = error.OversubscribedHuffmanTree }, // 30
505	.{ .input = "puff19", .err = error.OversubscribedHuffmanTree },
506	.{ .input = "puff20", .err = error.OversubscribedHuffmanTree },
507	.{ .input = "puff21", .err = error.OversubscribedHuffmanTree },
508	.{ .input = "puff22", .err = error.OversubscribedHuffmanTree },
509	.{ .input = "puff23", .err = error.InvalidDynamicBlockHeader }, // 35
510	.{ .input = "puff24", .err = error.InvalidDynamicBlockHeader },
511	.{ .input = "puff25", .err = error.OversubscribedHuffmanTree },
512	.{ .input = "puff26", .err = error.InvalidDynamicBlockHeader },
513	.{ .input = "puff27", .err = error.InvalidDynamicBlockHeader },
514	};
515
516	inline for (cases, 0..) \|c, case_no\| {
517	var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
518	var out = std.ArrayList(u8).init(testing.allocator);
519	defer out.deinit();
520	errdefer std.debug.print("test case failed {}\n", .{case_no});
521
522	if (c.err) \|expected_err\| {
523	try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
524	} else {
525	try decompress(.raw, in.reader(), out.writer());
526	try testing.expectEqualStrings(c.out, out.items);
527	}
528	}
529	}

lib/std/compress/flate/testdata/block_writer.zig created+606

...	@@ -0,0 +1,606 @@
1	const Token = @import("../Token.zig");
2
3	pub const TestCase = struct {
4	tokens: []const Token,
5	input: []const u8 = "", // File name of input data matching the tokens.
6	want: []const u8 = "", // File name of data with the expected output with input available.
7	want_no_input: []const u8 = "", // File name of the expected output when no input is available.
8	};
9
10	pub const testCases = blk: {
11	@setEvalBranchQuota(4096 * 2);
12
13	const L = Token.initLiteral;
14	const M = Token.initMatch;
15	const ml = M(1, 258); // Maximum length token. Used to reduce the size of writeBlockTests
16
17	break :blk &[_]TestCase{
18	TestCase{
19	.input = "huffman-null-max.input",
20	.want = "huffman-null-max.{s}.expect",
21	.want_no_input = "huffman-null-max.{s}.expect-noinput",
22	.tokens = &[_]Token{
23	L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
24	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
25	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
26	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
27	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
28	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
29	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
30	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
31	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
32	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
33	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
34	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
35	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, L(0x0), L(0x0),
36	},
37	},
38	TestCase{
39	.input = "huffman-pi.input",
40	.want = "huffman-pi.{s}.expect",
41	.want_no_input = "huffman-pi.{s}.expect-noinput",
42	.tokens = &[_]Token{
43	L('3'), L('.'), L('1'), L('4'), L('1'), L('5'), L('9'), L('2'),
44	L('6'), L('5'), L('3'), L('5'), L('8'), L('9'), L('7'), L('9'),
45	L('3'), L('2'), L('3'), L('8'), L('4'), L('6'), L('2'), L('6'),
46	L('4'), L('3'), L('3'), L('8'), L('3'), L('2'), L('7'), L('9'),
47	L('5'), L('0'), L('2'), L('8'), L('8'), L('4'), L('1'), L('9'),
48	L('7'), L('1'), L('6'), L('9'), L('3'), L('9'), L('9'), L('3'),
49	L('7'), L('5'), L('1'), L('0'), L('5'), L('8'), L('2'), L('0'),
50	L('9'), L('7'), L('4'), L('9'), L('4'), L('4'), L('5'), L('9'),
51	L('2'), L('3'), L('0'), L('7'), L('8'), L('1'), L('6'), L('4'),
52	L('0'), L('6'), L('2'), L('8'), L('6'), L('2'), L('0'), L('8'),
53	L('9'), L('9'), L('8'), L('6'), L('2'), L('8'), L('0'), L('3'),
54	L('4'), L('8'), L('2'), L('5'), L('3'), L('4'), L('2'), L('1'),
55	L('1'), L('7'), L('0'), L('6'), L('7'), L('9'), L('8'), L('2'),
56	L('1'), L('4'), L('8'), L('0'), L('8'), L('6'), L('5'), L('1'),
57	L('3'), L('2'), L('8'), L('2'), L('3'), L('0'), L('6'), L('6'),
58	L('4'), L('7'), L('0'), L('9'), L('3'), L('8'), L('4'), L('4'),
59	L('6'), L('0'), L('9'), L('5'), L('5'), L('0'), L('5'), L('8'),
60	L('2'), L('2'), L('3'), L('1'), L('7'), L('2'), L('5'), L('3'),
61	L('5'), L('9'), L('4'), L('0'), L('8'), L('1'), L('2'), L('8'),
62	L('4'), L('8'), L('1'), L('1'), L('1'), L('7'), L('4'), M(127, 4),
63	L('4'), L('1'), L('0'), L('2'), L('7'), L('0'), L('1'), L('9'),
64	L('3'), L('8'), L('5'), L('2'), L('1'), L('1'), L('0'), L('5'),
65	L('5'), L('5'), L('9'), L('6'), L('4'), L('4'), L('6'), L('2'),
66	L('2'), L('9'), L('4'), L('8'), L('9'), L('5'), L('4'), L('9'),
67	L('3'), L('0'), L('3'), L('8'), L('1'), M(19, 4), L('2'), L('8'),
68	L('8'), L('1'), L('0'), L('9'), L('7'), L('5'), L('6'), L('6'),
69	L('5'), L('9'), L('3'), L('3'), L('4'), L('4'), L('6'), M(72, 4),
70	L('7'), L('5'), L('6'), L('4'), L('8'), L('2'), L('3'), L('3'),
71	L('7'), L('8'), L('6'), L('7'), L('8'), L('3'), L('1'), L('6'),
72	L('5'), L('2'), L('7'), L('1'), L('2'), L('0'), L('1'), L('9'),
73	L('0'), L('9'), L('1'), L('4'), M(27, 4), L('5'), L('6'), L('6'),
74	L('9'), L('2'), L('3'), L('4'), L('6'), M(179, 4), L('6'), L('1'),
75	L('0'), L('4'), L('5'), L('4'), L('3'), L('2'), L('6'), M(51, 4),
76	L('1'), L('3'), L('3'), L('9'), L('3'), L('6'), L('0'), L('7'),
77	L('2'), L('6'), L('0'), L('2'), L('4'), L('9'), L('1'), L('4'),
78	L('1'), L('2'), L('7'), L('3'), L('7'), L('2'), L('4'), L('5'),
79	L('8'), L('7'), L('0'), L('0'), L('6'), L('6'), L('0'), L('6'),
80	L('3'), L('1'), L('5'), L('5'), L('8'), L('8'), L('1'), L('7'),
81	L('4'), L('8'), L('8'), L('1'), L('5'), L('2'), L('0'), L('9'),
82	L('2'), L('0'), L('9'), L('6'), L('2'), L('8'), L('2'), L('9'),
83	L('2'), L('5'), L('4'), L('0'), L('9'), L('1'), L('7'), L('1'),
84	L('5'), L('3'), L('6'), L('4'), L('3'), L('6'), L('7'), L('8'),
85	L('9'), L('2'), L('5'), L('9'), L('0'), L('3'), L('6'), L('0'),
86	L('0'), L('1'), L('1'), L('3'), L('3'), L('0'), L('5'), L('3'),
87	L('0'), L('5'), L('4'), L('8'), L('8'), L('2'), L('0'), L('4'),
88	L('6'), L('6'), L('5'), L('2'), L('1'), L('3'), L('8'), L('4'),
89	L('1'), L('4'), L('6'), L('9'), L('5'), L('1'), L('9'), L('4'),
90	L('1'), L('5'), L('1'), L('1'), L('6'), L('0'), L('9'), L('4'),
91	L('3'), L('3'), L('0'), L('5'), L('7'), L('2'), L('7'), L('0'),
92	L('3'), L('6'), L('5'), L('7'), L('5'), L('9'), L('5'), L('9'),
93	L('1'), L('9'), L('5'), L('3'), L('0'), L('9'), L('2'), L('1'),
94	L('8'), L('6'), L('1'), L('1'), L('7'), M(234, 4), L('3'), L('2'),
95	M(10, 4), L('9'), L('3'), L('1'), L('0'), L('5'), L('1'), L('1'),
96	L('8'), L('5'), L('4'), L('8'), L('0'), L('7'), M(271, 4), L('3'),
97	L('7'), L('9'), L('9'), L('6'), L('2'), L('7'), L('4'), L('9'),
98	L('5'), L('6'), L('7'), L('3'), L('5'), L('1'), L('8'), L('8'),
99	L('5'), L('7'), L('5'), L('2'), L('7'), L('2'), L('4'), L('8'),
100	L('9'), L('1'), L('2'), L('2'), L('7'), L('9'), L('3'), L('8'),
101	L('1'), L('8'), L('3'), L('0'), L('1'), L('1'), L('9'), L('4'),
102	L('9'), L('1'), L('2'), L('9'), L('8'), L('3'), L('3'), L('6'),
103	L('7'), L('3'), L('3'), L('6'), L('2'), L('4'), L('4'), L('0'),
104	L('6'), L('5'), L('6'), L('6'), L('4'), L('3'), L('0'), L('8'),
105	L('6'), L('0'), L('2'), L('1'), L('3'), L('9'), L('4'), L('9'),
106	L('4'), L('6'), L('3'), L('9'), L('5'), L('2'), L('2'), L('4'),
107	L('7'), L('3'), L('7'), L('1'), L('9'), L('0'), L('7'), L('0'),
108	L('2'), L('1'), L('7'), L('9'), L('8'), M(154, 5), L('7'), L('0'),
109	L('2'), L('7'), L('7'), L('0'), L('5'), L('3'), L('9'), L('2'),
110	L('1'), L('7'), L('1'), L('7'), L('6'), L('2'), L('9'), L('3'),
111	L('1'), L('7'), L('6'), L('7'), L('5'), M(563, 5), L('7'), L('4'),
112	L('8'), L('1'), M(7, 4), L('6'), L('6'), L('9'), L('4'), L('0'),
113	M(488, 4), L('0'), L('0'), L('0'), L('5'), L('6'), L('8'), L('1'),
114	L('2'), L('7'), L('1'), L('4'), L('5'), L('2'), L('6'), L('3'),
115	L('5'), L('6'), L('0'), L('8'), L('2'), L('7'), L('7'), L('8'),
116	L('5'), L('7'), L('7'), L('1'), L('3'), L('4'), L('2'), L('7'),
117	L('5'), L('7'), L('7'), L('8'), L('9'), L('6'), M(298, 4), L('3'),
118	L('6'), L('3'), L('7'), L('1'), L('7'), L('8'), L('7'), L('2'),
119	L('1'), L('4'), L('6'), L('8'), L('4'), L('4'), L('0'), L('9'),
120	L('0'), L('1'), L('2'), L('2'), L('4'), L('9'), L('5'), L('3'),
121	L('4'), L('3'), L('0'), L('1'), L('4'), L('6'), L('5'), L('4'),
122	L('9'), L('5'), L('8'), L('5'), L('3'), L('7'), L('1'), L('0'),
123	L('5'), L('0'), L('7'), L('9'), M(203, 4), L('6'), M(340, 4), L('8'),
124	L('9'), L('2'), L('3'), L('5'), L('4'), M(458, 4), L('9'), L('5'),
125	L('6'), L('1'), L('1'), L('2'), L('1'), L('2'), L('9'), L('0'),
126	L('2'), L('1'), L('9'), L('6'), L('0'), L('8'), L('6'), L('4'),
127	L('0'), L('3'), L('4'), L('4'), L('1'), L('8'), L('1'), L('5'),
128	L('9'), L('8'), L('1'), L('3'), L('6'), L('2'), L('9'), L('7'),
129	L('7'), L('4'), M(117, 4), L('0'), L('9'), L('9'), L('6'), L('0'),
130	L('5'), L('1'), L('8'), L('7'), L('0'), L('7'), L('2'), L('1'),
131	L('1'), L('3'), L('4'), L('9'), M(1, 5), L('8'), L('3'), L('7'),
132	L('2'), L('9'), L('7'), L('8'), L('0'), L('4'), L('9'), L('9'),
133	M(731, 4), L('9'), L('7'), L('3'), L('1'), L('7'), L('3'), L('2'),
134	L('8'), M(395, 4), L('6'), L('3'), L('1'), L('8'), L('5'), M(770, 4),
135	M(745, 4), L('4'), L('5'), L('5'), L('3'), L('4'), L('6'), L('9'),
136	L('0'), L('8'), L('3'), L('0'), L('2'), L('6'), L('4'), L('2'),
137	L('5'), L('2'), L('2'), L('3'), L('0'), M(740, 4), M(616, 4), L('8'),
138	L('5'), L('0'), L('3'), L('5'), L('2'), L('6'), L('1'), L('9'),
139	L('3'), L('1'), L('1'), M(531, 4), L('1'), L('0'), L('1'), L('0'),
140	L('0'), L('0'), L('3'), L('1'), L('3'), L('7'), L('8'), L('3'),
141	L('8'), L('7'), L('5'), L('2'), L('8'), L('8'), L('6'), L('5'),
142	L('8'), L('7'), L('5'), L('3'), L('3'), L('2'), L('0'), L('8'),
143	L('3'), L('8'), L('1'), L('4'), L('2'), L('0'), L('6'), M(321, 4),
144	M(300, 4), L('1'), L('4'), L('7'), L('3'), L('0'), L('3'), L('5'),
145	L('9'), M(815, 5), L('9'), L('0'), L('4'), L('2'), L('8'), L('7'),
146	L('5'), L('5'), L('4'), L('6'), L('8'), L('7'), L('3'), L('1'),
147	L('1'), L('5'), L('9'), L('5'), M(854, 4), L('3'), L('8'), L('8'),
148	L('2'), L('3'), L('5'), L('3'), L('7'), L('8'), L('7'), L('5'),
149	M(896, 5), L('9'), M(315, 4), L('1'), M(329, 4), L('8'), L('0'), L('5'),
150	L('3'), M(395, 4), L('2'), L('2'), L('6'), L('8'), L('0'), L('6'),
151	L('6'), L('1'), L('3'), L('0'), L('0'), L('1'), L('9'), L('2'),
152	L('7'), L('8'), L('7'), L('6'), L('6'), L('1'), L('1'), L('1'),
153	L('9'), L('5'), L('9'), M(568, 4), L('6'), M(293, 5), L('8'), L('9'),
154	L('3'), L('8'), L('0'), L('9'), L('5'), L('2'), L('5'), L('7'),
155	L('2'), L('0'), L('1'), L('0'), L('6'), L('5'), L('4'), L('8'),
156	L('5'), L('8'), L('6'), L('3'), L('2'), L('7'), M(155, 4), L('9'),
157	L('3'), L('6'), L('1'), L('5'), L('3'), M(545, 4), M(349, 5), L('2'),
158	L('3'), L('0'), L('3'), L('0'), L('1'), L('9'), L('5'), L('2'),
159	L('0'), L('3'), L('5'), L('3'), L('0'), L('1'), L('8'), L('5'),
160	L('2'), M(370, 4), M(118, 4), L('3'), L('6'), L('2'), L('2'), L('5'),
161	L('9'), L('9'), L('4'), L('1'), L('3'), M(597, 4), L('4'), L('9'),
162	L('7'), L('2'), L('1'), L('7'), M(223, 4), L('3'), L('4'), L('7'),
163	L('9'), L('1'), L('3'), L('1'), L('5'), L('1'), L('5'), L('5'),
164	L('7'), L('4'), L('8'), L('5'), L('7'), L('2'), L('4'), L('2'),
165	L('4'), L('5'), L('4'), L('1'), L('5'), L('0'), L('6'), L('9'),
166	M(320, 4), L('8'), L('2'), L('9'), L('5'), L('3'), L('3'), L('1'),
167	L('1'), L('6'), L('8'), L('6'), L('1'), L('7'), L('2'), L('7'),
168	L('8'), M(824, 4), L('9'), L('0'), L('7'), L('5'), L('0'), L('9'),
169	M(270, 4), L('7'), L('5'), L('4'), L('6'), L('3'), L('7'), L('4'),
170	L('6'), L('4'), L('9'), L('3'), L('9'), L('3'), L('1'), L('9'),
171	L('2'), L('5'), L('5'), L('0'), L('6'), L('0'), L('4'), L('0'),
172	L('0'), L('9'), M(620, 4), L('1'), L('6'), L('7'), L('1'), L('1'),
173	L('3'), L('9'), L('0'), L('0'), L('9'), L('8'), M(822, 4), L('4'),
174	L('0'), L('1'), L('2'), L('8'), L('5'), L('8'), L('3'), L('6'),
175	L('1'), L('6'), L('0'), L('3'), L('5'), L('6'), L('3'), L('7'),
176	L('0'), L('7'), L('6'), L('6'), L('0'), L('1'), L('0'), L('4'),
177	M(371, 4), L('8'), L('1'), L('9'), L('4'), L('2'), L('9'), M(1055, 5),
178	M(240, 4), M(652, 4), L('7'), L('8'), L('3'), L('7'), L('4'), M(1193, 4),
179	L('8'), L('2'), L('5'), L('5'), L('3'), L('7'), M(522, 5), L('2'),
180	L('6'), L('8'), M(47, 4), L('4'), L('0'), L('4'), L('7'), M(466, 4),
181	L('4'), M(1206, 4), M(910, 4), L('8'), L('4'), M(937, 4), L('6'), M(800, 6),
182	L('3'), L('3'), L('1'), L('3'), L('6'), L('7'), L('7'), L('0'),
183	L('2'), L('8'), L('9'), L('8'), L('9'), L('1'), L('5'), L('2'),
184	M(99, 4), L('5'), L('2'), L('1'), L('6'), L('2'), L('0'), L('5'),
185	L('6'), L('9'), L('6'), M(1042, 4), L('0'), L('5'), L('8'), M(1144, 4),
186	L('5'), M(1177, 4), L('5'), L('1'), L('1'), M(522, 4), L('8'), L('2'),
187	L('4'), L('3'), L('0'), L('0'), L('3'), L('5'), L('5'), L('8'),
188	L('7'), L('6'), L('4'), L('0'), L('2'), L('4'), L('7'), L('4'),
189	L('9'), L('6'), L('4'), L('7'), L('3'), L('2'), L('6'), L('3'),
190	M(1087, 4), L('9'), L('9'), L('2'), M(1100, 4), L('4'), L('2'), L('6'),
191	L('9'), M(710, 6), L('7'), M(471, 4), L('4'), M(1342, 4), M(1054, 4), L('9'),
192	L('3'), L('4'), L('1'), L('7'), M(430, 4), L('1'), L('2'), M(43, 4),
193	L('4'), M(415, 4), L('1'), L('5'), L('0'), L('3'), L('0'), L('2'),
194	L('8'), L('6'), L('1'), L('8'), L('2'), L('9'), L('7'), L('4'),
195	L('5'), L('5'), L('5'), L('7'), L('0'), L('6'), L('7'), L('4'),
196	M(310, 4), L('5'), L('0'), L('5'), L('4'), L('9'), L('4'), L('5'),
197	L('8'), M(454, 4), L('9'), M(82, 4), L('5'), L('6'), M(493, 4), L('7'),
198	L('2'), L('1'), L('0'), L('7'), L('9'), M(346, 4), L('3'), L('0'),
199	M(267, 4), L('3'), L('2'), L('1'), L('1'), L('6'), L('5'), L('3'),
200	L('4'), L('4'), L('9'), L('8'), L('7'), L('2'), L('0'), L('2'),
201	L('7'), M(284, 4), L('0'), L('2'), L('3'), L('6'), L('4'), M(559, 4),
202	L('5'), L('4'), L('9'), L('9'), L('1'), L('1'), L('9'), L('8'),
203	M(1049, 4), L('4'), M(284, 4), L('5'), L('3'), L('5'), L('6'), L('6'),
204	L('3'), L('6'), L('9'), M(1105, 4), L('2'), L('6'), L('5'), M(741, 4),
205	L('7'), L('8'), L('6'), L('2'), L('5'), L('5'), L('1'), M(987, 4),
206	L('1'), L('7'), L('5'), L('7'), L('4'), L('6'), L('7'), L('2'),
207	L('8'), L('9'), L('0'), L('9'), L('7'), L('7'), L('7'), L('7'),
208	M(1108, 5), L('0'), L('0'), L('0'), M(1534, 4), L('7'), L('0'), M(1248, 4),
209	L('6'), M(1002, 4), L('4'), L('9'), L('1'), M(1055, 4), M(664, 4), L('2'),
210	L('1'), L('4'), L('7'), L('7'), L('2'), L('3'), L('5'), L('0'),
211	L('1'), L('4'), L('1'), L('4'), M(1604, 4), L('3'), L('5'), L('6'),
212	M(1200, 4), L('1'), L('6'), L('1'), L('3'), L('6'), L('1'), L('1'),
213	L('5'), L('7'), L('3'), L('5'), L('2'), L('5'), M(1285, 4), L('3'),
214	L('4'), M(92, 4), L('1'), L('8'), M(1148, 4), L('8'), L('4'), M(1512, 4),
215	L('3'), L('3'), L('2'), L('3'), L('9'), L('0'), L('7'), L('3'),
216	L('9'), L('4'), L('1'), L('4'), L('3'), L('3'), L('3'), L('4'),
217	L('5'), L('4'), L('7'), L('7'), L('6'), L('2'), L('4'), M(579, 4),
218	L('2'), L('5'), L('1'), L('8'), L('9'), L('8'), L('3'), L('5'),
219	L('6'), L('9'), L('4'), L('8'), L('5'), L('5'), L('6'), L('2'),
220	L('0'), L('9'), L('9'), L('2'), L('1'), L('9'), L('2'), L('2'),
221	L('2'), L('1'), L('8'), L('4'), L('2'), L('7'), M(575, 4), L('2'),
222	M(187, 4), L('6'), L('8'), L('8'), L('7'), L('6'), L('7'), L('1'),
223	L('7'), L('9'), L('0'), M(86, 4), L('0'), M(263, 5), L('6'), L('6'),
224	M(1000, 4), L('8'), L('8'), L('6'), L('2'), L('7'), L('2'), M(1757, 4),
225	L('1'), L('7'), L('8'), L('6'), L('0'), L('8'), L('5'), L('7'),
226	M(116, 4), L('3'), M(765, 5), L('7'), L('9'), L('7'), L('6'), L('6'),
227	L('8'), L('1'), M(702, 4), L('0'), L('0'), L('9'), L('5'), L('3'),
228	L('8'), L('8'), M(1593, 4), L('3'), M(1702, 4), L('0'), L('6'), L('8'),
229	L('0'), L('0'), L('6'), L('4'), L('2'), L('2'), L('5'), L('1'),
230	L('2'), L('5'), L('2'), M(1404, 4), L('7'), L('3'), L('9'), L('2'),
231	M(664, 4), M(1141, 4), L('4'), M(1716, 5), L('8'), L('6'), L('2'), L('6'),
232	L('9'), L('4'), L('5'), M(486, 4), L('4'), L('1'), L('9'), L('6'),
233	L('5'), L('2'), L('8'), L('5'), L('0'), M(154, 4), M(925, 4), L('1'),
234	L('8'), L('6'), L('3'), M(447, 4), L('4'), M(341, 5), L('2'), L('0'),
235	L('3'), L('9'), M(1420, 4), L('4'), L('5'), M(701, 4), L('2'), L('3'),
236	L('7'), M(1069, 4), L('6'), M(1297, 4), L('5'), L('6'), M(1593, 4), L('7'),
237	L('1'), L('9'), L('1'), L('7'), L('2'), L('8'), M(370, 4), L('7'),
238	L('6'), L('4'), L('6'), L('5'), L('7'), L('5'), L('7'), L('3'),
239	L('9'), M(258, 4), L('3'), L('8'), L('9'), M(1865, 4), L('8'), L('3'),
240	L('2'), L('6'), L('4'), L('5'), L('9'), L('9'), L('5'), L('8'),
241	M(1704, 4), L('0'), L('4'), L('7'), L('8'), M(479, 4), M(809, 4), L('9'),
242	M(46, 4), L('6'), L('4'), L('0'), L('7'), L('8'), L('9'), L('5'),
243	L('1'), M(143, 4), L('6'), L('8'), L('3'), M(304, 4), L('2'), L('5'),
244	L('9'), L('5'), L('7'), L('0'), M(1129, 4), L('8'), L('2'), L('2'),
245	M(713, 4), L('2'), M(1564, 4), L('4'), L('0'), L('7'), L('7'), L('2'),
246	L('6'), L('7'), L('1'), L('9'), L('4'), L('7'), L('8'), M(794, 4),
247	L('8'), L('2'), L('6'), L('0'), L('1'), L('4'), L('7'), L('6'),
248	L('9'), L('9'), L('0'), L('9'), M(1257, 4), L('0'), L('1'), L('3'),
249	L('6'), L('3'), L('9'), L('4'), L('4'), L('3'), M(640, 4), L('3'),
250	L('0'), M(262, 4), L('2'), L('0'), L('3'), L('4'), L('9'), L('6'),
251	L('2'), L('5'), L('2'), L('4'), L('5'), L('1'), L('7'), M(950, 4),
252	L('9'), L('6'), L('5'), L('1'), L('4'), L('3'), L('1'), L('4'),
253	L('2'), L('9'), L('8'), L('0'), L('9'), L('1'), L('9'), L('0'),
254	L('6'), L('5'), L('9'), L('2'), M(643, 4), L('7'), L('2'), L('2'),
255	L('1'), L('6'), L('9'), L('6'), L('4'), L('6'), M(1050, 4), M(123, 4),
256	L('5'), M(1295, 4), L('4'), M(1382, 5), L('8'), M(1370, 4), L('9'), L('7'),
257	M(1404, 4), L('5'), L('4'), M(1182, 4), M(575, 4), L('7'), M(1627, 4), L('8'),
258	L('4'), L('6'), L('8'), L('1'), L('3'), M(141, 4), L('6'), L('8'),
259	L('3'), L('8'), L('6'), L('8'), L('9'), L('4'), L('2'), L('7'),
260	L('7'), L('4'), L('1'), L('5'), L('5'), L('9'), L('9'), L('1'),
261	L('8'), L('5'), M(91, 4), L('2'), L('4'), L('5'), L('9'), L('5'),
262	L('3'), L('9'), L('5'), L('9'), L('4'), L('3'), L('1'), M(1464, 4),
263	L('7'), M(19, 4), L('6'), L('8'), L('0'), L('8'), L('4'), L('5'),
264	M(744, 4), L('7'), L('3'), M(2079, 4), L('9'), L('5'), L('8'), L('4'),
265	L('8'), L('6'), L('5'), L('3'), L('8'), M(1769, 4), L('6'), L('2'),
266	M(243, 4), L('6'), L('0'), L('9'), M(1207, 4), L('6'), L('0'), L('8'),
267	L('0'), L('5'), L('1'), L('2'), L('4'), L('3'), L('8'), L('8'),
268	L('4'), M(315, 4), M(12, 4), L('4'), L('1'), L('3'), M(784, 4), L('7'),
269	L('6'), L('2'), L('7'), L('8'), M(834, 4), L('7'), L('1'), L('5'),
270	M(1436, 4), L('3'), L('5'), L('9'), L('9'), L('7'), L('7'), L('0'),
271	L('0'), L('1'), L('2'), L('9'), M(1139, 4), L('8'), L('9'), L('4'),
272	L('4'), L('1'), M(632, 4), L('6'), L('8'), L('5'), L('5'), M(96, 4),
273	L('4'), L('0'), L('6'), L('3'), M(2279, 4), L('2'), L('0'), L('7'),
274	L('2'), L('2'), M(345, 4), M(516, 5), L('4'), L('8'), L('1'), L('5'),
275	L('8'), M(518, 4), M(511, 4), M(635, 4), M(665, 4), L('3'), L('9'), L('4'),
276	L('5'), L('2'), L('2'), L('6'), L('7'), M(1175, 6), L('8'), M(1419, 4),
277	L('2'), L('1'), M(747, 4), L('2'), M(904, 4), L('5'), L('4'), L('6'),
278	L('6'), L('6'), M(1308, 4), L('2'), L('3'), L('9'), L('8'), L('6'),
279	L('4'), L('5'), L('6'), M(1221, 4), L('1'), L('6'), L('3'), L('5'),
280	M(596, 5), M(2066, 4), L('7'), M(2222, 4), L('9'), L('8'), M(1119, 4), L('9'),
281	L('3'), L('6'), L('3'), L('4'), M(1884, 4), L('7'), L('4'), L('3'),
282	L('2'), L('4'), M(1148, 4), L('1'), L('5'), L('0'), L('7'), L('6'),
283	M(1212, 4), L('7'), L('9'), L('4'), L('5'), L('1'), L('0'), L('9'),
284	M(63, 4), L('0'), L('9'), L('4'), L('0'), M(1703, 4), L('8'), L('8'),
285	L('7'), L('9'), L('7'), L('1'), L('0'), L('8'), L('9'), L('3'),
286	M(2289, 4), L('6'), L('9'), L('1'), L('3'), L('6'), L('8'), L('6'),
287	L('7'), L('2'), M(604, 4), M(511, 4), L('5'), M(1344, 4), M(1129, 4), M(2050, 4),
288	L('1'), L('7'), L('9'), L('2'), L('8'), L('6'), L('8'), M(2253, 4),
289	L('8'), L('7'), L('4'), L('7'), M(1951, 5), L('8'), L('2'), L('4'),
290	M(2427, 4), L('8'), M(604, 4), L('7'), L('1'), L('4'), L('9'), L('0'),
291	L('9'), L('6'), L('7'), L('5'), L('9'), L('8'), M(1776, 4), L('3'),
292	L('6'), L('5'), M(309, 4), L('8'), L('1'), M(93, 4), M(1862, 4), M(2359, 4),
293	L('6'), L('8'), L('2'), L('9'), M(1407, 4), L('8'), L('7'), L('2'),
294	L('2'), L('6'), L('5'), L('8'), L('8'), L('0'), M(1554, 4), L('5'),
295	M(586, 4), L('4'), L('2'), L('7'), L('0'), L('4'), L('7'), L('7'),
296	L('5'), L('5'), M(2079, 4), L('3'), L('7'), L('9'), L('6'), L('4'),
297	L('1'), L('4'), L('5'), L('1'), L('5'), L('2'), M(1534, 4), L('2'),
298	L('3'), L('4'), L('3'), L('6'), L('4'), L('5'), L('4'), M(1503, 4),
299	L('4'), L('4'), L('4'), L('7'), L('9'), L('5'), M(61, 4), M(1316, 4),
300	M(2279, 5), L('4'), L('1'), M(1323, 4), L('3'), M(773, 4), L('5'), L('2'),
301	L('3'), L('1'), M(2114, 5), L('1'), L('6'), L('6'), L('1'), M(2227, 4),
302	L('5'), L('9'), L('6'), L('9'), L('5'), L('3'), L('6'), L('2'),
303	L('3'), L('1'), L('4'), M(1536, 4), L('2'), L('4'), L('8'), L('4'),
304	L('9'), L('3'), L('7'), L('1'), L('8'), L('7'), L('1'), L('1'),
305	L('0'), L('1'), L('4'), L('5'), L('7'), L('6'), L('5'), L('4'),
306	M(1890, 4), L('0'), L('2'), L('7'), L('9'), L('9'), L('3'), L('4'),
307	L('4'), L('0'), L('3'), L('7'), L('4'), L('2'), L('0'), L('0'),
308	L('7'), M(2368, 4), L('7'), L('8'), L('5'), L('3'), L('9'), L('0'),
309	L('6'), L('2'), L('1'), L('9'), M(666, 5), M(838, 4), L('8'), L('4'),
310	L('7'), M(979, 5), L('8'), L('3'), L('3'), L('2'), L('1'), L('4'),
311	L('4'), L('5'), L('7'), L('1'), M(645, 4), M(1911, 4), L('4'), L('3'),
312	L('5'), L('0'), M(2345, 4), M(1129, 4), L('5'), L('3'), L('1'), L('9'),
313	L('1'), L('0'), L('4'), L('8'), L('4'), L('8'), L('1'), L('0'),
314	L('0'), L('5'), L('3'), L('7'), L('0'), L('6'), M(2237, 4), M(1438, 5),
315	M(1922, 5), L('1'), M(1370, 4), L('7'), M(796, 4), L('5'), M(2029, 4), M(1037, 4),
316	L('6'), L('3'), M(2013, 5), L('4'), M(2418, 4), M(847, 5), M(1014, 5), L('8'),
317	M(1326, 5), M(2184, 5), L('9'), M(392, 4), L('9'), L('1'), M(2255, 4), L('8'),
318	L('1'), L('4'), L('6'), L('7'), L('5'), L('1'), M(1580, 4), L('1'),
319	L('2'), L('3'), L('9'), M(426, 6), L('9'), L('0'), L('7'), L('1'),
320	L('8'), L('6'), L('4'), L('9'), L('4'), L('2'), L('3'), L('1'),
321	L('9'), L('6'), L('1'), L('5'), L('6'), M(493, 4), M(1725, 4), L('9'),
322	L('5'), M(2343, 4), M(1130, 4), M(284, 4), L('6'), L('0'), L('3'), L('8'),
323	M(2598, 4), M(368, 4), M(901, 4), L('6'), L('2'), M(1115, 4), L('5'), M(2125, 4),
324	L('6'), L('3'), L('8'), L('9'), L('3'), L('7'), L('7'), L('8'),
325	L('7'), M(2246, 4), M(249, 4), L('9'), L('7'), L('9'), L('2'), L('0'),
326	L('7'), L('7'), L('3'), M(1496, 4), L('2'), L('1'), L('8'), L('2'),
327	L('5'), L('6'), M(2016, 4), L('6'), L('6'), M(1751, 4), L('4'), L('2'),
328	M(1663, 5), L('6'), M(1767, 4), L('4'), L('4'), M(37, 4), L('5'), L('4'),
329	L('9'), L('2'), L('0'), L('2'), L('6'), L('0'), L('5'), M(2740, 4),
330	M(997, 5), L('2'), L('0'), L('1'), L('4'), L('9'), M(1235, 4), L('8'),
331	L('5'), L('0'), L('7'), L('3'), M(1434, 4), L('6'), L('6'), L('6'),
332	L('0'), M(405, 4), L('2'), L('4'), L('3'), L('4'), L('0'), M(136, 4),
333	L('0'), M(1900, 4), L('8'), L('6'), L('3'), M(2391, 4), M(2021, 4), M(1068, 4),
334	M(373, 4), L('5'), L('7'), L('9'), L('6'), L('2'), L('6'), L('8'),
335	L('5'), L('6'), M(321, 4), L('5'), L('0'), L('8'), M(1316, 4), L('5'),
336	L('8'), L('7'), L('9'), L('6'), L('9'), L('9'), M(1810, 4), L('5'),
337	L('7'), L('4'), M(2585, 4), L('8'), L('4'), L('0'), M(2228, 4), L('1'),
338	L('4'), L('5'), L('9'), L('1'), M(1933, 4), L('7'), L('0'), M(565, 4),
339	L('0'), L('1'), M(3048, 4), L('1'), L('2'), M(3189, 4), L('0'), M(964, 4),
340	L('3'), L('9'), M(2859, 4), M(275, 4), L('7'), L('1'), L('5'), M(945, 4),
341	L('4'), L('2'), L('0'), M(3059, 5), L('9'), M(3011, 4), L('0'), L('7'),
342	M(834, 4), M(1942, 4), M(2736, 4), M(3171, 4), L('2'), L('1'), M(2401, 4), L('2'),
343	L('5'), L('1'), M(1404, 4), M(2373, 4), L('9'), L('2'), M(435, 4), L('8'),
344	L('2'), L('6'), M(2919, 4), L('2'), M(633, 4), L('3'), L('2'), L('1'),
345	L('5'), L('7'), L('9'), L('1'), L('9'), L('8'), L('4'), L('1'),
346	L('4'), M(2172, 5), L('9'), L('1'), L('6'), L('4'), M(1769, 5), L('9'),
347	M(2905, 5), M(2268, 4), L('7'), L('2'), L('2'), M(802, 4), L('5'), M(2213, 4),
348	M(322, 4), L('9'), L('1'), L('0'), M(189, 4), M(3164, 4), L('5'), L('2'),
349	L('8'), L('0'), L('1'), L('7'), M(562, 4), L('7'), L('1'), L('2'),
350	M(2325, 4), L('8'), L('3'), L('2'), M(884, 4), L('1'), M(1418, 4), L('0'),
351	L('9'), L('3'), L('5'), L('3'), L('9'), L('6'), L('5'), L('7'),
352	M(1612, 4), L('1'), L('0'), L('8'), L('3'), M(106, 4), L('5'), L('1'),
353	M(1915, 4), M(3419, 4), L('1'), L('4'), L('4'), L('4'), L('2'), L('1'),
354	L('0'), L('0'), M(515, 4), L('0'), L('3'), M(413, 4), L('1'), L('1'),
355	L('0'), L('3'), M(3202, 4), M(10, 4), M(39, 4), M(1539, 6), L('5'), L('1'),
356	L('6'), M(1498, 4), M(2180, 5), M(2347, 4), L('5'), M(3139, 5), L('8'), L('5'),
357	L('1'), L('7'), L('1'), L('4'), L('3'), L('7'), M(1542, 4), M(110, 4),
358	L('1'), L('5'), L('5'), L('6'), L('5'), L('0'), L('8'), L('8'),
359	M(954, 4), L('9'), L('8'), L('9'), L('8'), L('5'), L('9'), L('9'),
360	L('8'), L('2'), L('3'), L('8'), M(464, 4), M(2491, 4), L('3'), M(365, 4),
361	M(1087, 4), M(2500, 4), L('8'), M(3590, 5), L('3'), L('2'), M(264, 4), L('5'),
362	M(774, 4), L('3'), M(459, 4), L('9'), M(1052, 4), L('9'), L('8'), M(2174, 4),
363	L('4'), M(3257, 4), L('7'), M(1612, 4), L('0'), L('7'), M(230, 4), L('4'),
364	L('8'), L('1'), L('4'), L('1'), M(1338, 4), L('8'), L('5'), L('9'),
365	L('4'), L('6'), L('1'), M(3018, 4), L('8'), L('0'),
366	},
367	},
368	TestCase{
369	.input = "huffman-rand-1k.input",
370	.want = "huffman-rand-1k.{s}.expect",
371	.want_no_input = "huffman-rand-1k.{s}.expect-noinput",
372	.tokens = &[_]Token{
373	L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xd), L(0x85), L(0x94), L(0x25), L(0x80), L(0xaf), L(0xc2), L(0xfe), L(0x8d),
374	L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b),
375	L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4),
376	L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde),
377	L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14),
378	L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xd), L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c),
379	L(0x99), L(0xdf), L(0xfd), L(0x70), L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5),
380	L(0xd4), L(0x80), L(0x59), L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20),
381	L(0x1b), L(0x46), L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4),
382	L(0x7b), L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
383	L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), L(0xa0),
384	L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), L(0x75), L(0xda),
385	L(0xea), L(0x9b), L(0xa), L(0x64), L(0xb), L(0xe0), L(0x23), L(0x29), L(0xbd), L(0xf7), L(0xe7), L(0x83), L(0x3c), L(0xfb),
386	L(0xdf), L(0xb3), L(0xae), L(0x4f), L(0xa4), L(0x47), L(0x55), L(0x99), L(0xde), L(0x2f), L(0x96), L(0x6e), L(0x1c), L(0x43),
387	L(0x4c), L(0x87), L(0xe2), L(0x7c), L(0xd9), L(0x5f), L(0x4c), L(0x7c), L(0xe8), L(0x90), L(0x3), L(0xdb), L(0x30), L(0x95),
388	L(0xd6), L(0x22), L(0xc), L(0x47), L(0xb8), L(0x4d), L(0x6b), L(0xbd), L(0x24), L(0x11), L(0xab), L(0x2c), L(0xd7), L(0xbe),
389	L(0x6e), L(0x7a), L(0xd6), L(0x8), L(0xa3), L(0x98), L(0xd8), L(0xdd), L(0x15), L(0x6a), L(0xfa), L(0x93), L(0x30), L(0x1),
390	L(0x25), L(0x1d), L(0xa2), L(0x74), L(0x86), L(0x4b), L(0x6a), L(0x95), L(0xe8), L(0xe1), L(0x4e), L(0xe), L(0x76), L(0xb9),
391	L(0x49), L(0xa9), L(0x5f), L(0xa0), L(0xa6), L(0x63), L(0x3c), L(0x7e), L(0x7e), L(0x20), L(0x13), L(0x4f), L(0xbb), L(0x66),
392	L(0x92), L(0xb8), L(0x2e), L(0xa4), L(0xfa), L(0x48), L(0xcb), L(0xae), L(0xb9), L(0x3c), L(0xaf), L(0xd3), L(0x1f), L(0xe1),
393	L(0xd5), L(0x8d), L(0x42), L(0x6d), L(0xf0), L(0xfc), L(0x8c), L(0xc), L(0x0), L(0xde), L(0x40), L(0xab), L(0x8b), L(0x47),
394	L(0x97), L(0x4e), L(0xa8), L(0xcf), L(0x8e), L(0xdb), L(0xa6), L(0x8b), L(0x20), L(0x9), L(0x84), L(0x7a), L(0x66), L(0xe5),
395	L(0x98), L(0x29), L(0x2), L(0x95), L(0xe6), L(0x38), L(0x32), L(0x60), L(0x3), L(0xe3), L(0x9a), L(0x1e), L(0x54), L(0xe8),
396	L(0x63), L(0x80), L(0x48), L(0x9c), L(0xe7), L(0x63), L(0x33), L(0x6e), L(0xa0), L(0x65), L(0x83), L(0xfa), L(0xc6), L(0xba),
397	L(0x7a), L(0x43), L(0x71), L(0x5), L(0xf5), L(0x68), L(0x69), L(0x85), L(0x9c), L(0xba), L(0x45), L(0xcd), L(0x6b), L(0xb),
398	L(0x19), L(0xd1), L(0xbb), L(0x7f), L(0x70), L(0x85), L(0x92), L(0xd1), L(0xb4), L(0x64), L(0x82), L(0xb1), L(0xe4), L(0x62),
399	L(0xc5), L(0x3c), L(0x46), L(0x1f), L(0x92), L(0x31), L(0x1c), L(0x4e), L(0x41), L(0x77), L(0xf7), L(0xe7), L(0x87), L(0xa2),
400	L(0xf), L(0x6e), L(0xe8), L(0x92), L(0x3), L(0x6b), L(0xa), L(0xe7), L(0xa9), L(0x3b), L(0x11), L(0xda), L(0x66), L(0x8a),
401	L(0x29), L(0xda), L(0x79), L(0xe1), L(0x64), L(0x8d), L(0xe3), L(0x54), L(0xd4), L(0xf5), L(0xef), L(0x64), L(0x87), L(0x3b),
402	L(0xf4), L(0xc2), L(0xf4), L(0x71), L(0x13), L(0xa9), L(0xe9), L(0xe0), L(0xa2), L(0x6), L(0x14), L(0xab), L(0x5d), L(0xa7),
403	L(0x96), L(0x0), L(0xd6), L(0xc3), L(0xcc), L(0x57), L(0xed), L(0x39), L(0x6a), L(0x25), L(0xcd), L(0x76), L(0xea), L(0xba),
404	L(0x3a), L(0xf2), L(0xa1), L(0x95), L(0x5d), L(0xe5), L(0x71), L(0xcf), L(0x9c), L(0x62), L(0x9e), L(0x6a), L(0xfa), L(0xd5),
405	L(0x31), L(0xd1), L(0xa8), L(0x66), L(0x30), L(0x33), L(0xaa), L(0x51), L(0x17), L(0x13), L(0x82), L(0x99), L(0xc8), L(0x14),
406	L(0x60), L(0x9f), L(0x4d), L(0x32), L(0x6d), L(0xda), L(0x19), L(0x26), L(0x21), L(0xdc), L(0x7e), L(0x2e), L(0x25), L(0x67),
407	L(0x72), L(0xca), L(0xf), L(0x92), L(0xcd), L(0xf6), L(0xd6), L(0xcb), L(0x97), L(0x8a), L(0x33), L(0x58), L(0x73), L(0x70),
408	L(0x91), L(0x1d), L(0xbf), L(0x28), L(0x23), L(0xa3), L(0xc), L(0xf1), L(0x83), L(0xc3), L(0xc8), L(0x56), L(0x77), L(0x68),
409	L(0xe3), L(0x82), L(0xba), L(0xb9), L(0x57), L(0x56), L(0x57), L(0x9c), L(0xc3), L(0xd6), L(0x14), L(0x5), L(0x3c), L(0xb1),
410	L(0xaf), L(0x93), L(0xc8), L(0x8a), L(0x57), L(0x7f), L(0x53), L(0xfa), L(0x2f), L(0xaa), L(0x6e), L(0x66), L(0x83), L(0xfa),
411	L(0x33), L(0xd1), L(0x21), L(0xab), L(0x1b), L(0x71), L(0xb4), L(0x7c), L(0xda), L(0xfd), L(0xfb), L(0x7f), L(0x20), L(0xab),
412	L(0x5e), L(0xd5), L(0xca), L(0xfd), L(0xdd), L(0xe0), L(0xee), L(0xda), L(0xba), L(0xa8), L(0x27), L(0x99), L(0x97), L(0x69),
413	L(0xc1), L(0x3c), L(0x82), L(0x8c), L(0xa), L(0x5c), L(0x2d), L(0x5b), L(0x88), L(0x3e), L(0x34), L(0x35), L(0x86), L(0x37),
414	L(0x46), L(0x79), L(0xe1), L(0xaa), L(0x19), L(0xfb), L(0xaa), L(0xde), L(0x15), L(0x9), L(0xd), L(0x1a), L(0x57), L(0xff),
415	L(0xb5), L(0xf), L(0xf3), L(0x2b), L(0x5a), L(0x6a), L(0x4d), L(0x19), L(0x77), L(0x71), L(0x45), L(0xdf), L(0x4f), L(0xb3),
416	L(0xec), L(0xf1), L(0xeb), L(0x18), L(0x53), L(0x3e), L(0x3b), L(0x47), L(0x8), L(0x9a), L(0x73), L(0xa0), L(0x5c), L(0x8c),
417	L(0x5f), L(0xeb), L(0xf), L(0x3a), L(0xc2), L(0x43), L(0x67), L(0xb4), L(0x66), L(0x67), L(0x80), L(0x58), L(0xe), L(0xc1),
418	L(0xec), L(0x40), L(0xd4), L(0x22), L(0x94), L(0xca), L(0xf9), L(0xe8), L(0x92), L(0xe4), L(0x69), L(0x38), L(0xbe), L(0x67),
419	L(0x64), L(0xca), L(0x50), L(0xc7), L(0x6), L(0x67), L(0x42), L(0x6e), L(0xa3), L(0xf0), L(0xb7), L(0x6c), L(0xf2), L(0xe8),
420	L(0x5f), L(0xb1), L(0xaf), L(0xe7), L(0xdb), L(0xbb), L(0x77), L(0xb5), L(0xf8), L(0xcb), L(0x8), L(0xc4), L(0x75), L(0x7e),
421	L(0xc0), L(0xf9), L(0x1c), L(0x7f), L(0x3c), L(0x89), L(0x2f), L(0xd2), L(0x58), L(0x3a), L(0xe2), L(0xf8), L(0x91), L(0xb6),
422	L(0x7b), L(0x24), L(0x27), L(0xe9), L(0xae), L(0x84), L(0x8b), L(0xde), L(0x74), L(0xac), L(0xfd), L(0xd9), L(0xb7), L(0x69),
423	L(0x2a), L(0xec), L(0x32), L(0x6f), L(0xf0), L(0x92), L(0x84), L(0xf1), L(0x40), L(0xc), L(0x8a), L(0xbc), L(0x39), L(0x6e),
424	L(0x2e), L(0x73), L(0xd4), L(0x6e), L(0x8a), L(0x74), L(0x2a), L(0xdc), L(0x60), L(0x1f), L(0xa3), L(0x7), L(0xde), L(0x75),
425	L(0x8b), L(0x74), L(0xc8), L(0xfe), L(0x63), L(0x75), L(0xf6), L(0x3d), L(0x63), L(0xac), L(0x33), L(0x89), L(0xc3), L(0xf0),
426	L(0xf8), L(0x2d), L(0x6b), L(0xb4), L(0x9e), L(0x74), L(0x8b), L(0x5c), L(0x33), L(0xb4), L(0xca), L(0xa8), L(0xe4), L(0x99),
427	L(0xb6), L(0x90), L(0xa1), L(0xef), L(0xf), L(0xd3), L(0x61), L(0xb2), L(0xc6), L(0x1a), L(0x94), L(0x7c), L(0x44), L(0x55),
428	L(0xf4), L(0x45), L(0xff), L(0x9e), L(0xa5), L(0x5a), L(0xc6), L(0xa0), L(0xe8), L(0x2a), L(0xc1), L(0x8d), L(0x6f), L(0x34),
429	L(0x11), L(0xb9), L(0xbe), L(0x4e), L(0xd9), L(0x87), L(0x97), L(0x73), L(0xcf), L(0x3d), L(0x23), L(0xae), L(0xd5), L(0x1a),
430	L(0x5e), L(0xae), L(0x5d), L(0x6a), L(0x3), L(0xf9), L(0x22), L(0xd), L(0x10), L(0xd9), L(0x47), L(0x69), L(0x15), L(0x3f),
431	L(0xee), L(0x52), L(0xa3), L(0x8), L(0xd2), L(0x3c), L(0x51), L(0xf4), L(0xf8), L(0x9d), L(0xe4), L(0x98), L(0x89), L(0xc8),
432	L(0x67), L(0x39), L(0xd5), L(0x5e), L(0x35), L(0x78), L(0x27), L(0xe8), L(0x3c), L(0x80), L(0xae), L(0x79), L(0x71), L(0xd2),
433	L(0x93), L(0xf4), L(0xaa), L(0x51), L(0x12), L(0x1c), L(0x4b), L(0x1b), L(0xe5), L(0x6e), L(0x15), L(0x6f), L(0xe4), L(0xbb),
434	L(0x51), L(0x9b), L(0x45), L(0x9f), L(0xf9), L(0xc4), L(0x8c), L(0x2a), L(0xfb), L(0x1a), L(0xdf), L(0x55), L(0xd3), L(0x48),
435	L(0x93), L(0x27), L(0x1), L(0x26), L(0xc2), L(0x6b), L(0x55), L(0x6d), L(0xa2), L(0xfb), L(0x84), L(0x8b), L(0xc9), L(0x9e),
436	L(0x28), L(0xc2), L(0xef), L(0x1a), L(0x24), L(0xec), L(0x9b), L(0xae), L(0xbd), L(0x60), L(0xe9), L(0x15), L(0x35), L(0xee),
437	L(0x42), L(0xa4), L(0x33), L(0x5b), L(0xfa), L(0xf), L(0xb6), L(0xf7), L(0x1), L(0xa6), L(0x2), L(0x4c), L(0xca), L(0x90),
438	L(0x58), L(0x3a), L(0x96), L(0x41), L(0xe7), L(0xcb), L(0x9), L(0x8c), L(0xdb), L(0x85), L(0x4d), L(0xa8), L(0x89), L(0xf3),
439	L(0xb5), L(0x8e), L(0xfd), L(0x75), L(0x5b), L(0x4f), L(0xed), L(0xde), L(0x3f), L(0xeb), L(0x38), L(0xa3), L(0xbe), L(0xb0),
440	L(0x73), L(0xfc), L(0xb8), L(0x54), L(0xf7), L(0x4c), L(0x30), L(0x67), L(0x2e), L(0x38), L(0xa2), L(0x54), L(0x18), L(0xba),
441	L(0x8), L(0xbf), L(0xf2), L(0x39), L(0xd5), L(0xfe), L(0xa5), L(0x41), L(0xc6), L(0x66), L(0x66), L(0xba), L(0x81), L(0xef),
442	L(0x67), L(0xe4), L(0xe6), L(0x3c), L(0xc), L(0xca), L(0xa4), L(0xa), L(0x79), L(0xb3), L(0x57), L(0x8b), L(0x8a), L(0x75),
443	L(0x98), L(0x18), L(0x42), L(0x2f), L(0x29), L(0xa3), L(0x82), L(0xef), L(0x9f), L(0x86), L(0x6), L(0x23), L(0xe1), L(0x75),
444	L(0xfa), L(0x8), L(0xb1), L(0xde), L(0x17), L(0x4a),
445	},
446	},
447	TestCase{
448	.input = "huffman-rand-limit.input",
449	.want = "huffman-rand-limit.{s}.expect",
450	.want_no_input = "huffman-rand-limit.{s}.expect-noinput",
451	.tokens = &[_]Token{
452	L(0x61), M(1, 74), L(0xa), L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xa), L(0x85), L(0x94), L(0x25), L(0x80),
453	L(0xaf), L(0xc2), L(0xfe), L(0x8d), L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde),
454	L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73),
455	L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15),
456	L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9),
457	L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xa),
458	L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), L(0x99), L(0xdf), L(0xfd), L(0x70),
459	L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), L(0xd4), L(0x80), L(0x59),
460	L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), L(0x1b), L(0x46),
461	L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), L(0x7b),
462	L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
463	L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f),
464	L(0xa0), L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4),
465	L(0x75), L(0xda), L(0xea), L(0x9b), L(0xa),
466	},
467	},
468	TestCase{
469	.input = "huffman-shifts.input",
470	.want = "huffman-shifts.{s}.expect",
471	.want_no_input = "huffman-shifts.{s}.expect-noinput",
472	.tokens = &[_]Token{
473	L('1'), L('0'), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
474	M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
475	M(2, 258), M(2, 76), L(0xd), L(0xa), L('2'), L('3'), M(2, 258), M(2, 258),
476	M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 256),
477	},
478	},
479	TestCase{
480	.input = "huffman-text-shift.input",
481	.want = "huffman-text-shift.{s}.expect",
482	.want_no_input = "huffman-text-shift.{s}.expect-noinput",
483	.tokens = &[_]Token{
484	L('/'), L('/'), L('C'), L('o'), L('p'), L('y'), L('r'), L('i'),
485	L('g'), L('h'), L('t'), L('2'), L('0'), L('0'), L('9'), L('T'),
486	L('h'), L('G'), L('o'), L('A'), L('u'), L('t'), L('h'), L('o'),
487	L('r'), L('.'), L('A'), L('l'), L('l'), M(23, 5), L('r'), L('r'),
488	L('v'), L('d'), L('.'), L(0xd), L(0xa), L('/'), L('/'), L('U'),
489	L('o'), L('f'), L('t'), L('h'), L('i'), L('o'), L('u'), L('r'),
490	L('c'), L('c'), L('o'), L('d'), L('i'), L('g'), L('o'), L('v'),
491	L('r'), L('n'), L('d'), L('b'), L('y'), L('B'), L('S'), L('D'),
492	L('-'), L('t'), L('y'), L('l'), M(33, 4), L('l'), L('i'), L('c'),
493	L('n'), L('t'), L('h'), L('t'), L('c'), L('n'), L('b'), L('f'),
494	L('o'), L('u'), L('n'), L('d'), L('i'), L('n'), L('t'), L('h'),
495	L('L'), L('I'), L('C'), L('E'), L('N'), L('S'), L('E'), L('f'),
496	L('i'), L('l'), L('.'), L(0xd), L(0xa), L(0xd), L(0xa), L('p'),
497	L('c'), L('k'), L('g'), L('m'), L('i'), L('n'), M(11, 4), L('i'),
498	L('m'), L('p'), L('o'), L('r'), L('t'), L('"'), L('o'), L('"'),
499	M(13, 4), L('f'), L('u'), L('n'), L('c'), L('m'), L('i'), L('n'),
500	L('('), L(')'), L('{'), L(0xd), L(0xa), L(0x9), L('v'), L('r'),
501	L('b'), L('='), L('m'), L('k'), L('('), L('['), L(']'), L('b'),
502	L('y'), L('t'), L(','), L('6'), L('5'), L('5'), L('3'), L('5'),
503	L(')'), L(0xd), L(0xa), L(0x9), L('f'), L(','), L('_'), L(':'),
504	L('='), L('o'), L('.'), L('C'), L('r'), L('t'), L('('), L('"'),
505	L('h'), L('u'), L('f'), L('f'), L('m'), L('n'), L('-'), L('n'),
506	L('u'), L('l'), L('l'), L('-'), L('m'), L('x'), L('.'), L('i'),
507	L('n'), L('"'), M(34, 5), L('.'), L('W'), L('r'), L('i'), L('t'),
508	L('('), L('b'), L(')'), L(0xd), L(0xa), L('}'), L(0xd), L(0xa),
509	L('A'), L('B'), L('C'), L('D'), L('E'), L('F'), L('G'), L('H'),
510	L('I'), L('J'), L('K'), L('L'), L('M'), L('N'), L('O'), L('P'),
511	L('Q'), L('R'), L('S'), L('T'), L('U'), L('V'), L('X'), L('x'),
512	L('y'), L('z'), L('!'), L('"'), L('#'), L(0xc2), L(0xa4), L('%'),
513	L('&'), L('/'), L('?'), L('"'),
514	},
515	},
516	TestCase{
517	.input = "huffman-text.input",
518	.want = "huffman-text.{s}.expect",
519	.want_no_input = "huffman-text.{s}.expect-noinput",
520	.tokens = &[_]Token{
521	L('/'), L('/'), L(' '), L('z'), L('i'), L('g'), L(' '), L('v'),
522	L('0'), L('.'), L('1'), L('0'), L('.'), L('0'), L(0xa), L('/'),
523	L('/'), L(' '), L('c'), L('r'), L('e'), L('a'), L('t'), L('e'),
524	L(' '), L('a'), L(' '), L('f'), L('i'), L('l'), L('e'), M(5, 4),
525	L('l'), L('e'), L('d'), L(' '), L('w'), L('i'), L('t'), L('h'),
526	L(' '), L('0'), L('x'), L('0'), L('0'), L(0xa), L('c'), L('o'),
527	L('n'), L('s'), L('t'), L(' '), L('s'), L('t'), L('d'), L(' '),
528	L('='), L(' '), L('@'), L('i'), L('m'), L('p'), L('o'), L('r'),
529	L('t'), L('('), L('"'), L('s'), L('t'), L('d'), L('"'), L(')'),
530	L(';'), L(0xa), L(0xa), L('p'), L('u'), L('b'), L(' '), L('f'),
531	L('n'), L(' '), L('m'), L('a'), L('i'), L('n'), L('('), L(')'),
532	L(' '), L('!'), L('v'), L('o'), L('i'), L('d'), L(' '), L('{'),
533	L(0xa), L(' '), L(' '), L(' '), L(' '), L('v'), L('a'), L('r'),
534	L(' '), L('b'), L(' '), L('='), L(' '), L('['), L('1'), L(']'),
535	L('u'), L('8'), L('{'), L('0'), L('}'), L(' '), L(''), L(''),
536	L(' '), L('6'), L('5'), L('5'), L('3'), L('5'), L(';'), M(31, 5),
537	M(86, 6), L('f'), L(' '), L('='), L(' '), L('t'), L('r'), L('y'),
538	M(94, 4), L('.'), L('f'), L('s'), L('.'), L('c'), L('w'), L('d'),
539	L('('), L(')'), L('.'), M(144, 6), L('F'), L('i'), L('l'), L('e'),
540	L('('), M(43, 5), M(1, 4), L('"'), L('h'), L('u'), L('f'), L('f'),
541	L('m'), L('a'), L('n'), L('-'), L('n'), L('u'), L('l'), L('l'),
542	L('-'), L('m'), L('a'), L('x'), L('.'), L('i'), L('n'), L('"'),
543	L(','), M(31, 9), L('.'), L('{'), L(' '), L('.'), L('r'), L('e'),
544	L('a'), L('d'), M(79, 5), L('u'), L('e'), L(' '), L('}'), M(27, 6),
545	L(')'), M(108, 6), L('d'), L('e'), L('f'), L('e'), L('r'), L(' '),
546	L('f'), L('.'), L('c'), L('l'), L('o'), L('s'), L('e'), L('('),
547	M(183, 4), M(22, 4), L('_'), M(124, 7), L('f'), L('.'), L('w'), L('r'),
548	L('i'), L('t'), L('e'), L('A'), L('l'), L('l'), L('('), L('b'),
549	L('['), L('0'), L('.'), L('.'), L(']'), L(')'), L(';'), L(0xa),
550	L('}'), L(0xa),
551	},
552	},
553	TestCase{
554	.input = "huffman-zero.input",
555	.want = "huffman-zero.{s}.expect",
556	.want_no_input = "huffman-zero.{s}.expect-noinput",
557	.tokens = &[_]Token{ L(0x30), ml, M(1, 49) },
558	},
559	TestCase{
560	.input = "",
561	.want = "",
562	.want_no_input = "null-long-match.{s}.expect-noinput",
563	.tokens = &[_]Token{
564	L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
565	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
566	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
567	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
568	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
569	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
570	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
571	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
572	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
573	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
574	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
575	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
576	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
577	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
578	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
579	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
580	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
581	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
582	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
583	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
584	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
585	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
586	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
587	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
588	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
589	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
590	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
591	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
592	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
593	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
594	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
595	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
596	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
597	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
598	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
599	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
600	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
601	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
602	ml, ml, ml, M(1, 8),
603	},
604	},
605	};
606	};

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.input differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.input created+1

...	@@ -0,0 +1 @@
1	3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input created+4

...	@@ -0,0 +1,4 @@
1	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2	��vH
3	��%�� ɷ��}��>��ls��m�IGH��1Y�4�[�� 0[\|]o#�
4	�-#��ul��pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O��{��k�*�w(��b� ��kQC9/��lu>�5�C.��u�

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.input created+2

...	@@ -0,0 +1,2 @@
1	101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2	232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input created+14

...	@@ -0,0 +1,14 @@
1	//Copyright2009ThGoAuthor.Allrightrrvd.
2	//UofthiourccodigovrndbyBSD-tyl
3	//licnthtcnbfoundinthLICENSEfil.
4
5	pckgmin
6
7	import"o"
8
9	funcmin(){
10	vrb=mk([]byt,65535)
11	f,_:=o.Crt("huffmn-null-mx.in")
12	f.Writ(b)
13	}
14	ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text.input created+14

...	@@ -0,0 +1,14 @@
1	// zig v0.10.0
2	// create a file filled with 0x00
3	const std = @import("std");
4
5	pub fn main() !void {
6	var b = [1]u8{0} ** 65535;
7	const f = try std.fs.cwd().createFile(
8	"huffman-null-max.in",
9	.{ .read = true },
10	);
11	defer f.close();
12
13	_ = try f.writeAll(b[0..]);
14	}

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.input created+1

...	@@ -0,0 +1 @@
1	00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
	\ No newline at end of file

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect differ

lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput differ

lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput differ

lib/std/compress/flate/testdata/fuzz/deflate-stream.expect created+22

...	@@ -0,0 +1,22 @@
1	[
2	{ id: "brieflz",
3	name: "BriefLZ",
4	libraryUrl: "https://github.com/jibsen/brieflz",
5	license: "MIT",
6	revision: "bcaa6a1ee7ccf005512b5c23aa92b40cf75f9ed1",
7	codecs: [ { name: "brieflz" } ], },
8	{ id: "brotli",
9	name: "Brotli",
10	libraryUrl: "https://github.com/google/brotli",
11	license: "Apache 2.0",
12	revision: "1dd66ef114fd244778d9dcb5da09c28b49a0df33",
13	codecs: [ { name: "brotli",
14	levels: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11],
15	streaming: true } ], },
16	{ id: "bsc",
17	name: "bsc",
18	libraryUrl: "http://libbsc.com/",
19	license: "Apache 2.0",
20	revision: "b2b07421381b19b2fada8b291f3cdead10578abc",
21	codecs: [ { name: "bsc" } ] }
22	]

lib/std/compress/flate/testdata/fuzz/deflate-stream.input created+3

...	@@ -0,0 +1,3 @@
1	��=o�0��+N��ڭR��K�}>W!AI@j+�{
2	�\|��<��F�x[�ش�\�9f;P�%�0৷#��iu��V��UWDQ��L�YF��tTG��U��_��\|S�Q��D��<M�4)�Xk%M��e�SdűK�0�]Cas[v��;�ɕMSV��J��@N�5Ea�tJN��Y�$Y�[eѤVs�27��ܺ8��}wӆ��.H1��A�`� c�3PW��%��uY@߮�jN��]$
3	,<��L�4<K��sa�2�i�s#�p1Z�V�4˵��؎��o
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet01.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet01.input differ

lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet02.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/empty-distance-alphabet02.input differ

lib/std/compress/flate/testdata/fuzz/end-of-stream.input created+1

...	@@ -0,0 +1 @@
1	��=o�0�
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/fuzz1.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz1.input differ

lib/std/compress/flate/testdata/fuzz/fuzz2.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz2.input differ

lib/std/compress/flate/testdata/fuzz/fuzz3.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz3.input differ

lib/std/compress/flate/testdata/fuzz/fuzz4.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/fuzz4.input differ

lib/std/compress/flate/testdata/fuzz/invalid-distance.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/invalid-distance.input differ

lib/std/compress/flate/testdata/fuzz/invalid-tree01.input created+1

...	@@ -0,0 +1 @@
1	000
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/invalid-tree02.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/invalid-tree02.input differ

lib/std/compress/flate/testdata/fuzz/invalid-tree03.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/invalid-tree03.input differ

lib/std/compress/flate/testdata/fuzz/lengths-overflow.input created+1

...	@@ -0,0 +1 @@
1	�$��9
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/out-of-codes.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/out-of-codes.input differ

lib/std/compress/flate/testdata/fuzz/puff01.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff01.input differ

lib/std/compress/flate/testdata/fuzz/puff02.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff02.input differ

lib/std/compress/flate/testdata/fuzz/puff03.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff03.input differ

lib/std/compress/flate/testdata/fuzz/puff04.input created+1

...	@@ -0,0 +1 @@
1	~��
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff05.input created+1

...	@@ -0,0 +1 @@
1
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff06.input created+1

...	@@ -0,0 +1 @@
1	�I�$I�$��
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff07.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff07.input differ

lib/std/compress/flate/testdata/fuzz/puff08.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff08.input differ

lib/std/compress/flate/testdata/fuzz/puff09.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff09.input differ

lib/std/compress/flate/testdata/fuzz/puff10.input created+1

...	@@ -0,0 +1 @@
1
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff11.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff11.input differ

lib/std/compress/flate/testdata/fuzz/puff12.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff12.input differ

lib/std/compress/flate/testdata/fuzz/puff13.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff13.input differ

lib/std/compress/flate/testdata/fuzz/puff14.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff14.input differ

lib/std/compress/flate/testdata/fuzz/puff15.input created+1

...	@@ -0,0 +1 @@
1	�I�$I�$��Ä
	\ No newline at end of file

lib/std/compress/flate/testdata/fuzz/puff16.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff16.input differ

lib/std/compress/flate/testdata/fuzz/puff17.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff17.input differ

lib/std/compress/flate/testdata/fuzz/puff18.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff18.input differ

lib/std/compress/flate/testdata/fuzz/puff19.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff19.input differ

lib/std/compress/flate/testdata/fuzz/puff20.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff20.input differ

lib/std/compress/flate/testdata/fuzz/puff21.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff21.input differ

lib/std/compress/flate/testdata/fuzz/puff22.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff22.input differ

lib/std/compress/flate/testdata/fuzz/puff23.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff23.input differ

lib/std/compress/flate/testdata/fuzz/puff24.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff24.input differ

lib/std/compress/flate/testdata/fuzz/puff25.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff25.input differ

lib/std/compress/flate/testdata/fuzz/puff26.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff26.input differ

lib/std/compress/flate/testdata/fuzz/puff27.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/puff27.input differ

lib/std/compress/flate/testdata/fuzz/roundtrip1.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/roundtrip1.input differ

lib/std/compress/flate/testdata/fuzz/roundtrip2.input created

Binary files /dev/null and b/lib/std/compress/flate/testdata/fuzz/roundtrip2.input differ

lib/std/compress/flate/testdata/rfc1951.txt created+955

...	@@ -0,0 +1,955 @@
1
2
3
4
5
6
7	Network Working Group P. Deutsch
8	Request for Comments: 1951 Aladdin Enterprises
9	Category: Informational May 1996
10
11
12	DEFLATE Compressed Data Format Specification version 1.3
13
14	Status of This Memo
15
16	This memo provides information for the Internet community. This memo
17	does not specify an Internet standard of any kind. Distribution of
18	this memo is unlimited.
19
20	IESG Note:
21
22	The IESG takes no position on the validity of any Intellectual
23	Property Rights statements contained in this document.
24
25	Notices
26
27	Copyright (c) 1996 L. Peter Deutsch
28
29	Permission is granted to copy and distribute this document for any
30	purpose and without charge, including translations into other
31	languages and incorporation into compilations, provided that the
32	copyright notice and this notice are preserved, and that any
33	substantive changes or deletions from the original are clearly
34	marked.
35
36	A pointer to the latest version of this and related documentation in
37	HTML format can be found at the URL
38	<ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40	Abstract
41
42	This specification defines a lossless compressed data format that
43	compresses data using a combination of the LZ77 algorithm and Huffman
44	coding, with efficiency comparable to the best currently available
45	general-purpose compression methods. The data can be produced or
46	consumed, even for an arbitrarily long sequentially presented input
47	data stream, using only an a priori bounded amount of intermediate
48	storage. The format can be implemented readily in a manner not
49	covered by patents.
50
51
52
53
54
55
56
57
58	Deutsch Informational [Page 1]
59
60	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
61
62
63	Table of Contents
64
65	1. Introduction ................................................... 2
66	1.1. Purpose ................................................... 2
67	1.2. Intended audience ......................................... 3
68	1.3. Scope ..................................................... 3
69	1.4. Compliance ................................................ 3
70	1.5. Definitions of terms and conventions used ................ 3
71	1.6. Changes from previous versions ............................ 4
72	2. Compressed representation overview ............................. 4
73	3. Detailed specification ......................................... 5
74	3.1. Overall conventions ....................................... 5
75	3.1.1. Packing into bytes .................................. 5
76	3.2. Compressed block format ................................... 6
77	3.2.1. Synopsis of prefix and Huffman coding ............... 6
78	3.2.2. Use of Huffman coding in the "deflate" format ....... 7
79	3.2.3. Details of block format ............................. 9
80	3.2.4. Non-compressed blocks (BTYPE=00) ................... 11
81	3.2.5. Compressed blocks (length and distance codes) ...... 11
82	3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12
83	3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13
84	3.3. Compliance ............................................... 14
85	4. Compression algorithm details ................................. 14
86	5. References .................................................... 16
87	6. Security Considerations ....................................... 16
88	7. Source code ................................................... 16
89	8. Acknowledgements .............................................. 16
90	9. Author's Address .............................................. 17
91
92	1. Introduction
93
94	1.1. Purpose
95
96	The purpose of this specification is to define a lossless
97	compressed data format that:
98	* Is independent of CPU type, operating system, file system,
99	and character set, and hence can be used for interchange;
100	* Can be produced or consumed, even for an arbitrarily long
101	sequentially presented input data stream, using only an a
102	priori bounded amount of intermediate storage, and hence
103	can be used in data communications or similar structures
104	such as Unix filters;
105	* Compresses data with efficiency comparable to the best
106	currently available general-purpose compression methods,
107	and in particular considerably better than the "compress"
108	program;
109	* Can be implemented readily in a manner not covered by
110	patents, and hence can be practiced freely;
111
112
113
114	Deutsch Informational [Page 2]
115
116	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
117
118
119	* Is compatible with the file format produced by the current
120	widely used gzip utility, in that conforming decompressors
121	will be able to read data produced by the existing gzip
122	compressor.
123
124	The data format defined by this specification does not attempt to:
125
126	* Allow random access to compressed data;
127	* Compress specialized data (e.g., raster graphics) as well
128	as the best currently available specialized algorithms.
129
130	A simple counting argument shows that no lossless compression
131	algorithm can compress every possible input data set. For the
132	format defined here, the worst case expansion is 5 bytes per 32K-
133	byte block, i.e., a size increase of 0.015% for large data sets.
134	English text usually compresses by a factor of 2.5 to 3;
135	executable files usually compress somewhat less; graphical data
136	such as raster images may compress much more.
137
138	1.2. Intended audience
139
140	This specification is intended for use by implementors of software
141	to compress data into "deflate" format and/or decompress data from
142	"deflate" format.
143
144	The text of the specification assumes a basic background in
145	programming at the level of bits and other primitive data
146	representations. Familiarity with the technique of Huffman coding
147	is helpful but not required.
148
149	1.3. Scope
150
151	The specification specifies a method for representing a sequence
152	of bytes as a (usually shorter) sequence of bits, and a method for
153	packing the latter bit sequence into bytes.
154
155	1.4. Compliance
156
157	Unless otherwise indicated below, a compliant decompressor must be
158	able to accept and decompress any data set that conforms to all
159	the specifications presented here; a compliant compressor must
160	produce data sets that conform to all the specifications presented
161	here.
162
163	1.5. Definitions of terms and conventions used
164
165	Byte: 8 bits stored or transmitted as a unit (same as an octet).
166	For this specification, a byte is exactly 8 bits, even on machines
167
168
169
170	Deutsch Informational [Page 3]
171
172	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
173
174
175	which store a character on a number of bits different from eight.
176	See below, for the numbering of bits within a byte.
177
178	String: a sequence of arbitrary bytes.
179
180	1.6. Changes from previous versions
181
182	There have been no technical changes to the deflate format since
183	version 1.1 of this specification. In version 1.2, some
184	terminology was changed. Version 1.3 is a conversion of the
185	specification to RFC style.
186
187	2. Compressed representation overview
188
189	A compressed data set consists of a series of blocks, corresponding
190	to successive blocks of input data. The block sizes are arbitrary,
191	except that non-compressible blocks are limited to 65,535 bytes.
192
193	Each block is compressed using a combination of the LZ77 algorithm
194	and Huffman coding. The Huffman trees for each block are independent
195	of those for previous or subsequent blocks; the LZ77 algorithm may
196	use a reference to a duplicated string occurring in a previous block,
197	up to 32K input bytes before.
198
199	Each block consists of two parts: a pair of Huffman code trees that
200	describe the representation of the compressed data part, and a
201	compressed data part. (The Huffman trees themselves are compressed
202	using Huffman encoding.) The compressed data consists of a series of
203	elements of two types: literal bytes (of strings that have not been
204	detected as duplicated within the previous 32K input bytes), and
205	pointers to duplicated strings, where a pointer is represented as a
206	pair <length, backward distance>. The representation used in the
207	"deflate" format limits distances to 32K bytes and lengths to 258
208	bytes, but does not limit the size of a block, except for
209	uncompressible blocks, which are limited as noted above.
210
211	Each type of value (literals, distances, and lengths) in the
212	compressed data is represented using a Huffman code, using one code
213	tree for literals and lengths and a separate code tree for distances.
214	The code trees for each block appear in a compact form just before
215	the compressed data for that block.
216
217
218
219
220
221
222
223
224
225
226	Deutsch Informational [Page 4]
227
228	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
229
230
231	3. Detailed specification
232
233	3.1. Overall conventions In the diagrams below, a box like this:
234
235	+---+
236	\| \| <-- the vertical bars might be missing
237	+---+
238
239	represents one byte; a box like this:
240
241	+==============+
242	\| \|
243	+==============+
244
245	represents a variable number of bytes.
246
247	Bytes stored within a computer do not have a "bit order", since
248	they are always treated as a unit. However, a byte considered as
249	an integer between 0 and 255 does have a most- and least-
250	significant bit, and since we write numbers with the most-
251	significant digit on the left, we also write bytes with the most-
252	significant bit on the left. In the diagrams below, we number the
253	bits of a byte so that bit 0 is the least-significant bit, i.e.,
254	the bits are numbered:
255
256	+--------+
257	\|76543210\|
258	+--------+
259
260	Within a computer, a number may occupy multiple bytes. All
261	multi-byte numbers in the format described here are stored with
262	the least-significant byte first (at the lower memory address).
263	For example, the decimal number 520 is stored as:
264
265	0 1
266	+--------+--------+
267	\|00001000\|00000010\|
268	+--------+--------+
269	^ ^
270	\| \|
271	\| + more significant byte = 2 x 256
272	+ less significant byte = 8
273
274	3.1.1. Packing into bytes
275
276	This document does not address the issue of the order in which
277	bits of a byte are transmitted on a bit-sequential medium,
278	since the final data format described here is byte- rather than
279
280
281
282	Deutsch Informational [Page 5]
283
284	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
285
286
287	bit-oriented. However, we describe the compressed block format
288	in below, as a sequence of data elements of various bit
289	lengths, not a sequence of bytes. We must therefore specify
290	how to pack these data elements into bytes to form the final
291	compressed byte sequence:
292
293	* Data elements are packed into bytes in order of
294	increasing bit number within the byte, i.e., starting
295	with the least-significant bit of the byte.
296	* Data elements other than Huffman codes are packed
297	starting with the least-significant bit of the data
298	element.
299	* Huffman codes are packed starting with the most-
300	significant bit of the code.
301
302	In other words, if one were to print out the compressed data as
303	a sequence of bytes, starting with the first byte at the
304	right margin and proceeding to the left, with the most-
305	significant bit of each byte on the left as usual, one would be
306	able to parse the result from right to left, with fixed-width
307	elements in the correct MSB-to-LSB order and Huffman codes in
308	bit-reversed order (i.e., with the first bit of the code in the
309	relative LSB position).
310
311	3.2. Compressed block format
312
313	3.2.1. Synopsis of prefix and Huffman coding
314
315	Prefix coding represents symbols from an a priori known
316	alphabet by bit sequences (codes), one code for each symbol, in
317	a manner such that different symbols may be represented by bit
318	sequences of different lengths, but a parser can always parse
319	an encoded string unambiguously symbol-by-symbol.
320
321	We define a prefix code in terms of a binary tree in which the
322	two edges descending from each non-leaf node are labeled 0 and
323	1 and in which the leaf nodes correspond one-for-one with (are
324	labeled with) the symbols of the alphabet; then the code for a
325	symbol is the sequence of 0's and 1's on the edges leading from
326	the root to the leaf labeled with that symbol. For example:
327
328
329
330
331
332
333
334
335
336
337
338	Deutsch Informational [Page 6]
339
340	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
341
342
343	/\ Symbol Code
344	0 1 ------ ----
345	/ \ A 00
346	/\ B B 1
347	0 1 C 011
348	/ \ D 010
349	A /\
350	0 1
351	/ \
352	D C
353
354	A parser can decode the next symbol from an encoded input
355	stream by walking down the tree from the root, at each step
356	choosing the edge corresponding to the next input bit.
357
358	Given an alphabet with known symbol frequencies, the Huffman
359	algorithm allows the construction of an optimal prefix code
360	(one which represents strings with those symbol frequencies
361	using the fewest bits of any possible prefix codes for that
362	alphabet). Such a code is called a Huffman code. (See
363	reference [1] in Chapter 5, references for additional
364	information on Huffman codes.)
365
366	Note that in the "deflate" format, the Huffman codes for the
367	various alphabets must not exceed certain maximum code lengths.
368	This constraint complicates the algorithm for computing code
369	lengths from symbol frequencies. Again, see Chapter 5,
370	references for details.
371
372	3.2.2. Use of Huffman coding in the "deflate" format
373
374	The Huffman codes used for each alphabet in the "deflate"
375	format have two additional rules:
376
377	* All codes of a given bit length have lexicographically
378	consecutive values, in the same order as the symbols
379	they represent;
380
381	* Shorter codes lexicographically precede longer codes.
382
383
384
385
386
387
388
389
390
391
392
393
394	Deutsch Informational [Page 7]
395
396	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
397
398
399	We could recode the example above to follow this rule as
400	follows, assuming that the order of the alphabet is ABCD:
401
402	Symbol Code
403	------ ----
404	A 10
405	B 0
406	C 110
407	D 111
408
409	I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are
410	lexicographically consecutive.
411
412	Given this rule, we can define the Huffman code for an alphabet
413	just by giving the bit lengths of the codes for each symbol of
414	the alphabet in order; this is sufficient to determine the
415	actual codes. In our example, the code is completely defined
416	by the sequence of bit lengths (2, 1, 3, 3). The following
417	algorithm generates the codes as integers, intended to be read
418	from most- to least-significant bit. The code lengths are
419	initially in tree[I].Len; the codes are produced in
420	tree[I].Code.
421
422	1) Count the number of codes for each code length. Let
423	bl_count[N] be the number of codes of length N, N >= 1.
424
425	2) Find the numerical value of the smallest code for each
426	code length:
427
428	code = 0;
429	bl_count[0] = 0;
430	for (bits = 1; bits <= MAX_BITS; bits++) {
431	code = (code + bl_count[bits-1]) << 1;
432	next_code[bits] = code;
433	}
434
435	3) Assign numerical values to all codes, using consecutive
436	values for all codes of the same length with the base
437	values determined at step 2. Codes that are never used
438	(which have a bit length of zero) must not be assigned a
439	value.
440
441	for (n = 0; n <= max_code; n++) {
442	len = tree[n].Len;
443	if (len != 0) {
444	tree[n].Code = next_code[len];
445	next_code[len]++;
446	}
447
448
449
450	Deutsch Informational [Page 8]
451
452	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
453
454
455	}
456
457	Example:
458
459	Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3,
460	3, 2, 4, 4). After step 1, we have:
461
462	N bl_count[N]
463	- -----------
464	2 1
465	3 5
466	4 2
467
468	Step 2 computes the following next_code values:
469
470	N next_code[N]
471	- ------------
472	1 0
473	2 0
474	3 2
475	4 14
476
477	Step 3 produces the following code values:
478
479	Symbol Length Code
480	------ ------ ----
481	A 3 010
482	B 3 011
483	C 3 100
484	D 3 101
485	E 3 110
486	F 2 00
487	G 4 1110
488	H 4 1111
489
490	3.2.3. Details of block format
491
492	Each block of compressed data begins with 3 header bits
493	containing the following data:
494
495	first bit BFINAL
496	next 2 bits BTYPE
497
498	Note that the header bits do not necessarily begin on a byte
499	boundary, since a block does not necessarily occupy an integral
500	number of bytes.
501
502
503
504
505
506	Deutsch Informational [Page 9]
507
508	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
509
510
511	BFINAL is set if and only if this is the last block of the data
512	set.
513
514	BTYPE specifies how the data are compressed, as follows:
515
516	00 - no compression
517	01 - compressed with fixed Huffman codes
518	10 - compressed with dynamic Huffman codes
519	11 - reserved (error)
520
521	The only difference between the two compressed cases is how the
522	Huffman codes for the literal/length and distance alphabets are
523	defined.
524
525	In all cases, the decoding algorithm for the actual data is as
526	follows:
527
528	do
529	read block header from input stream.
530	if stored with no compression
531	skip any remaining bits in current partially
532	processed byte
533	read LEN and NLEN (see next section)
534	copy LEN bytes of data to output
535	otherwise
536	if compressed with dynamic Huffman codes
537	read representation of code trees (see
538	subsection below)
539	loop (until end of block code recognized)
540	decode literal/length value from input stream
541	if value < 256
542	copy value (literal byte) to output stream
543	otherwise
544	if value = end of block (256)
545	break from loop
546	otherwise (value = 257..285)
547	decode distance from input stream
548
549	move backwards distance bytes in the output
550	stream, and copy length bytes from this
551	position to the output stream.
552	end loop
553	while not last block
554
555	Note that a duplicated string reference may refer to a string
556	in a previous block; i.e., the backward distance may cross one
557	or more block boundaries. However a distance cannot refer past
558	the beginning of the output stream. (An application using a
559
560
561
562	Deutsch Informational [Page 10]
563
564	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
565
566
567	preset dictionary might discard part of the output stream; a
568	distance can refer to that part of the output stream anyway)
569	Note also that the referenced string may overlap the current
570	position; for example, if the last 2 bytes decoded have values
571	X and Y, a string reference with <length = 5, distance = 2>
572	adds X,Y,X,Y,X to the output stream.
573
574	We now specify each compression method in turn.
575
576	3.2.4. Non-compressed blocks (BTYPE=00)
577
578	Any bits of input up to the next byte boundary are ignored.
579	The rest of the block consists of the following information:
580
581	0 1 2 3 4...
582	+---+---+---+---+================================+
583	\| LEN \| NLEN \|... LEN bytes of literal data...\|
584	+---+---+---+---+================================+
585
586	LEN is the number of data bytes in the block. NLEN is the
587	one's complement of LEN.
588
589	3.2.5. Compressed blocks (length and distance codes)
590
591	As noted above, encoded data blocks in the "deflate" format
592	consist of sequences of symbols drawn from three conceptually
593	distinct alphabets: either literal bytes, from the alphabet of
594	byte values (0..255), or <length, backward distance> pairs,
595	where the length is drawn from (3..258) and the distance is
596	drawn from (1..32,768). In fact, the literal and length
597	alphabets are merged into a single alphabet (0..285), where
598	values 0..255 represent literal bytes, the value 256 indicates
599	end-of-block, and values 257..285 represent length codes
600	(possibly in conjunction with extra bits following the symbol
601	code) as follows:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618	Deutsch Informational [Page 11]
619
620	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
621
622
623	Extra Extra Extra
624	Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
625	---- ---- ------ ---- ---- ------- ---- ---- -------
626	257 0 3 267 1 15,16 277 4 67-82
627	258 0 4 268 1 17,18 278 4 83-98
628	259 0 5 269 2 19-22 279 4 99-114
629	260 0 6 270 2 23-26 280 4 115-130
630	261 0 7 271 2 27-30 281 5 131-162
631	262 0 8 272 2 31-34 282 5 163-194
632	263 0 9 273 3 35-42 283 5 195-226
633	264 0 10 274 3 43-50 284 5 227-257
634	265 1 11,12 275 3 51-58 285 0 258
635	266 1 13,14 276 3 59-66
636
637	The extra bits should be interpreted as a machine integer
638	stored with the most-significant bit first, e.g., bits 1110
639	represent the value 14.
640
641	Extra Extra Extra
642	Code Bits Dist Code Bits Dist Code Bits Distance
643	---- ---- ---- ---- ---- ------ ---- ---- --------
644	0 0 1 10 4 33-48 20 9 1025-1536
645	1 0 2 11 4 49-64 21 9 1537-2048
646	2 0 3 12 5 65-96 22 10 2049-3072
647	3 0 4 13 5 97-128 23 10 3073-4096
648	4 1 5,6 14 6 129-192 24 11 4097-6144
649	5 1 7,8 15 6 193-256 25 11 6145-8192
650	6 2 9-12 16 7 257-384 26 12 8193-12288
651	7 2 13-16 17 7 385-512 27 12 12289-16384
652	8 3 17-24 18 8 513-768 28 13 16385-24576
653	9 3 25-32 19 8 769-1024 29 13 24577-32768
654
655	3.2.6. Compression with fixed Huffman codes (BTYPE=01)
656
657	The Huffman codes for the two alphabets are fixed, and are not
658	represented explicitly in the data. The Huffman code lengths
659	for the literal/length alphabet are:
660
661	Lit Value Bits Codes
662	--------- ---- -----
663	0 - 143 8 00110000 through
664	10111111
665	144 - 255 9 110010000 through
666	111111111
667	256 - 279 7 0000000 through
668	0010111
669	280 - 287 8 11000000 through
670	11000111
671
672
673
674	Deutsch Informational [Page 12]
675
676	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
677
678
679	The code lengths are sufficient to generate the actual codes,
680	as described above; we show the codes in the table for added
681	clarity. Literal/length values 286-287 will never actually
682	occur in the compressed data, but participate in the code
683	construction.
684
685	Distance codes 0-31 are represented by (fixed-length) 5-bit
686	codes, with possible additional bits as shown in the table
687	shown in Paragraph 3.2.5, above. Note that distance codes 30-
688	31 will never actually occur in the compressed data.
689
690	3.2.7. Compression with dynamic Huffman codes (BTYPE=10)
691
692	The Huffman codes for the two alphabets appear in the block
693	immediately after the header bits and before the actual
694	compressed data, first the literal/length code and then the
695	distance code. Each code is defined by a sequence of code
696	lengths, as discussed in Paragraph 3.2.2, above. For even
697	greater compactness, the code length sequences themselves are
698	compressed using a Huffman code. The alphabet for code lengths
699	is as follows:
700
701	0 - 15: Represent code lengths of 0 - 15
702	16: Copy the previous code length 3 - 6 times.
703	The next 2 bits indicate repeat length
704	(0 = 3, ... , 3 = 6)
705	Example: Codes 8, 16 (+2 bits 11),
706	16 (+2 bits 10) will expand to
707	12 code lengths of 8 (1 + 6 + 5)
708	17: Repeat a code length of 0 for 3 - 10 times.
709	(3 bits of length)
710	18: Repeat a code length of 0 for 11 - 138 times
711	(7 bits of length)
712
713	A code length of 0 indicates that the corresponding symbol in
714	the literal/length or distance alphabet will not occur in the
715	block, and should not participate in the Huffman code
716	construction algorithm given earlier. If only one distance
717	code is used, it is encoded using one bit, not zero bits; in
718	this case there is a single code length of one, with one unused
719	code. One distance code of zero bits means that there are no
720	distance codes used at all (the data is all literals).
721
722	We can now define the format of the block:
723
724	5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286)
725	5 Bits: HDIST, # of Distance codes - 1 (1 - 32)
726	4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19)
727
728
729
730	Deutsch Informational [Page 13]
731
732	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
733
734
735	(HCLEN + 4) x 3 bits: code lengths for the code length
736	alphabet given just above, in the order: 16, 17, 18,
737	0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
738
739	These code lengths are interpreted as 3-bit integers
740	(0-7); as above, a code length of 0 means the
741	corresponding symbol (literal/length or distance code
742	length) is not used.
743
744	HLIT + 257 code lengths for the literal/length alphabet,
745	encoded using the code length Huffman code
746
747	HDIST + 1 code lengths for the distance alphabet,
748	encoded using the code length Huffman code
749
750	The actual compressed data of the block,
751	encoded using the literal/length and distance Huffman
752	codes
753
754	The literal/length symbol 256 (end of data),
755	encoded using the literal/length Huffman code
756
757	The code length repeat codes can cross from HLIT + 257 to the
758	HDIST + 1 code lengths. In other words, all code lengths form
759	a single sequence of HLIT + HDIST + 258 values.
760
761	3.3. Compliance
762
763	A compressor may limit further the ranges of values specified in
764	the previous section and still be compliant; for example, it may
765	limit the range of backward pointers to some value smaller than
766	32K. Similarly, a compressor may limit the size of blocks so that
767	a compressible block fits in memory.
768
769	A compliant decompressor must accept the full range of possible
770	values defined in the previous section, and must accept blocks of
771	arbitrary size.
772
773	4. Compression algorithm details
774
775	While it is the intent of this document to define the "deflate"
776	compressed data format without reference to any particular
777	compression algorithm, the format is related to the compressed
778	formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below);
779	since many variations of LZ77 are patented, it is strongly
780	recommended that the implementor of a compressor follow the general
781	algorithm presented here, which is known not to be patented per se.
782	The material in this section is not part of the definition of the
783
784
785
786	Deutsch Informational [Page 14]
787
788	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
789
790
791	specification per se, and a compressor need not follow it in order to
792	be compliant.
793
794	The compressor terminates a block when it determines that starting a
795	new block with fresh trees would be useful, or when the block size
796	fills up the compressor's block buffer.
797
798	The compressor uses a chained hash table to find duplicated strings,
799	using a hash function that operates on 3-byte sequences. At any
800	given point during compression, let XYZ be the next 3 input bytes to
801	be examined (not necessarily all different, of course). First, the
802	compressor examines the hash chain for XYZ. If the chain is empty,
803	the compressor simply writes out X as a literal byte and advances one
804	byte in the input. If the hash chain is not empty, indicating that
805	the sequence XYZ (or, if we are unlucky, some other 3 bytes with the
806	same hash function value) has occurred recently, the compressor
807	compares all strings on the XYZ hash chain with the actual input data
808	sequence starting at the current point, and selects the longest
809	match.
810
811	The compressor searches the hash chains starting with the most recent
812	strings, to favor small distances and thus take advantage of the
813	Huffman encoding. The hash chains are singly linked. There are no
814	deletions from the hash chains; the algorithm simply discards matches
815	that are too old. To avoid a worst-case situation, very long hash
816	chains are arbitrarily truncated at a certain length, determined by a
817	run-time parameter.
818
819	To improve overall compression, the compressor optionally defers the
820	selection of matches ("lazy matching"): after a match of length N has
821	been found, the compressor searches for a longer match starting at
822	the next input byte. If it finds a longer match, it truncates the
823	previous match to a length of one (thus producing a single literal
824	byte) and then emits the longer match. Otherwise, it emits the
825	original match, and, as described above, advances N bytes before
826	continuing.
827
828	Run-time parameters also control this "lazy match" procedure. If
829	compression ratio is most important, the compressor attempts a
830	complete second search regardless of the length of the first match.
831	In the normal case, if the current match is "long enough", the
832	compressor reduces the search for a longer match, thus speeding up
833	the process. If speed is most important, the compressor inserts new
834	strings in the hash table only when no match was found, or when the
835	match is not "too long". This degrades the compression ratio but
836	saves time since there are both fewer insertions and fewer searches.
837
838
839
840
841
842	Deutsch Informational [Page 15]
843
844	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
845
846
847	5. References
848
849	[1] Huffman, D. A., "A Method for the Construction of Minimum
850	Redundancy Codes", Proceedings of the Institute of Radio
851	Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101.
852
853	[2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data
854	Compression", IEEE Transactions on Information Theory, Vol. 23,
855	No. 3, pp. 337-343.
856
857	[3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources,
858	available in ftp://ftp.uu.net/pub/archiving/zip/doc/
859
860	[4] Gailly, J.-L., and Adler, M., GZIP documentation and sources,
861	available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/
862
863	[5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix
864	encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169.
865
866	[6] Hirschberg and Lelewer, "Efficient decoding of prefix codes,"
867	Comm. ACM, 33,4, April 1990, pp. 449-459.
868
869	6. Security Considerations
870
871	Any data compression method involves the reduction of redundancy in
872	the data. Consequently, any corruption of the data is likely to have
873	severe effects and be difficult to correct. Uncompressed text, on
874	the other hand, will probably still be readable despite the presence
875	of some corrupted bytes.
876
877	It is recommended that systems using this data format provide some
878	means of validating the integrity of the compressed data. See
879	reference [3], for example.
880
881	7. Source code
882
883	Source code for a C language implementation of a "deflate" compliant
884	compressor and decompressor is available within the zlib package at
885	ftp://ftp.uu.net/pub/archiving/zip/zlib/.
886
887	8. Acknowledgements
888
889	Trademarks cited in this document are the property of their
890	respective owners.
891
892	Phil Katz designed the deflate format. Jean-Loup Gailly and Mark
893	Adler wrote the related software described in this specification.
894	Glenn Randers-Pehrson converted this document to RFC and HTML format.
895
896
897
898	Deutsch Informational [Page 16]
899
900	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
901
902
903	9. Author's Address
904
905	L. Peter Deutsch
906	Aladdin Enterprises
907	203 Santa Margarita Ave.
908	Menlo Park, CA 94025
909
910	Phone: (415) 322-0103 (AM only)
911	FAX: (415) 322-1734
912	EMail: <ghost@aladdin.com>
913
914	Questions about the technical content of this specification can be
915	sent by email to:
916
917	Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
918	Mark Adler <madler@alumni.caltech.edu>
919
920	Editorial comments on this specification can be sent by email to:
921
922	L. Peter Deutsch <ghost@aladdin.com> and
923	Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954	Deutsch Informational [Page 17]
955

lib/std/compress/gzip.zig+48-364

...	@@ -1,382 +1,66 @@	...	@@ -1,382 +1,66 @@
1	//	1	const deflate = @import("flate/deflate.zig");
2	// Compressor/Decompressor for GZIP data streams (RFC1952)	2	const inflate = @import("flate/inflate.zig");
3		3
4	const std = @import("../std.zig");	4	/// Decompress compressed data from reader and write plain data to the writer.
5	const io = std.io;	5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	const fs = std.fs;	6	try inflate.decompress(.gzip, reader, writer);
7	const testing = std.testing;
8	const mem = std.mem;
9	const deflate = std.compress.deflate;
10
11	const magic = &[2]u8{ 0x1f, 0x8b };
12
13	// Flags for the FLG field in the header
14	const FTEXT = 1 << 0;
15	const FHCRC = 1 << 1;
16	const FEXTRA = 1 << 2;
17	const FNAME = 1 << 3;
18	const FCOMMENT = 1 << 4;
19
20	const max_string_len = 1024;
21
22	pub const Header = struct {
23	extra: ?[]const u8 = null,
24	filename: ?[]const u8 = null,
25	comment: ?[]const u8 = null,
26	modification_time: u32 = 0,
27	operating_system: u8 = 255,
28	};
29
30	pub fn Decompress(comptime ReaderType: type) type {
31	return struct {
32	const Self = @This();
33
34	pub const Error = ReaderType.Error \|\|
35	deflate.Decompressor(ReaderType).Error \|\|
36	error{ CorruptedData, WrongChecksum };
37	pub const Reader = io.Reader(*Self, Error, read);
38
39	allocator: mem.Allocator,
40	inflater: deflate.Decompressor(ReaderType),
41	in_reader: ReaderType,
42	hasher: std.hash.Crc32,
43	read_amt: u32,
44
45	info: Header,
46
47	fn init(allocator: mem.Allocator, in_reader: ReaderType) !Self {
48	var hasher = std.compress.hashedReader(in_reader, std.hash.Crc32.init());
49	const hashed_reader = hasher.reader();
50
51	// gzip header format is specified in RFC1952
52	const header = try hashed_reader.readBytesNoEof(10);
53
54	// Check the ID1/ID2 fields
55	if (!std.mem.eql(u8, header[0..2], magic))
56	return error.BadHeader;
57
58	const CM = header[2];
59	// The CM field must be 8 to indicate the use of DEFLATE
60	if (CM != 8) return error.InvalidCompression;
61	// Flags
62	const FLG = header[3];
63	// Modification time, as a Unix timestamp.
64	// If zero there's no timestamp available.
65	const MTIME = mem.readInt(u32, header[4..8], .little);
66	// Extra flags
67	const XFL = header[8];
68	// Operating system where the compression took place
69	const OS = header[9];
70	_ = XFL;
71
72	const extra = if (FLG & FEXTRA != 0) blk: {
73	const len = try hashed_reader.readInt(u16, .little);
74	const tmp_buf = try allocator.alloc(u8, len);
75	errdefer allocator.free(tmp_buf);
76
77	try hashed_reader.readNoEof(tmp_buf);
78	break :blk tmp_buf;
79	} else null;
80	errdefer if (extra) \|p\| allocator.free(p);
81
82	const filename = if (FLG & FNAME != 0)
83	try hashed_reader.readUntilDelimiterAlloc(allocator, 0, max_string_len)
84	else
85	null;
86	errdefer if (filename) \|p\| allocator.free(p);
87
88	const comment = if (FLG & FCOMMENT != 0)
89	try hashed_reader.readUntilDelimiterAlloc(allocator, 0, max_string_len)
90	else
91	null;
92	errdefer if (comment) \|p\| allocator.free(p);
93
94	if (FLG & FHCRC != 0) {
95	const hash = try in_reader.readInt(u16, .little);
96	if (hash != @as(u16, @truncate(hasher.hasher.final())))
97	return error.WrongChecksum;
98	}
99
100	return .{
101	.allocator = allocator,
102	.inflater = try deflate.decompressor(allocator, in_reader, null),
103	.in_reader = in_reader,
104	.hasher = std.hash.Crc32.init(),
105	.info = .{
106	.filename = filename,
107	.comment = comment,
108	.extra = extra,
109	.modification_time = MTIME,
110	.operating_system = OS,
111	},
112	.read_amt = 0,
113	};
114	}
115
116	pub fn deinit(self: *Self) void {
117	self.inflater.deinit();
118	if (self.info.extra) \|extra\|
119	self.allocator.free(extra);
120	if (self.info.filename) \|filename\|
121	self.allocator.free(filename);
122	if (self.info.comment) \|comment\|
123	self.allocator.free(comment);
124	}
125
126	/// Implements the io.Reader interface
127	pub fn read(self: *Self, buffer: []u8) Error!usize {
128	if (buffer.len == 0)
129	return 0;
130
131	// Read from the compressed stream and update the computed checksum
132	const r = try self.inflater.read(buffer);
133	if (r != 0) {
134	self.hasher.update(buffer[0..r]);
135	self.read_amt +%= @truncate(r);
136	return r;
137	}
138
139	try self.inflater.close();
140
141	// We've reached the end of stream, check if the checksum matches
142	const hash = try self.in_reader.readInt(u32, .little);
143	if (hash != self.hasher.final())
144	return error.WrongChecksum;
145
146	// The ISIZE field is the size of the uncompressed input modulo 2^32
147	const input_size = try self.in_reader.readInt(u32, .little);
148	if (self.read_amt != input_size)
149	return error.CorruptedData;
150
151	return 0;
152	}
153
154	pub fn reader(self: *Self) Reader {
155	return .{ .context = self };
156	}
157	};
158	}	7	}
159		8
160	pub fn decompress(allocator: mem.Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {	9	/// Decompressor type
161	return Decompress(@TypeOf(reader)).init(allocator, reader);	10	pub fn Decompressor(comptime ReaderType: type) type {
		11	return inflate.Inflate(.gzip, ReaderType);
162	}	12	}
163		13
164	pub const CompressOptions = struct {	14	/// Create Decompressor which will read compressed data from reader.
165	header: Header = .{},	15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
166	hash_header: bool = true,	16	return inflate.decompressor(.gzip, reader);
167	level: deflate.Compression = .default_compression,	17	}
168	};
169
170	pub fn Compress(comptime WriterType: type) type {
171	return struct {
172	const Self = @This();
173
174	pub const Error = WriterType.Error \|\|
175	deflate.Compressor(WriterType).Error;
176	pub const Writer = io.Writer(*Self, Error, write);
177
178	allocator: mem.Allocator,
179	deflater: deflate.Compressor(WriterType),
180	out_writer: WriterType,
181	hasher: std.hash.Crc32,
182	write_amt: u32,
183
184	fn init(allocator: mem.Allocator, out_writer: WriterType, options: CompressOptions) !Self {
185	var hasher = std.compress.hashedWriter(out_writer, std.hash.Crc32.init());
186	const hashed_writer = hasher.writer();
187
188	// ID1/ID2
189	try hashed_writer.writeAll(magic);
190	// CM
191	try hashed_writer.writeByte(8);
192	// Flags
193	try hashed_writer.writeByte(
194	@as(u8, if (options.hash_header) FHCRC else 0) \|
195	@as(u8, if (options.header.extra) \|_\| FEXTRA else 0) \|
196	@as(u8, if (options.header.filename) \|_\| FNAME else 0) \|
197	@as(u8, if (options.header.comment) \|_\| FCOMMENT else 0),
198	);
199	// Modification time
200	try hashed_writer.writeInt(u32, options.header.modification_time, .little);
201	// Extra flags
202	try hashed_writer.writeByte(0);
203	// Operating system
204	try hashed_writer.writeByte(options.header.operating_system);
205
206	if (options.header.extra) \|extra\| {
207	try hashed_writer.writeInt(u16, @intCast(extra.len), .little);
208	try hashed_writer.writeAll(extra);
209	}
210
211	if (options.header.filename) \|filename\| {
212	try hashed_writer.writeAll(filename);
213	try hashed_writer.writeByte(0);
214	}
215
216	if (options.header.comment) \|comment\| {
217	try hashed_writer.writeAll(comment);
218	try hashed_writer.writeByte(0);
219	}
220
221	if (options.hash_header) {
222	try out_writer.writeInt(
223	u16,
224	@truncate(hasher.hasher.final()),
225	.little,
226	);
227	}
228
229	return .{
230	.allocator = allocator,
231	.deflater = try deflate.compressor(allocator, out_writer, .{ .level = options.level }),
232	.out_writer = out_writer,
233	.hasher = std.hash.Crc32.init(),
234	.write_amt = 0,
235	};
236	}
237
238	pub fn deinit(self: *Self) void {
239	self.deflater.deinit();
240	}
241
242	/// Implements the io.Writer interface
243	pub fn write(self: *Self, buffer: []const u8) Error!usize {
244	if (buffer.len == 0)
245	return 0;
246
247	// Write to the compressed stream and update the computed checksum
248	const r = try self.deflater.write(buffer);
249	self.hasher.update(buffer[0..r]);
250	self.write_amt +%= @truncate(r);
251	return r;
252	}
253
254	pub fn writer(self: *Self) Writer {
255	return .{ .context = self };
256	}
257		18
258	pub fn flush(self: *Self) Error!void {	19	/// Compression level, trades between speed and compression size.
259	try self.deflater.flush();	20	pub const Options = deflate.Options;
260	}
261		21
262	pub fn close(self: *Self) Error!void {	22	/// Compress plain data from reader and write compressed data to the writer.
263	try self.deflater.close();	23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
264	try self.out_writer.writeInt(u32, self.hasher.final(), .little);	24	try deflate.compress(.gzip, reader, writer, options);
265	try self.out_writer.writeInt(u32, self.write_amt, .little);
266	}
267	};
268	}	25	}
269		26
270	pub fn compress(allocator: mem.Allocator, writer: anytype, options: CompressOptions) !Compress(@TypeOf(writer)) {	27	/// Compressor type
271	return Compress(@TypeOf(writer)).init(allocator, writer, options);	28	pub fn Compressor(comptime WriterType: type) type {
		29	return deflate.Compressor(.gzip, WriterType);
272	}	30	}
273		31
274	fn testReader(expected: []const u8, data: []const u8) !void {	32	/// Create Compressor which outputs compressed data to the writer.
275	var in_stream = io.fixedBufferStream(data);	33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
276		34	return try deflate.compressor(.gzip, writer, options);
277	var gzip_stream = try decompress(testing.allocator, in_stream.reader());
278	defer gzip_stream.deinit();
279
280	// Read and decompress the whole file
281	const buf = try gzip_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
282	defer testing.allocator.free(buf);
283
284	// Check against the reference
285	try testing.expectEqualSlices(u8, expected, buf);
286	}	35	}
287		36
288	fn testWriter(expected: []const u8, data: []const u8, options: CompressOptions) !void {	37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
289	var actual = std.ArrayList(u8).init(testing.allocator);	38	/// compression, less memory requirements but bigger compressed sizes.
290	defer actual.deinit();	39	pub const huffman = struct {
		40	pub fn compress(reader: anytype, writer: anytype) !void {
		41	try deflate.huffman.compress(.gzip, reader, writer);
		42	}
291		43
292	var gzip_stream = try compress(testing.allocator, actual.writer(), options);	44	pub fn Compressor(comptime WriterType: type) type {
293	defer gzip_stream.deinit();	45	return deflate.huffman.Compressor(.gzip, WriterType);
294		46	}
295	// Write and compress the whole file
296	try gzip_stream.writer().writeAll(data);
297	try gzip_stream.close();
298
299	// Check against the reference
300	try testing.expectEqualSlices(u8, expected, actual.items);
301	}
302		47
303	// All the test cases are obtained by compressing the RFC1952 text	48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
304	//	49	return deflate.huffman.compressor(.gzip, writer);
305	// https://tools.ietf.org/rfc/rfc1952.txt length=25037 bytes	50	}
306	// SHA256=164ef0897b4cbec63abf1b57f069f3599bd0fb7c72c2a4dee21bd7e03ec9af67	51	};
307	test "compressed data" {
308	const plain = @embedFile("testdata/rfc1952.txt");
309	const compressed = @embedFile("testdata/rfc1952.txt.gz");
310	try testReader(plain, compressed);
311	try testWriter(compressed, plain, .{
312	.header = .{
313	.filename = "rfc1952.txt",
314	.modification_time = 1706533053,
315	.operating_system = 3,
316	},
317	});
318	}
319
320	test "sanity checks" {
321	// Truncated header
322	try testing.expectError(
323	error.EndOfStream,
324	testReader(undefined, &[_]u8{ 0x1f, 0x8B }),
325	);
326	// Wrong CM
327	try testing.expectError(
328	error.InvalidCompression,
329	testReader(undefined, &[_]u8{
330	0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
331	0x00, 0x03,
332	}),
333	);
334	// Wrong checksum
335	try testing.expectError(
336	error.WrongChecksum,
337	testReader(undefined, &[_]u8{
338	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
339	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
340	0x00, 0x00, 0x00, 0x00,
341	}),
342	);
343	// Truncated checksum
344	try testing.expectError(
345	error.EndOfStream,
346	testReader(undefined, &[_]u8{
347	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
348	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
349	}),
350	);
351	// Wrong initial size
352	try testing.expectError(
353	error.CorruptedData,
354	testReader(undefined, &[_]u8{
355	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
356	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
357	0x00, 0x00, 0x00, 0x01,
358	}),
359	);
360	// Truncated initial size field
361	try testing.expectError(
362	error.EndOfStream,
363	testReader(undefined, &[_]u8{
364	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
365	0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
366	0x00, 0x00, 0x00,
367	}),
368	);
369	}
370		52
371	test "header checksum" {	53	// No compression store only. Compressed size is slightly bigger than plain.
372	try testReader("", &[_]u8{	54	pub const store = struct {
373	// GZIP header	55	pub fn compress(reader: anytype, writer: anytype) !void {
374	0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,	56	try deflate.store.compress(.gzip, reader, writer);
		57	}
375		58
376	// header.FHCRC (should cover entire header)	59	pub fn Compressor(comptime WriterType: type) type {
377	0x99, 0xd6,	60	return deflate.store.Compressor(.gzip, WriterType);
		61	}
378		62
379	// GZIP data	63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
380	0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,	64	return deflate.store.compressor(.gzip, writer);
381	});	65	}
382	}	66	};

lib/std/compress/testdata/rfc1951.txt deleted-955

...	@@ -1,955 +0,0 @@
1
2
3
4
5
6
7	Network Working Group P. Deutsch
8	Request for Comments: 1951 Aladdin Enterprises
9	Category: Informational May 1996
10
11
12	DEFLATE Compressed Data Format Specification version 1.3
13
14	Status of This Memo
15
16	This memo provides information for the Internet community. This memo
17	does not specify an Internet standard of any kind. Distribution of
18	this memo is unlimited.
19
20	IESG Note:
21
22	The IESG takes no position on the validity of any Intellectual
23	Property Rights statements contained in this document.
24
25	Notices
26
27	Copyright (c) 1996 L. Peter Deutsch
28
29	Permission is granted to copy and distribute this document for any
30	purpose and without charge, including translations into other
31	languages and incorporation into compilations, provided that the
32	copyright notice and this notice are preserved, and that any
33	substantive changes or deletions from the original are clearly
34	marked.
35
36	A pointer to the latest version of this and related documentation in
37	HTML format can be found at the URL
38	<ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40	Abstract
41
42	This specification defines a lossless compressed data format that
43	compresses data using a combination of the LZ77 algorithm and Huffman
44	coding, with efficiency comparable to the best currently available
45	general-purpose compression methods. The data can be produced or
46	consumed, even for an arbitrarily long sequentially presented input
47	data stream, using only an a priori bounded amount of intermediate
48	storage. The format can be implemented readily in a manner not
49	covered by patents.
50
51
52
53
54
55
56
57
58	Deutsch Informational [Page 1]
59
60	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
61
62
63	Table of Contents
64
65	1. Introduction ................................................... 2
66	1.1. Purpose ................................................... 2
67	1.2. Intended audience ......................................... 3
68	1.3. Scope ..................................................... 3
69	1.4. Compliance ................................................ 3
70	1.5. Definitions of terms and conventions used ................ 3
71	1.6. Changes from previous versions ............................ 4
72	2. Compressed representation overview ............................. 4
73	3. Detailed specification ......................................... 5
74	3.1. Overall conventions ....................................... 5
75	3.1.1. Packing into bytes .................................. 5
76	3.2. Compressed block format ................................... 6
77	3.2.1. Synopsis of prefix and Huffman coding ............... 6
78	3.2.2. Use of Huffman coding in the "deflate" format ....... 7
79	3.2.3. Details of block format ............................. 9
80	3.2.4. Non-compressed blocks (BTYPE=00) ................... 11
81	3.2.5. Compressed blocks (length and distance codes) ...... 11
82	3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12
83	3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13
84	3.3. Compliance ............................................... 14
85	4. Compression algorithm details ................................. 14
86	5. References .................................................... 16
87	6. Security Considerations ....................................... 16
88	7. Source code ................................................... 16
89	8. Acknowledgements .............................................. 16
90	9. Author's Address .............................................. 17
91
92	1. Introduction
93
94	1.1. Purpose
95
96	The purpose of this specification is to define a lossless
97	compressed data format that:
98	* Is independent of CPU type, operating system, file system,
99	and character set, and hence can be used for interchange;
100	* Can be produced or consumed, even for an arbitrarily long
101	sequentially presented input data stream, using only an a
102	priori bounded amount of intermediate storage, and hence
103	can be used in data communications or similar structures
104	such as Unix filters;
105	* Compresses data with efficiency comparable to the best
106	currently available general-purpose compression methods,
107	and in particular considerably better than the "compress"
108	program;
109	* Can be implemented readily in a manner not covered by
110	patents, and hence can be practiced freely;
111
112
113
114	Deutsch Informational [Page 2]
115
116	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
117
118
119	* Is compatible with the file format produced by the current
120	widely used gzip utility, in that conforming decompressors
121	will be able to read data produced by the existing gzip
122	compressor.
123
124	The data format defined by this specification does not attempt to:
125
126	* Allow random access to compressed data;
127	* Compress specialized data (e.g., raster graphics) as well
128	as the best currently available specialized algorithms.
129
130	A simple counting argument shows that no lossless compression
131	algorithm can compress every possible input data set. For the
132	format defined here, the worst case expansion is 5 bytes per 32K-
133	byte block, i.e., a size increase of 0.015% for large data sets.
134	English text usually compresses by a factor of 2.5 to 3;
135	executable files usually compress somewhat less; graphical data
136	such as raster images may compress much more.
137
138	1.2. Intended audience
139
140	This specification is intended for use by implementors of software
141	to compress data into "deflate" format and/or decompress data from
142	"deflate" format.
143
144	The text of the specification assumes a basic background in
145	programming at the level of bits and other primitive data
146	representations. Familiarity with the technique of Huffman coding
147	is helpful but not required.
148
149	1.3. Scope
150
151	The specification specifies a method for representing a sequence
152	of bytes as a (usually shorter) sequence of bits, and a method for
153	packing the latter bit sequence into bytes.
154
155	1.4. Compliance
156
157	Unless otherwise indicated below, a compliant decompressor must be
158	able to accept and decompress any data set that conforms to all
159	the specifications presented here; a compliant compressor must
160	produce data sets that conform to all the specifications presented
161	here.
162
163	1.5. Definitions of terms and conventions used
164
165	Byte: 8 bits stored or transmitted as a unit (same as an octet).
166	For this specification, a byte is exactly 8 bits, even on machines
167
168
169
170	Deutsch Informational [Page 3]
171
172	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
173
174
175	which store a character on a number of bits different from eight.
176	See below, for the numbering of bits within a byte.
177
178	String: a sequence of arbitrary bytes.
179
180	1.6. Changes from previous versions
181
182	There have been no technical changes to the deflate format since
183	version 1.1 of this specification. In version 1.2, some
184	terminology was changed. Version 1.3 is a conversion of the
185	specification to RFC style.
186
187	2. Compressed representation overview
188
189	A compressed data set consists of a series of blocks, corresponding
190	to successive blocks of input data. The block sizes are arbitrary,
191	except that non-compressible blocks are limited to 65,535 bytes.
192
193	Each block is compressed using a combination of the LZ77 algorithm
194	and Huffman coding. The Huffman trees for each block are independent
195	of those for previous or subsequent blocks; the LZ77 algorithm may
196	use a reference to a duplicated string occurring in a previous block,
197	up to 32K input bytes before.
198
199	Each block consists of two parts: a pair of Huffman code trees that
200	describe the representation of the compressed data part, and a
201	compressed data part. (The Huffman trees themselves are compressed
202	using Huffman encoding.) The compressed data consists of a series of
203	elements of two types: literal bytes (of strings that have not been
204	detected as duplicated within the previous 32K input bytes), and
205	pointers to duplicated strings, where a pointer is represented as a
206	pair <length, backward distance>. The representation used in the
207	"deflate" format limits distances to 32K bytes and lengths to 258
208	bytes, but does not limit the size of a block, except for
209	uncompressible blocks, which are limited as noted above.
210
211	Each type of value (literals, distances, and lengths) in the
212	compressed data is represented using a Huffman code, using one code
213	tree for literals and lengths and a separate code tree for distances.
214	The code trees for each block appear in a compact form just before
215	the compressed data for that block.
216
217
218
219
220
221
222
223
224
225
226	Deutsch Informational [Page 4]
227
228	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
229
230
231	3. Detailed specification
232
233	3.1. Overall conventions In the diagrams below, a box like this:
234
235	+---+
236	\| \| <-- the vertical bars might be missing
237	+---+
238
239	represents one byte; a box like this:
240
241	+==============+
242	\| \|
243	+==============+
244
245	represents a variable number of bytes.
246
247	Bytes stored within a computer do not have a "bit order", since
248	they are always treated as a unit. However, a byte considered as
249	an integer between 0 and 255 does have a most- and least-
250	significant bit, and since we write numbers with the most-
251	significant digit on the left, we also write bytes with the most-
252	significant bit on the left. In the diagrams below, we number the
253	bits of a byte so that bit 0 is the least-significant bit, i.e.,
254	the bits are numbered:
255
256	+--------+
257	\|76543210\|
258	+--------+
259
260	Within a computer, a number may occupy multiple bytes. All
261	multi-byte numbers in the format described here are stored with
262	the least-significant byte first (at the lower memory address).
263	For example, the decimal number 520 is stored as:
264
265	0 1
266	+--------+--------+
267	\|00001000\|00000010\|
268	+--------+--------+
269	^ ^
270	\| \|
271	\| + more significant byte = 2 x 256
272	+ less significant byte = 8
273
274	3.1.1. Packing into bytes
275
276	This document does not address the issue of the order in which
277	bits of a byte are transmitted on a bit-sequential medium,
278	since the final data format described here is byte- rather than
279
280
281
282	Deutsch Informational [Page 5]
283
284	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
285
286
287	bit-oriented. However, we describe the compressed block format
288	in below, as a sequence of data elements of various bit
289	lengths, not a sequence of bytes. We must therefore specify
290	how to pack these data elements into bytes to form the final
291	compressed byte sequence:
292
293	* Data elements are packed into bytes in order of
294	increasing bit number within the byte, i.e., starting
295	with the least-significant bit of the byte.
296	* Data elements other than Huffman codes are packed
297	starting with the least-significant bit of the data
298	element.
299	* Huffman codes are packed starting with the most-
300	significant bit of the code.
301
302	In other words, if one were to print out the compressed data as
303	a sequence of bytes, starting with the first byte at the
304	right margin and proceeding to the left, with the most-
305	significant bit of each byte on the left as usual, one would be
306	able to parse the result from right to left, with fixed-width
307	elements in the correct MSB-to-LSB order and Huffman codes in
308	bit-reversed order (i.e., with the first bit of the code in the
309	relative LSB position).
310
311	3.2. Compressed block format
312
313	3.2.1. Synopsis of prefix and Huffman coding
314
315	Prefix coding represents symbols from an a priori known
316	alphabet by bit sequences (codes), one code for each symbol, in
317	a manner such that different symbols may be represented by bit
318	sequences of different lengths, but a parser can always parse
319	an encoded string unambiguously symbol-by-symbol.
320
321	We define a prefix code in terms of a binary tree in which the
322	two edges descending from each non-leaf node are labeled 0 and
323	1 and in which the leaf nodes correspond one-for-one with (are
324	labeled with) the symbols of the alphabet; then the code for a
325	symbol is the sequence of 0's and 1's on the edges leading from
326	the root to the leaf labeled with that symbol. For example:
327
328
329
330
331
332
333
334
335
336
337
338	Deutsch Informational [Page 6]
339
340	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
341
342
343	/\ Symbol Code
344	0 1 ------ ----
345	/ \ A 00
346	/\ B B 1
347	0 1 C 011
348	/ \ D 010
349	A /\
350	0 1
351	/ \
352	D C
353
354	A parser can decode the next symbol from an encoded input
355	stream by walking down the tree from the root, at each step
356	choosing the edge corresponding to the next input bit.
357
358	Given an alphabet with known symbol frequencies, the Huffman
359	algorithm allows the construction of an optimal prefix code
360	(one which represents strings with those symbol frequencies
361	using the fewest bits of any possible prefix codes for that
362	alphabet). Such a code is called a Huffman code. (See
363	reference [1] in Chapter 5, references for additional
364	information on Huffman codes.)
365
366	Note that in the "deflate" format, the Huffman codes for the
367	various alphabets must not exceed certain maximum code lengths.
368	This constraint complicates the algorithm for computing code
369	lengths from symbol frequencies. Again, see Chapter 5,
370	references for details.
371
372	3.2.2. Use of Huffman coding in the "deflate" format
373
374	The Huffman codes used for each alphabet in the "deflate"
375	format have two additional rules:
376
377	* All codes of a given bit length have lexicographically
378	consecutive values, in the same order as the symbols
379	they represent;
380
381	* Shorter codes lexicographically precede longer codes.
382
383
384
385
386
387
388
389
390
391
392
393
394	Deutsch Informational [Page 7]
395
396	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
397
398
399	We could recode the example above to follow this rule as
400	follows, assuming that the order of the alphabet is ABCD:
401
402	Symbol Code
403	------ ----
404	A 10
405	B 0
406	C 110
407	D 111
408
409	I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are
410	lexicographically consecutive.
411
412	Given this rule, we can define the Huffman code for an alphabet
413	just by giving the bit lengths of the codes for each symbol of
414	the alphabet in order; this is sufficient to determine the
415	actual codes. In our example, the code is completely defined
416	by the sequence of bit lengths (2, 1, 3, 3). The following
417	algorithm generates the codes as integers, intended to be read
418	from most- to least-significant bit. The code lengths are
419	initially in tree[I].Len; the codes are produced in
420	tree[I].Code.
421
422	1) Count the number of codes for each code length. Let
423	bl_count[N] be the number of codes of length N, N >= 1.
424
425	2) Find the numerical value of the smallest code for each
426	code length:
427
428	code = 0;
429	bl_count[0] = 0;
430	for (bits = 1; bits <= MAX_BITS; bits++) {
431	code = (code + bl_count[bits-1]) << 1;
432	next_code[bits] = code;
433	}
434
435	3) Assign numerical values to all codes, using consecutive
436	values for all codes of the same length with the base
437	values determined at step 2. Codes that are never used
438	(which have a bit length of zero) must not be assigned a
439	value.
440
441	for (n = 0; n <= max_code; n++) {
442	len = tree[n].Len;
443	if (len != 0) {
444	tree[n].Code = next_code[len];
445	next_code[len]++;
446	}
447
448
449
450	Deutsch Informational [Page 8]
451
452	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
453
454
455	}
456
457	Example:
458
459	Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3,
460	3, 2, 4, 4). After step 1, we have:
461
462	N bl_count[N]
463	- -----------
464	2 1
465	3 5
466	4 2
467
468	Step 2 computes the following next_code values:
469
470	N next_code[N]
471	- ------------
472	1 0
473	2 0
474	3 2
475	4 14
476
477	Step 3 produces the following code values:
478
479	Symbol Length Code
480	------ ------ ----
481	A 3 010
482	B 3 011
483	C 3 100
484	D 3 101
485	E 3 110
486	F 2 00
487	G 4 1110
488	H 4 1111
489
490	3.2.3. Details of block format
491
492	Each block of compressed data begins with 3 header bits
493	containing the following data:
494
495	first bit BFINAL
496	next 2 bits BTYPE
497
498	Note that the header bits do not necessarily begin on a byte
499	boundary, since a block does not necessarily occupy an integral
500	number of bytes.
501
502
503
504
505
506	Deutsch Informational [Page 9]
507
508	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
509
510
511	BFINAL is set if and only if this is the last block of the data
512	set.
513
514	BTYPE specifies how the data are compressed, as follows:
515
516	00 - no compression
517	01 - compressed with fixed Huffman codes
518	10 - compressed with dynamic Huffman codes
519	11 - reserved (error)
520
521	The only difference between the two compressed cases is how the
522	Huffman codes for the literal/length and distance alphabets are
523	defined.
524
525	In all cases, the decoding algorithm for the actual data is as
526	follows:
527
528	do
529	read block header from input stream.
530	if stored with no compression
531	skip any remaining bits in current partially
532	processed byte
533	read LEN and NLEN (see next section)
534	copy LEN bytes of data to output
535	otherwise
536	if compressed with dynamic Huffman codes
537	read representation of code trees (see
538	subsection below)
539	loop (until end of block code recognized)
540	decode literal/length value from input stream
541	if value < 256
542	copy value (literal byte) to output stream
543	otherwise
544	if value = end of block (256)
545	break from loop
546	otherwise (value = 257..285)
547	decode distance from input stream
548
549	move backwards distance bytes in the output
550	stream, and copy length bytes from this
551	position to the output stream.
552	end loop
553	while not last block
554
555	Note that a duplicated string reference may refer to a string
556	in a previous block; i.e., the backward distance may cross one
557	or more block boundaries. However a distance cannot refer past
558	the beginning of the output stream. (An application using a
559
560
561
562	Deutsch Informational [Page 10]
563
564	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
565
566
567	preset dictionary might discard part of the output stream; a
568	distance can refer to that part of the output stream anyway)
569	Note also that the referenced string may overlap the current
570	position; for example, if the last 2 bytes decoded have values
571	X and Y, a string reference with <length = 5, distance = 2>
572	adds X,Y,X,Y,X to the output stream.
573
574	We now specify each compression method in turn.
575
576	3.2.4. Non-compressed blocks (BTYPE=00)
577
578	Any bits of input up to the next byte boundary are ignored.
579	The rest of the block consists of the following information:
580
581	0 1 2 3 4...
582	+---+---+---+---+================================+
583	\| LEN \| NLEN \|... LEN bytes of literal data...\|
584	+---+---+---+---+================================+
585
586	LEN is the number of data bytes in the block. NLEN is the
587	one's complement of LEN.
588
589	3.2.5. Compressed blocks (length and distance codes)
590
591	As noted above, encoded data blocks in the "deflate" format
592	consist of sequences of symbols drawn from three conceptually
593	distinct alphabets: either literal bytes, from the alphabet of
594	byte values (0..255), or <length, backward distance> pairs,
595	where the length is drawn from (3..258) and the distance is
596	drawn from (1..32,768). In fact, the literal and length
597	alphabets are merged into a single alphabet (0..285), where
598	values 0..255 represent literal bytes, the value 256 indicates
599	end-of-block, and values 257..285 represent length codes
600	(possibly in conjunction with extra bits following the symbol
601	code) as follows:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618	Deutsch Informational [Page 11]
619
620	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
621
622
623	Extra Extra Extra
624	Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
625	---- ---- ------ ---- ---- ------- ---- ---- -------
626	257 0 3 267 1 15,16 277 4 67-82
627	258 0 4 268 1 17,18 278 4 83-98
628	259 0 5 269 2 19-22 279 4 99-114
629	260 0 6 270 2 23-26 280 4 115-130
630	261 0 7 271 2 27-30 281 5 131-162
631	262 0 8 272 2 31-34 282 5 163-194
632	263 0 9 273 3 35-42 283 5 195-226
633	264 0 10 274 3 43-50 284 5 227-257
634	265 1 11,12 275 3 51-58 285 0 258
635	266 1 13,14 276 3 59-66
636
637	The extra bits should be interpreted as a machine integer
638	stored with the most-significant bit first, e.g., bits 1110
639	represent the value 14.
640
641	Extra Extra Extra
642	Code Bits Dist Code Bits Dist Code Bits Distance
643	---- ---- ---- ---- ---- ------ ---- ---- --------
644	0 0 1 10 4 33-48 20 9 1025-1536
645	1 0 2 11 4 49-64 21 9 1537-2048
646	2 0 3 12 5 65-96 22 10 2049-3072
647	3 0 4 13 5 97-128 23 10 3073-4096
648	4 1 5,6 14 6 129-192 24 11 4097-6144
649	5 1 7,8 15 6 193-256 25 11 6145-8192
650	6 2 9-12 16 7 257-384 26 12 8193-12288
651	7 2 13-16 17 7 385-512 27 12 12289-16384
652	8 3 17-24 18 8 513-768 28 13 16385-24576
653	9 3 25-32 19 8 769-1024 29 13 24577-32768
654
655	3.2.6. Compression with fixed Huffman codes (BTYPE=01)
656
657	The Huffman codes for the two alphabets are fixed, and are not
658	represented explicitly in the data. The Huffman code lengths
659	for the literal/length alphabet are:
660
661	Lit Value Bits Codes
662	--------- ---- -----
663	0 - 143 8 00110000 through
664	10111111
665	144 - 255 9 110010000 through
666	111111111
667	256 - 279 7 0000000 through
668	0010111
669	280 - 287 8 11000000 through
670	11000111
671
672
673
674	Deutsch Informational [Page 12]
675
676	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
677
678
679	The code lengths are sufficient to generate the actual codes,
680	as described above; we show the codes in the table for added
681	clarity. Literal/length values 286-287 will never actually
682	occur in the compressed data, but participate in the code
683	construction.
684
685	Distance codes 0-31 are represented by (fixed-length) 5-bit
686	codes, with possible additional bits as shown in the table
687	shown in Paragraph 3.2.5, above. Note that distance codes 30-
688	31 will never actually occur in the compressed data.
689
690	3.2.7. Compression with dynamic Huffman codes (BTYPE=10)
691
692	The Huffman codes for the two alphabets appear in the block
693	immediately after the header bits and before the actual
694	compressed data, first the literal/length code and then the
695	distance code. Each code is defined by a sequence of code
696	lengths, as discussed in Paragraph 3.2.2, above. For even
697	greater compactness, the code length sequences themselves are
698	compressed using a Huffman code. The alphabet for code lengths
699	is as follows:
700
701	0 - 15: Represent code lengths of 0 - 15
702	16: Copy the previous code length 3 - 6 times.
703	The next 2 bits indicate repeat length
704	(0 = 3, ... , 3 = 6)
705	Example: Codes 8, 16 (+2 bits 11),
706	16 (+2 bits 10) will expand to
707	12 code lengths of 8 (1 + 6 + 5)
708	17: Repeat a code length of 0 for 3 - 10 times.
709	(3 bits of length)
710	18: Repeat a code length of 0 for 11 - 138 times
711	(7 bits of length)
712
713	A code length of 0 indicates that the corresponding symbol in
714	the literal/length or distance alphabet will not occur in the
715	block, and should not participate in the Huffman code
716	construction algorithm given earlier. If only one distance
717	code is used, it is encoded using one bit, not zero bits; in
718	this case there is a single code length of one, with one unused
719	code. One distance code of zero bits means that there are no
720	distance codes used at all (the data is all literals).
721
722	We can now define the format of the block:
723
724	5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286)
725	5 Bits: HDIST, # of Distance codes - 1 (1 - 32)
726	4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19)
727
728
729
730	Deutsch Informational [Page 13]
731
732	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
733
734
735	(HCLEN + 4) x 3 bits: code lengths for the code length
736	alphabet given just above, in the order: 16, 17, 18,
737	0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
738
739	These code lengths are interpreted as 3-bit integers
740	(0-7); as above, a code length of 0 means the
741	corresponding symbol (literal/length or distance code
742	length) is not used.
743
744	HLIT + 257 code lengths for the literal/length alphabet,
745	encoded using the code length Huffman code
746
747	HDIST + 1 code lengths for the distance alphabet,
748	encoded using the code length Huffman code
749
750	The actual compressed data of the block,
751	encoded using the literal/length and distance Huffman
752	codes
753
754	The literal/length symbol 256 (end of data),
755	encoded using the literal/length Huffman code
756
757	The code length repeat codes can cross from HLIT + 257 to the
758	HDIST + 1 code lengths. In other words, all code lengths form
759	a single sequence of HLIT + HDIST + 258 values.
760
761	3.3. Compliance
762
763	A compressor may limit further the ranges of values specified in
764	the previous section and still be compliant; for example, it may
765	limit the range of backward pointers to some value smaller than
766	32K. Similarly, a compressor may limit the size of blocks so that
767	a compressible block fits in memory.
768
769	A compliant decompressor must accept the full range of possible
770	values defined in the previous section, and must accept blocks of
771	arbitrary size.
772
773	4. Compression algorithm details
774
775	While it is the intent of this document to define the "deflate"
776	compressed data format without reference to any particular
777	compression algorithm, the format is related to the compressed
778	formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below);
779	since many variations of LZ77 are patented, it is strongly
780	recommended that the implementor of a compressor follow the general
781	algorithm presented here, which is known not to be patented per se.
782	The material in this section is not part of the definition of the
783
784
785
786	Deutsch Informational [Page 14]
787
788	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
789
790
791	specification per se, and a compressor need not follow it in order to
792	be compliant.
793
794	The compressor terminates a block when it determines that starting a
795	new block with fresh trees would be useful, or when the block size
796	fills up the compressor's block buffer.
797
798	The compressor uses a chained hash table to find duplicated strings,
799	using a hash function that operates on 3-byte sequences. At any
800	given point during compression, let XYZ be the next 3 input bytes to
801	be examined (not necessarily all different, of course). First, the
802	compressor examines the hash chain for XYZ. If the chain is empty,
803	the compressor simply writes out X as a literal byte and advances one
804	byte in the input. If the hash chain is not empty, indicating that
805	the sequence XYZ (or, if we are unlucky, some other 3 bytes with the
806	same hash function value) has occurred recently, the compressor
807	compares all strings on the XYZ hash chain with the actual input data
808	sequence starting at the current point, and selects the longest
809	match.
810
811	The compressor searches the hash chains starting with the most recent
812	strings, to favor small distances and thus take advantage of the
813	Huffman encoding. The hash chains are singly linked. There are no
814	deletions from the hash chains; the algorithm simply discards matches
815	that are too old. To avoid a worst-case situation, very long hash
816	chains are arbitrarily truncated at a certain length, determined by a
817	run-time parameter.
818
819	To improve overall compression, the compressor optionally defers the
820	selection of matches ("lazy matching"): after a match of length N has
821	been found, the compressor searches for a longer match starting at
822	the next input byte. If it finds a longer match, it truncates the
823	previous match to a length of one (thus producing a single literal
824	byte) and then emits the longer match. Otherwise, it emits the
825	original match, and, as described above, advances N bytes before
826	continuing.
827
828	Run-time parameters also control this "lazy match" procedure. If
829	compression ratio is most important, the compressor attempts a
830	complete second search regardless of the length of the first match.
831	In the normal case, if the current match is "long enough", the
832	compressor reduces the search for a longer match, thus speeding up
833	the process. If speed is most important, the compressor inserts new
834	strings in the hash table only when no match was found, or when the
835	match is not "too long". This degrades the compression ratio but
836	saves time since there are both fewer insertions and fewer searches.
837
838
839
840
841
842	Deutsch Informational [Page 15]
843
844	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
845
846
847	5. References
848
849	[1] Huffman, D. A., "A Method for the Construction of Minimum
850	Redundancy Codes", Proceedings of the Institute of Radio
851	Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101.
852
853	[2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data
854	Compression", IEEE Transactions on Information Theory, Vol. 23,
855	No. 3, pp. 337-343.
856
857	[3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources,
858	available in ftp://ftp.uu.net/pub/archiving/zip/doc/
859
860	[4] Gailly, J.-L., and Adler, M., GZIP documentation and sources,
861	available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/
862
863	[5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix
864	encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169.
865
866	[6] Hirschberg and Lelewer, "Efficient decoding of prefix codes,"
867	Comm. ACM, 33,4, April 1990, pp. 449-459.
868
869	6. Security Considerations
870
871	Any data compression method involves the reduction of redundancy in
872	the data. Consequently, any corruption of the data is likely to have
873	severe effects and be difficult to correct. Uncompressed text, on
874	the other hand, will probably still be readable despite the presence
875	of some corrupted bytes.
876
877	It is recommended that systems using this data format provide some
878	means of validating the integrity of the compressed data. See
879	reference [3], for example.
880
881	7. Source code
882
883	Source code for a C language implementation of a "deflate" compliant
884	compressor and decompressor is available within the zlib package at
885	ftp://ftp.uu.net/pub/archiving/zip/zlib/.
886
887	8. Acknowledgements
888
889	Trademarks cited in this document are the property of their
890	respective owners.
891
892	Phil Katz designed the deflate format. Jean-Loup Gailly and Mark
893	Adler wrote the related software described in this specification.
894	Glenn Randers-Pehrson converted this document to RFC and HTML format.
895
896
897
898	Deutsch Informational [Page 16]
899
900	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
901
902
903	9. Author's Address
904
905	L. Peter Deutsch
906	Aladdin Enterprises
907	203 Santa Margarita Ave.
908	Menlo Park, CA 94025
909
910	Phone: (415) 322-0103 (AM only)
911	FAX: (415) 322-1734
912	EMail: <ghost@aladdin.com>
913
914	Questions about the technical content of this specification can be
915	sent by email to:
916
917	Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
918	Mark Adler <madler@alumni.caltech.edu>
919
920	Editorial comments on this specification can be sent by email to:
921
922	L. Peter Deutsch <ghost@aladdin.com> and
923	Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954	Deutsch Informational [Page 17]
955

lib/std/compress/testdata/rfc1951.txt.fixed.z.9 deleted

Binary files a/lib/std/compress/testdata/rfc1951.txt.fixed.z.9 and /dev/null differ

lib/std/compress/testdata/rfc1951.txt.z.0 deleted

Binary files a/lib/std/compress/testdata/rfc1951.txt.z.0 and /dev/null differ

lib/std/compress/testdata/rfc1951.txt.z.9 deleted

Binary files a/lib/std/compress/testdata/rfc1951.txt.z.9 and /dev/null differ

lib/std/compress/testdata/rfc1952.txt deleted-675

...	@@ -1,675 +0,0 @@
1
2
3
4
5
6
7	Network Working Group P. Deutsch
8	Request for Comments: 1952 Aladdin Enterprises
9	Category: Informational May 1996
10
11
12	GZIP file format specification version 4.3
13
14	Status of This Memo
15
16	This memo provides information for the Internet community. This memo
17	does not specify an Internet standard of any kind. Distribution of
18	this memo is unlimited.
19
20	IESG Note:
21
22	The IESG takes no position on the validity of any Intellectual
23	Property Rights statements contained in this document.
24
25	Notices
26
27	Copyright (c) 1996 L. Peter Deutsch
28
29	Permission is granted to copy and distribute this document for any
30	purpose and without charge, including translations into other
31	languages and incorporation into compilations, provided that the
32	copyright notice and this notice are preserved, and that any
33	substantive changes or deletions from the original are clearly
34	marked.
35
36	A pointer to the latest version of this and related documentation in
37	HTML format can be found at the URL
38	<ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40	Abstract
41
42	This specification defines a lossless compressed data format that is
43	compatible with the widely used GZIP utility. The format includes a
44	cyclic redundancy check value for detecting data corruption. The
45	format presently uses the DEFLATE method of compression but can be
46	easily extended to use other compression methods. The format can be
47	implemented readily in a manner not covered by patents.
48
49
50
51
52
53
54
55
56
57
58	Deutsch Informational [Page 1]
59
60	RFC 1952 GZIP File Format Specification May 1996
61
62
63	Table of Contents
64
65	1. Introduction ................................................... 2
66	1.1. Purpose ................................................... 2
67	1.2. Intended audience ......................................... 3
68	1.3. Scope ..................................................... 3
69	1.4. Compliance ................................................ 3
70	1.5. Definitions of terms and conventions used ................. 3
71	1.6. Changes from previous versions ............................ 3
72	2. Detailed specification ......................................... 4
73	2.1. Overall conventions ....................................... 4
74	2.2. File format ............................................... 5
75	2.3. Member format ............................................. 5
76	2.3.1. Member header and trailer ........................... 6
77	2.3.1.1. Extra field ................................... 8
78	2.3.1.2. Compliance .................................... 9
79	3. References .................................................. 9
80	4. Security Considerations .................................... 10
81	5. Acknowledgements ........................................... 10
82	6. Author's Address ........................................... 10
83	7. Appendix: Jean-Loup Gailly's gzip utility .................. 11
84	8. Appendix: Sample CRC Code .................................. 11
85
86	1. Introduction
87
88	1.1. Purpose
89
90	The purpose of this specification is to define a lossless
91	compressed data format that:
92
93	* Is independent of CPU type, operating system, file system,
94	and character set, and hence can be used for interchange;
95	* Can compress or decompress a data stream (as opposed to a
96	randomly accessible file) to produce another data stream,
97	using only an a priori bounded amount of intermediate
98	storage, and hence can be used in data communications or
99	similar structures such as Unix filters;
100	* Compresses data with efficiency comparable to the best
101	currently available general-purpose compression methods,
102	and in particular considerably better than the "compress"
103	program;
104	* Can be implemented readily in a manner not covered by
105	patents, and hence can be practiced freely;
106	* Is compatible with the file format produced by the current
107	widely used gzip utility, in that conforming decompressors
108	will be able to read data produced by the existing gzip
109	compressor.
110
111
112
113
114	Deutsch Informational [Page 2]
115
116	RFC 1952 GZIP File Format Specification May 1996
117
118
119	The data format defined by this specification does not attempt to:
120
121	* Provide random access to compressed data;
122	* Compress specialized data (e.g., raster graphics) as well as
123	the best currently available specialized algorithms.
124
125	1.2. Intended audience
126
127	This specification is intended for use by implementors of software
128	to compress data into gzip format and/or decompress data from gzip
129	format.
130
131	The text of the specification assumes a basic background in
132	programming at the level of bits and other primitive data
133	representations.
134
135	1.3. Scope
136
137	The specification specifies a compression method and a file format
138	(the latter assuming only that a file can store a sequence of
139	arbitrary bytes). It does not specify any particular interface to
140	a file system or anything about character sets or encodings
141	(except for file names and comments, which are optional).
142
143	1.4. Compliance
144
145	Unless otherwise indicated below, a compliant decompressor must be
146	able to accept and decompress any file that conforms to all the
147	specifications presented here; a compliant compressor must produce
148	files that conform to all the specifications presented here. The
149	material in the appendices is not part of the specification per se
150	and is not relevant to compliance.
151
152	1.5. Definitions of terms and conventions used
153
154	byte: 8 bits stored or transmitted as a unit (same as an octet).
155	(For this specification, a byte is exactly 8 bits, even on
156	machines which store a character on a number of bits different
157	from 8.) See below for the numbering of bits within a byte.
158
159	1.6. Changes from previous versions
160
161	There have been no technical changes to the gzip format since
162	version 4.1 of this specification. In version 4.2, some
163	terminology was changed, and the sample CRC code was rewritten for
164	clarity and to eliminate the requirement for the caller to do pre-
165	and post-conditioning. Version 4.3 is a conversion of the
166	specification to RFC style.
167
168
169
170	Deutsch Informational [Page 3]
171
172	RFC 1952 GZIP File Format Specification May 1996
173
174
175	2. Detailed specification
176
177	2.1. Overall conventions
178
179	In the diagrams below, a box like this:
180
181	+---+
182	\| \| <-- the vertical bars might be missing
183	+---+
184
185	represents one byte; a box like this:
186
187	+==============+
188	\| \|
189	+==============+
190
191	represents a variable number of bytes.
192
193	Bytes stored within a computer do not have a "bit order", since
194	they are always treated as a unit. However, a byte considered as
195	an integer between 0 and 255 does have a most- and least-
196	significant bit, and since we write numbers with the most-
197	significant digit on the left, we also write bytes with the most-
198	significant bit on the left. In the diagrams below, we number the
199	bits of a byte so that bit 0 is the least-significant bit, i.e.,
200	the bits are numbered:
201
202	+--------+
203	\|76543210\|
204	+--------+
205
206	This document does not address the issue of the order in which
207	bits of a byte are transmitted on a bit-sequential medium, since
208	the data format described here is byte- rather than bit-oriented.
209
210	Within a computer, a number may occupy multiple bytes. All
211	multi-byte numbers in the format described here are stored with
212	the least-significant byte first (at the lower memory address).
213	For example, the decimal number 520 is stored as:
214
215	0 1
216	+--------+--------+
217	\|00001000\|00000010\|
218	+--------+--------+
219	^ ^
220	\| \|
221	\| + more significant byte = 2 x 256
222	+ less significant byte = 8
223
224
225
226	Deutsch Informational [Page 4]
227
228	RFC 1952 GZIP File Format Specification May 1996
229
230
231	2.2. File format
232
233	A gzip file consists of a series of "members" (compressed data
234	sets). The format of each member is specified in the following
235	section. The members simply appear one after another in the file,
236	with no additional information before, between, or after them.
237
238	2.3. Member format
239
240	Each member has the following structure:
241
242	+---+---+---+---+---+---+---+---+---+---+
243	\|ID1\|ID2\|CM \|FLG\| MTIME \|XFL\|OS \| (more-->)
244	+---+---+---+---+---+---+---+---+---+---+
245
246	(if FLG.FEXTRA set)
247
248	+---+---+=================================+
249	\| XLEN \|...XLEN bytes of "extra field"...\| (more-->)
250	+---+---+=================================+
251
252	(if FLG.FNAME set)
253
254	+=========================================+
255	\|...original file name, zero-terminated...\| (more-->)
256	+=========================================+
257
258	(if FLG.FCOMMENT set)
259
260	+===================================+
261	\|...file comment, zero-terminated...\| (more-->)
262	+===================================+
263
264	(if FLG.FHCRC set)
265
266	+---+---+
267	\| CRC16 \|
268	+---+---+
269
270	+=======================+
271	\|...compressed blocks...\| (more-->)
272	+=======================+
273
274	0 1 2 3 4 5 6 7
275	+---+---+---+---+---+---+---+---+
276	\| CRC32 \| ISIZE \|
277	+---+---+---+---+---+---+---+---+
278
279
280
281
282	Deutsch Informational [Page 5]
283
284	RFC 1952 GZIP File Format Specification May 1996
285
286
287	2.3.1. Member header and trailer
288
289	ID1 (IDentification 1)
290	ID2 (IDentification 2)
291	These have the fixed values ID1 = 31 (0x1f, \037), ID2 = 139
292	(0x8b, \213), to identify the file as being in gzip format.
293
294	CM (Compression Method)
295	This identifies the compression method used in the file. CM
296	= 0-7 are reserved. CM = 8 denotes the "deflate"
297	compression method, which is the one customarily used by
298	gzip and which is documented elsewhere.
299
300	FLG (FLaGs)
301	This flag byte is divided into individual bits as follows:
302
303	bit 0 FTEXT
304	bit 1 FHCRC
305	bit 2 FEXTRA
306	bit 3 FNAME
307	bit 4 FCOMMENT
308	bit 5 reserved
309	bit 6 reserved
310	bit 7 reserved
311
312	If FTEXT is set, the file is probably ASCII text. This is
313	an optional indication, which the compressor may set by
314	checking a small amount of the input data to see whether any
315	non-ASCII characters are present. In case of doubt, FTEXT
316	is cleared, indicating binary data. For systems which have
317	different file formats for ascii text and binary data, the
318	decompressor can use FTEXT to choose the appropriate format.
319	We deliberately do not specify the algorithm used to set
320	this bit, since a compressor always has the option of
321	leaving it cleared and a decompressor always has the option
322	of ignoring it and letting some other program handle issues
323	of data conversion.
324
325	If FHCRC is set, a CRC16 for the gzip header is present,
326	immediately before the compressed data. The CRC16 consists
327	of the two least significant bytes of the CRC32 for all
328	bytes of the gzip header up to and not including the CRC16.
329	[The FHCRC bit was never set by versions of gzip up to
330	1.2.4, even though it was documented with a different
331	meaning in gzip 1.2.4.]
332
333	If FEXTRA is set, optional extra fields are present, as
334	described in a following section.
335
336
337
338	Deutsch Informational [Page 6]
339
340	RFC 1952 GZIP File Format Specification May 1996
341
342
343	If FNAME is set, an original file name is present,
344	terminated by a zero byte. The name must consist of ISO
345	8859-1 (LATIN-1) characters; on operating systems using
346	EBCDIC or any other character set for file names, the name
347	must be translated to the ISO LATIN-1 character set. This
348	is the original name of the file being compressed, with any
349	directory components removed, and, if the file being
350	compressed is on a file system with case insensitive names,
351	forced to lower case. There is no original file name if the
352	data was compressed from a source other than a named file;
353	for example, if the source was stdin on a Unix system, there
354	is no file name.
355
356	If FCOMMENT is set, a zero-terminated file comment is
357	present. This comment is not interpreted; it is only
358	intended for human consumption. The comment must consist of
359	ISO 8859-1 (LATIN-1) characters. Line breaks should be
360	denoted by a single line feed character (10 decimal).
361
362	Reserved FLG bits must be zero.
363
364	MTIME (Modification TIME)
365	This gives the most recent modification time of the original
366	file being compressed. The time is in Unix format, i.e.,
367	seconds since 00:00:00 GMT, Jan. 1, 1970. (Note that this
368	may cause problems for MS-DOS and other systems that use
369	local rather than Universal time.) If the compressed data
370	did not come from a file, MTIME is set to the time at which
371	compression started. MTIME = 0 means no time stamp is
372	available.
373
374	XFL (eXtra FLags)
375	These flags are available for use by specific compression
376	methods. The "deflate" method (CM = 8) sets these flags as
377	follows:
378
379	XFL = 2 - compressor used maximum compression,
380	slowest algorithm
381	XFL = 4 - compressor used fastest algorithm
382
383	OS (Operating System)
384	This identifies the type of file system on which compression
385	took place. This may be useful in determining end-of-line
386	convention for text files. The currently defined values are
387	as follows:
388
389
390
391
392
393
394	Deutsch Informational [Page 7]
395
396	RFC 1952 GZIP File Format Specification May 1996
397
398
399	0 - FAT filesystem (MS-DOS, OS/2, NT/Win32)
400	1 - Amiga
401	2 - VMS (or OpenVMS)
402	3 - Unix
403	4 - VM/CMS
404	5 - Atari TOS
405	6 - HPFS filesystem (OS/2, NT)
406	7 - Macintosh
407	8 - Z-System
408	9 - CP/M
409	10 - TOPS-20
410	11 - NTFS filesystem (NT)
411	12 - QDOS
412	13 - Acorn RISCOS
413	255 - unknown
414
415	XLEN (eXtra LENgth)
416	If FLG.FEXTRA is set, this gives the length of the optional
417	extra field. See below for details.
418
419	CRC32 (CRC-32)
420	This contains a Cyclic Redundancy Check value of the
421	uncompressed data computed according to CRC-32 algorithm
422	used in the ISO 3309 standard and in section 8.1.1.6.2 of
423	ITU-T recommendation V.42. (See http://www.iso.ch for
424	ordering ISO documents. See gopher://info.itu.ch for an
425	online version of ITU-T V.42.)
426
427	ISIZE (Input SIZE)
428	This contains the size of the original (uncompressed) input
429	data modulo 2^32.
430
431	2.3.1.1. Extra field
432
433	If the FLG.FEXTRA bit is set, an "extra field" is present in
434	the header, with total length XLEN bytes. It consists of a
435	series of subfields, each of the form:
436
437	+---+---+---+---+==================================+
438	\|SI1\|SI2\| LEN \|... LEN bytes of subfield data ...\|
439	+---+---+---+---+==================================+
440
441	SI1 and SI2 provide a subfield ID, typically two ASCII letters
442	with some mnemonic value. Jean-Loup Gailly
443	<gzip@prep.ai.mit.edu> is maintaining a registry of subfield
444	IDs; please send him any subfield ID you wish to use. Subfield
445	IDs with SI2 = 0 are reserved for future use. The following
446	IDs are currently defined:
447
448
449
450	Deutsch Informational [Page 8]
451
452	RFC 1952 GZIP File Format Specification May 1996
453
454
455	SI1 SI2 Data
456	---------- ---------- ----
457	0x41 ('A') 0x70 ('P') Apollo file type information
458
459	LEN gives the length of the subfield data, excluding the 4
460	initial bytes.
461
462	2.3.1.2. Compliance
463
464	A compliant compressor must produce files with correct ID1,
465	ID2, CM, CRC32, and ISIZE, but may set all the other fields in
466	the fixed-length part of the header to default values (255 for
467	OS, 0 for all others). The compressor must set all reserved
468	bits to zero.
469
470	A compliant decompressor must check ID1, ID2, and CM, and
471	provide an error indication if any of these have incorrect
472	values. It must examine FEXTRA/XLEN, FNAME, FCOMMENT and FHCRC
473	at least so it can skip over the optional fields if they are
474	present. It need not examine any other part of the header or
475	trailer; in particular, a decompressor may ignore FTEXT and OS
476	and always produce binary output, and still be compliant. A
477	compliant decompressor must give an error indication if any
478	reserved bit is non-zero, since such a bit could indicate the
479	presence of a new field that would cause subsequent data to be
480	interpreted incorrectly.
481
482	3. References
483
484	[1] "Information Processing - 8-bit single-byte coded graphic
485	character sets - Part 1: Latin alphabet No.1" (ISO 8859-1:1987).
486	The ISO 8859-1 (Latin-1) character set is a superset of 7-bit
487	ASCII. Files defining this character set are available as
488	iso_8859-1.* in ftp://ftp.uu.net/graphics/png/documents/
489
490	[2] ISO 3309
491
492	[3] ITU-T recommendation V.42
493
494	[4] Deutsch, L.P.,"DEFLATE Compressed Data Format Specification",
495	available in ftp://ftp.uu.net/pub/archiving/zip/doc/
496
497	[5] Gailly, J.-L., GZIP documentation, available as gzip-*.tar in
498	ftp://prep.ai.mit.edu/pub/gnu/
499
500	[6] Sarwate, D.V., "Computation of Cyclic Redundancy Checks via Table
501	Look-Up", Communications of the ACM, 31(8), pp.1008-1013.
502
503
504
505
506	Deutsch Informational [Page 9]
507
508	RFC 1952 GZIP File Format Specification May 1996
509
510
511	[7] Schwaderer, W.D., "CRC Calculation", April 85 PC Tech Journal,
512	pp.118-133.
513
514	[8] ftp://ftp.adelaide.edu.au/pub/rocksoft/papers/crc_v3.txt,
515	describing the CRC concept.
516
517	4. Security Considerations
518
519	Any data compression method involves the reduction of redundancy in
520	the data. Consequently, any corruption of the data is likely to have
521	severe effects and be difficult to correct. Uncompressed text, on
522	the other hand, will probably still be readable despite the presence
523	of some corrupted bytes.
524
525	It is recommended that systems using this data format provide some
526	means of validating the integrity of the compressed data, such as by
527	setting and checking the CRC-32 check value.
528
529	5. Acknowledgements
530
531	Trademarks cited in this document are the property of their
532	respective owners.
533
534	Jean-Loup Gailly designed the gzip format and wrote, with Mark Adler,
535	the related software described in this specification. Glenn
536	Randers-Pehrson converted this document to RFC and HTML format.
537
538	6. Author's Address
539
540	L. Peter Deutsch
541	Aladdin Enterprises
542	203 Santa Margarita Ave.
543	Menlo Park, CA 94025
544
545	Phone: (415) 322-0103 (AM only)
546	FAX: (415) 322-1734
547	EMail: <ghost@aladdin.com>
548
549	Questions about the technical content of this specification can be
550	sent by email to:
551
552	Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
553	Mark Adler <madler@alumni.caltech.edu>
554
555	Editorial comments on this specification can be sent by email to:
556
557	L. Peter Deutsch <ghost@aladdin.com> and
558	Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
559
560
561
562	Deutsch Informational [Page 10]
563
564	RFC 1952 GZIP File Format Specification May 1996
565
566
567	7. Appendix: Jean-Loup Gailly's gzip utility
568
569	The most widely used implementation of gzip compression, and the
570	original documentation on which this specification is based, were
571	created by Jean-Loup Gailly <gzip@prep.ai.mit.edu>. Since this
572	implementation is a de facto standard, we mention some more of its
573	features here. Again, the material in this section is not part of
574	the specification per se, and implementations need not follow it to
575	be compliant.
576
577	When compressing or decompressing a file, gzip preserves the
578	protection, ownership, and modification time attributes on the local
579	file system, since there is no provision for representing protection
580	attributes in the gzip file format itself. Since the file format
581	includes a modification time, the gzip decompressor provides a
582	command line switch that assigns the modification time from the file,
583	rather than the local modification time of the compressed input, to
584	the decompressed output.
585
586	8. Appendix: Sample CRC Code
587
588	The following sample code represents a practical implementation of
589	the CRC (Cyclic Redundancy Check). (See also ISO 3309 and ITU-T V.42
590	for a formal specification.)
591
592	The sample code is in the ANSI C programming language. Non C users
593	may find it easier to read with these hints:
594
595	& Bitwise AND operator.
596	^ Bitwise exclusive-OR operator.
597	>> Bitwise right shift operator. When applied to an
598	unsigned quantity, as here, right shift inserts zero
599	bit(s) at the left.
600	! Logical NOT operator.
601	++ "n++" increments the variable n.
602	0xNNN 0x introduces a hexadecimal (base 16) constant.
603	Suffix L indicates a long value (at least 32 bits).
604
605	/* Table of CRCs of all 8-bit messages. */
606	unsigned long crc_table[256];
607
608	/* Flag: has the table been computed? Initially false. */
609	int crc_table_computed = 0;
610
611	/* Make the table for a fast CRC. */
612	void make_crc_table(void)
613	{
614	unsigned long c;
615
616
617
618	Deutsch Informational [Page 11]
619
620	RFC 1952 GZIP File Format Specification May 1996
621
622
623	int n, k;
624	for (n = 0; n < 256; n++) {
625	c = (unsigned long) n;
626	for (k = 0; k < 8; k++) {
627	if (c & 1) {
628	c = 0xedb88320L ^ (c >> 1);
629	} else {
630	c = c >> 1;
631	}
632	}
633	crc_table[n] = c;
634	}
635	crc_table_computed = 1;
636	}
637
638	/*
639	Update a running crc with the bytes buf[0..len-1] and return
640	the updated crc. The crc should be initialized to zero. Pre- and
641	post-conditioning (one's complement) is performed within this
642	function so it shouldn't be done by the caller. Usage example:
643
644	unsigned long crc = 0L;
645
646	while (read_buffer(buffer, length) != EOF) {
647	crc = update_crc(crc, buffer, length);
648	}
649	if (crc != original_crc) error();
650	*/
651	unsigned long update_crc(unsigned long crc,
652	unsigned char *buf, int len)
653	{
654	unsigned long c = crc ^ 0xffffffffL;
655	int n;
656
657	if (!crc_table_computed)
658	make_crc_table();
659	for (n = 0; n < len; n++) {
660	c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8);
661	}
662	return c ^ 0xffffffffL;
663	}
664
665	/* Return the CRC of the bytes buf[0..len-1]. */
666	unsigned long crc(unsigned char *buf, int len)
667	{
668	return update_crc(0L, buf, len);
669	}
670
671
672
673
674	Deutsch Informational [Page 12]
675

lib/std/compress/testdata/rfc1952.txt.gz deleted

Binary files a/lib/std/compress/testdata/rfc1952.txt.gz and /dev/null differ

lib/std/compress/zlib.zig+46-262

...	@@ -1,282 +1,66 @@	...	@@ -1,282 +1,66 @@
1	//	1	const deflate = @import("flate/deflate.zig");
2	// Compressor/Decompressor for ZLIB data streams (RFC1950)	2	const inflate = @import("flate/inflate.zig");
3		3
4	const std = @import("std");	4	/// Decompress compressed data from reader and write plain data to the writer.
5	const io = std.io;	5	pub fn decompress(reader: anytype, writer: anytype) !void {
6	const fs = std.fs;	6	try inflate.decompress(.zlib, reader, writer);
7	const testing = std.testing;
8	const mem = std.mem;
9	const deflate = std.compress.deflate;
10
11	// Zlib header format as specified in RFC1950
12	const ZLibHeader = packed struct {
13	checksum: u5,
14	preset_dict: u1,
15	compression_level: u2,
16	compression_method: u4,
17	compression_info: u4,
18
19	const DEFLATE = 8;
20	const WINDOW_32K = 7;
21	};
22
23	pub fn DecompressStream(comptime ReaderType: type) type {
24	return struct {
25	const Self = @This();
26
27	pub const Error = ReaderType.Error \|\|
28	deflate.Decompressor(ReaderType).Error \|\|
29	error{ WrongChecksum, Unsupported };
30	pub const Reader = io.Reader(*Self, Error, read);
31
32	allocator: mem.Allocator,
33	inflater: deflate.Decompressor(ReaderType),
34	in_reader: ReaderType,
35	hasher: std.hash.Adler32,
36
37	fn init(allocator: mem.Allocator, source: ReaderType) !Self {
38	// Zlib header format is specified in RFC1950
39	const header_u16 = try source.readInt(u16, .big);
40
41	// verify the header checksum
42	if (header_u16 % 31 != 0)
43	return error.BadHeader;
44	const header = @as(ZLibHeader, @bitCast(header_u16));
45
46	// The CM field must be 8 to indicate the use of DEFLATE
47	if (header.compression_method != ZLibHeader.DEFLATE)
48	return error.InvalidCompression;
49	// CINFO is the base-2 logarithm of the LZ77 window size, minus 8.
50	// Values above 7 are unspecified and therefore rejected.
51	if (header.compression_info > ZLibHeader.WINDOW_32K)
52	return error.InvalidWindowSize;
53
54	const dictionary = null;
55	// TODO: Support this case
56	if (header.preset_dict != 0)
57	return error.Unsupported;
58
59	return Self{
60	.allocator = allocator,
61	.inflater = try deflate.decompressor(allocator, source, dictionary),
62	.in_reader = source,
63	.hasher = std.hash.Adler32.init(),
64	};
65	}
66
67	pub fn deinit(self: *Self) void {
68	self.inflater.deinit();
69	}
70
71	// Implements the io.Reader interface
72	pub fn read(self: *Self, buffer: []u8) Error!usize {
73	if (buffer.len == 0)
74	return 0;
75
76	// Read from the compressed stream and update the computed checksum
77	const r = try self.inflater.read(buffer);
78	if (r != 0) {
79	self.hasher.update(buffer[0..r]);
80	return r;
81	}
82
83	// We've reached the end of stream, check if the checksum matches
84	const hash = try self.in_reader.readInt(u32, .big);
85	if (hash != self.hasher.final())
86	return error.WrongChecksum;
87
88	return 0;
89	}
90
91	pub fn reader(self: *Self) Reader {
92	return .{ .context = self };
93	}
94	};
95	}	7	}
96		8
97	pub fn decompressStream(allocator: mem.Allocator, reader: anytype) !DecompressStream(@TypeOf(reader)) {	9	/// Decompressor type
98	return DecompressStream(@TypeOf(reader)).init(allocator, reader);	10	pub fn Decompressor(comptime ReaderType: type) type {
		11	return inflate.Inflate(.zlib, ReaderType);
99	}	12	}
100		13
101	pub const CompressionLevel = enum(u2) {	14	/// Create Decompressor which will read compressed data from reader.
102	no_compression = 0,	15	pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
103	fastest = 1,	16	return inflate.decompressor(.zlib, reader);
104	default = 2,
105	maximum = 3,
106	};
107
108	pub const CompressStreamOptions = struct {
109	level: CompressionLevel = .default,
110	};
111
112	pub fn CompressStream(comptime WriterType: type) type {
113	return struct {
114	const Self = @This();
115
116	const Error = WriterType.Error \|\|
117	deflate.Compressor(WriterType).Error;
118	pub const Writer = io.Writer(*Self, Error, write);
119
120	allocator: mem.Allocator,
121	deflator: deflate.Compressor(WriterType),
122	in_writer: WriterType,
123	hasher: std.hash.Adler32,
124
125	fn init(allocator: mem.Allocator, dest: WriterType, options: CompressStreamOptions) !Self {
126	var header = ZLibHeader{
127	.compression_info = ZLibHeader.WINDOW_32K,
128	.compression_method = ZLibHeader.DEFLATE,
129	.compression_level = @intFromEnum(options.level),
130	.preset_dict = 0,
131	.checksum = 0,
132	};
133	header.checksum = @as(u5, @truncate(31 - @as(u16, @bitCast(header)) % 31));
134
135	try dest.writeInt(u16, @as(u16, @bitCast(header)), .big);
136
137	const compression_level: deflate.Compression = switch (options.level) {
138	.no_compression => .no_compression,
139	.fastest => .best_speed,
140	.default => .default_compression,
141	.maximum => .best_compression,
142	};
143
144	return Self{
145	.allocator = allocator,
146	.deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }),
147	.in_writer = dest,
148	.hasher = std.hash.Adler32.init(),
149	};
150	}
151
152	pub fn write(self: *Self, bytes: []const u8) Error!usize {
153	if (bytes.len == 0) {
154	return 0;
155	}
156
157	const w = try self.deflator.write(bytes);
158
159	self.hasher.update(bytes[0..w]);
160	return w;
161	}
162
163	pub fn writer(self: *Self) Writer {
164	return .{ .context = self };
165	}
166
167	pub fn deinit(self: *Self) void {
168	self.deflator.deinit();
169	}
170
171	pub fn finish(self: *Self) !void {
172	const hash = self.hasher.final();
173	try self.deflator.close();
174	try self.in_writer.writeInt(u32, hash, .big);
175	}
176	};
177	}	17	}
178		18
179	pub fn compressStream(allocator: mem.Allocator, writer: anytype, options: CompressStreamOptions) !CompressStream(@TypeOf(writer)) {	19	/// Compression level, trades between speed and compression size.
180	return CompressStream(@TypeOf(writer)).init(allocator, writer, options);	20	pub const Options = deflate.Options;
181	}
182
183	fn testDecompress(data: []const u8, expected: []const u8) !void {
184	var in_stream = io.fixedBufferStream(data);
185
186	var zlib_stream = try decompressStream(testing.allocator, in_stream.reader());
187	defer zlib_stream.deinit();
188		21
189	// Read and decompress the whole file	22	/// Compress plain data from reader and write compressed data to the writer.
190	const buf = try zlib_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));	23	pub fn compress(reader: anytype, writer: anytype, options: Options) !void {
191	defer testing.allocator.free(buf);	24	try deflate.compress(.zlib, reader, writer, options);
192
193	// Check against the reference
194	try testing.expectEqualSlices(u8, expected, buf);
195	}	25	}
196		26
197	// All the test cases are obtained by compressing the RFC1951 text	27	/// Compressor type
198	//	28	pub fn Compressor(comptime WriterType: type) type {
199	// https://tools.ietf.org/rfc/rfc1951.txt length=36944 bytes	29	return deflate.Compressor(.zlib, WriterType);
200	// SHA256=5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009
201	test "compressed data" {
202	const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
203
204	// Compressed with compression level = 0
205	try testDecompress(
206	@embedFile("testdata/rfc1951.txt.z.0"),
207	rfc1951_txt,
208	);
209	// Compressed with compression level = 9
210	try testDecompress(
211	@embedFile("testdata/rfc1951.txt.z.9"),
212	rfc1951_txt,
213	);
214	// Compressed with compression level = 9 and fixed Huffman codes
215	try testDecompress(
216	@embedFile("testdata/rfc1951.txt.fixed.z.9"),
217	rfc1951_txt,
218	);
219	}	30	}
220		31
221	test "don't read past deflate stream's end" {	32	/// Create Compressor which outputs compressed data to the writer.
222	try testDecompress(&[_]u8{	33	pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
223	0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,	34	return try deflate.compressor(.zlib, writer, options);
224	0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
225	0x83, 0x95, 0x0b, 0xf5,
226	}, &[_]u8{
227	0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
228	0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
229	0x00, 0x00, 0xff, 0xff, 0xff,
230	});
231	}	35	}
232		36
233	test "sanity checks" {	37	/// Huffman only compression. Without Lempel-Ziv match searching. Faster
234	// Truncated header	38	/// compression, less memory requirements but bigger compressed sizes.
235	try testing.expectError(	39	pub const huffman = struct {
236	error.EndOfStream,	40	pub fn compress(reader: anytype, writer: anytype) !void {
237	testDecompress(&[_]u8{0x78}, ""),	41	try deflate.huffman.compress(.zlib, reader, writer);
238	);	42	}
239	// Failed FCHECK check
240	try testing.expectError(
241	error.BadHeader,
242	testDecompress(&[_]u8{ 0x78, 0x9D }, ""),
243	);
244	// Wrong CM
245	try testing.expectError(
246	error.InvalidCompression,
247	testDecompress(&[_]u8{ 0x79, 0x94 }, ""),
248	);
249	// Wrong CINFO
250	try testing.expectError(
251	error.InvalidWindowSize,
252	testDecompress(&[_]u8{ 0x88, 0x98 }, ""),
253	);
254	// Wrong checksum
255	try testing.expectError(
256	error.WrongChecksum,
257	testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
258	);
259	// Truncated checksum
260	try testing.expectError(
261	error.EndOfStream,
262	testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
263	);
264	}
265		43
266	test "compress data" {	44	pub fn Compressor(comptime WriterType: type) type {
267	const allocator = testing.allocator;	45	return deflate.huffman.Compressor(.zlib, WriterType);
268	const rfc1951_txt = @embedFile("testdata/rfc1951.txt");	46	}
269		47
270	for (std.meta.tags(CompressionLevel)) \|level\| {	48	pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
271	var compressed_data = std.ArrayList(u8).init(allocator);	49	return deflate.huffman.compressor(.zlib, writer);
272	defer compressed_data.deinit();	50	}
		51	};
273		52
274	var compressor = try compressStream(allocator, compressed_data.writer(), .{ .level = level });	53	// No compression store only. Compressed size is slightly bigger than plain.
275	defer compressor.deinit();	54	pub const store = struct {
		55	pub fn compress(reader: anytype, writer: anytype) !void {
		56	try deflate.store.compress(.zlib, reader, writer);
		57	}
276		58
277	try compressor.writer().writeAll(rfc1951_txt);	59	pub fn Compressor(comptime WriterType: type) type {
278	try compressor.finish();	60	return deflate.store.Compressor(.zlib, WriterType);
		61	}
279		62
280	try testDecompress(compressed_data.items, rfc1951_txt);	63	pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
		64	return deflate.store.compressor(.zlib, writer);
281	}	65	}
282	}	66	};

lib/std/debug.zig+1-2

...	@@ -1212,8 +1212,7 @@ pub fn readElfDebugInfo(	...	@@ -1212,8 +1212,7 @@ pub fn readElfDebugInfo(
1212	const chdr = section_reader.readStruct(elf.Chdr) catch continue;	1212	const chdr = section_reader.readStruct(elf.Chdr) catch continue;
1213	if (chdr.ch_type != .ZLIB) continue;	1213	if (chdr.ch_type != .ZLIB) continue;
1214		1214
1215	var zlib_stream = std.compress.zlib.decompressStream(allocator, section_stream.reader()) catch continue;	1215	var zlib_stream = std.compress.zlib.decompressor(section_stream.reader());
1216	defer zlib_stream.deinit();
1217		1216
1218	const decompressed_section = try allocator.alloc(u8, chdr.ch_size);	1217	const decompressed_section = try allocator.alloc(u8, chdr.ch_size);
1219	errdefer allocator.free(decompressed_section);	1218	errdefer allocator.free(decompressed_section);

lib/std/http/Client.zig+8-8

lib/std/http/Server.zig+6-6

src/Package/Fetch.zig+6-1

 @@ -1099,7 +1099,12 @@ fn unpackResource(
     switch (file_type) {
         .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),
-        .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip),
+        .@"tar.gz" => {
+            const reader = resource.reader();
+            var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
+            var dcp = std.compress.gzip.decompressor(br.reader());
+            try unpackTarball(f, tmp_directory.handle, dcp.reader());
+        },
         .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),
         .@"tar.zst" => try unpackTarballCompressed(f, tmp_directory.handle, resource, ZstdWrapper),
         .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) {

src/Package/Fetch/git.zig+3-6

src/arch/x86_64/CodeGen.zig+14-1

...	@@ -7276,7 +7276,20 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn	...	@@ -7276,7 +7276,20 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn
7276	else => \|vector_index\| @intFromEnum(vector_index) * val_bit_size,	7276	else => \|vector_index\| @intFromEnum(vector_index) * val_bit_size,
7277	};	7277	};
7278	if (ptr_bit_off % 8 == 0) {	7278	if (ptr_bit_off % 8 == 0) {
7279	try self.load(dst_mcv, ptr_ty, ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8))));	7279	{
		7280	const mat_ptr_mcv: MCValue = switch (ptr_mcv) {
		7281	.immediate, .register, .register_offset, .lea_frame => ptr_mcv,
		7282	else => .{ .register = try self.copyToTmpRegister(ptr_ty, ptr_mcv) },
		7283	};
		7284	const mat_ptr_lock = switch (mat_ptr_mcv) {
		7285	.register => \|mat_ptr_reg\| self.register_manager.lockReg(mat_ptr_reg),
		7286	else => null,
		7287	};
		7288	defer if (mat_ptr_lock) \|lock\| self.register_manager.unlockReg(lock);
		7289
		7290	try self.load(dst_mcv, ptr_ty, mat_ptr_mcv.offset(@intCast(@divExact(ptr_bit_off, 8))));
		7291	}
		7292
7280	if (val_abi_size * 8 > val_bit_size) {	7293	if (val_abi_size * 8 > val_bit_size) {
7281	if (dst_mcv.isRegister()) {	7294	if (dst_mcv.isRegister()) {
7282	try self.truncateRegister(val_ty, dst_mcv.getReg().?);	7295	try self.truncateRegister(val_ty, dst_mcv.getReg().?);

src/link/Elf/Object.zig+1-3

 @@ -902,9 +902,7 @@ pub fn codeDecompressAlloc(self: Object, elf_file: *Elf, atom_index: Atom.Index)
         switch (chdr.ch_type) {
             .ZLIB => {
                 var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]);
-                var zlib_stream = std.compress.zlib.decompressStream(gpa, stream.reader()) catch
+                var zlib_stream = std.compress.zlib.decompressor(stream.reader());
-                    return error.InputOutput;
-                defer zlib_stream.deinit();
                 const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;
                 const decomp = try gpa.alloc(u8, size);
                 const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput;

src/objcopy.zig+1-2

...	@@ -1298,8 +1298,7 @@ const ElfFileHelper = struct {	...	@@ -1298,8 +1298,7 @@ const ElfFileHelper = struct {
1298	try compressed_stream.writer().writeAll(prefix);	1298	try compressed_stream.writer().writeAll(prefix);
1299		1299
1300	{	1300	{
1301	var compressor = try std.compress.zlib.compressStream(allocator, compressed_stream.writer(), .{});	1301	var compressor = try std.compress.zlib.compressor(compressed_stream.writer(), .{});
1302	defer compressor.deinit();
1303		1302
1304	var buf: [8000]u8 = undefined;	1303	var buf: [8000]u8 = undefined;
1305	while (true) {	1304	while (true) {

author	Andrew Kelley <andrew@ziglang.org> 2024-02-15 10:55:40-08:00
committer	GitHub <noreply@github.com> 2024-02-15 10:55:40-08:00
log	57d6f789de1d5fed5006aa3cefeb5b005bbdf6d6
tree	5bf9efbcd7ad173d714bd549bae90af66761fb7b
parent	7204eccf5cbf32977b779181de871559b478511d
parent	99cb201438e9458547082b35e1dd7c7c46c8c1bd
signature	Signed by PGP key B5690EEEBB952194

...	@@ -404,8 +404,8 @@ pub const RequestTransfer = union(enum) {	...	@@ -404,8 +404,8 @@ pub const RequestTransfer = union(enum) {
404		404
405	/// The decompressor for response messages.	405	/// The decompressor for response messages.
406	pub const Compression = union(enum) {	406	pub const Compression = union(enum) {
407	pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Request.TransferReader);	407	pub const DeflateDecompressor = std.compress.zlib.Decompressor(Request.TransferReader);
408	pub const GzipDecompressor = std.compress.gzip.Decompress(Request.TransferReader);	408	pub const GzipDecompressor = std.compress.gzip.Decompressor(Request.TransferReader);
409	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});	409	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Request.TransferReader, .{});
410		410
411	deflate: DeflateDecompressor,	411	deflate: DeflateDecompressor,
...	@@ -601,8 +601,8 @@ pub const Request = struct {	...	@@ -601,8 +601,8 @@ pub const Request = struct {
601	pub fn deinit(req: *Request) void {	601	pub fn deinit(req: *Request) void {
602	switch (req.response.compression) {	602	switch (req.response.compression) {
603	.none => {},	603	.none => {},
604	.deflate => \|*deflate\| deflate.deinit(),	604	.deflate => {},
605	.gzip => \|*gzip\| gzip.deinit(),	605	.gzip => {},
606	.zstd => \|*zstd\| zstd.deinit(),	606	.zstd => \|*zstd\| zstd.deinit(),
607	}	607	}
608		608
...	@@ -632,8 +632,8 @@ pub const Request = struct {	...	@@ -632,8 +632,8 @@ pub const Request = struct {
632		632
633	switch (req.response.compression) {	633	switch (req.response.compression) {
634	.none => {},	634	.none => {},
635	.deflate => \|*deflate\| deflate.deinit(),	635	.deflate => {},
636	.gzip => \|*gzip\| gzip.deinit(),	636	.gzip => {},
637	.zstd => \|*zstd\| zstd.deinit(),	637	.zstd => \|*zstd\| zstd.deinit(),
638	}	638	}
639		639
...	@@ -941,10 +941,10 @@ pub const Request = struct {	...	@@ -941,10 +941,10 @@ pub const Request = struct {
941	.identity => req.response.compression = .none,	941	.identity => req.response.compression = .none,
942	.compress, .@"x-compress" => return error.CompressionNotSupported,	942	.compress, .@"x-compress" => return error.CompressionNotSupported,
943	.deflate => req.response.compression = .{	943	.deflate => req.response.compression = .{
944	.deflate = std.compress.zlib.decompressStream(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,	944	.deflate = std.compress.zlib.decompressor(req.transferReader()),
945	},	945	},
946	.gzip, .@"x-gzip" => req.response.compression = .{	946	.gzip, .@"x-gzip" => req.response.compression = .{
947	.gzip = std.compress.gzip.decompress(req.client.allocator, req.transferReader()) catch return error.CompressionInitializationFailed,	947	.gzip = std.compress.gzip.decompressor(req.transferReader()),
948	},	948	},
949	.zstd => req.response.compression = .{	949	.zstd => req.response.compression = .{
950	.zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),	950	.zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),

...	@@ -195,8 +195,8 @@ pub const ResponseTransfer = union(enum) {	...	@@ -195,8 +195,8 @@ pub const ResponseTransfer = union(enum) {
195		195
196	/// The decompressor for request messages.	196	/// The decompressor for request messages.
197	pub const Compression = union(enum) {	197	pub const Compression = union(enum) {
198	pub const DeflateDecompressor = std.compress.zlib.DecompressStream(Response.TransferReader);	198	pub const DeflateDecompressor = std.compress.zlib.Decompressor(Response.TransferReader);
199	pub const GzipDecompressor = std.compress.gzip.Decompress(Response.TransferReader);	199	pub const GzipDecompressor = std.compress.gzip.Decompressor(Response.TransferReader);
200	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});	200	pub const ZstdDecompressor = std.compress.zstd.DecompressStream(Response.TransferReader, .{});
201		201
202	deflate: DeflateDecompressor,	202	deflate: DeflateDecompressor,
...	@@ -420,8 +420,8 @@ pub const Response = struct {	...	@@ -420,8 +420,8 @@ pub const Response = struct {
420		420
421	switch (res.request.compression) {	421	switch (res.request.compression) {
422	.none => {},	422	.none => {},
423	.deflate => \|*deflate\| deflate.deinit(),	423	.deflate => {},
424	.gzip => \|*gzip\| gzip.deinit(),	424	.gzip => {},
425	.zstd => \|*zstd\| zstd.deinit(),	425	.zstd => \|*zstd\| zstd.deinit(),
426	}	426	}
427		427
...	@@ -605,10 +605,10 @@ pub const Response = struct {	...	@@ -605,10 +605,10 @@ pub const Response = struct {
605	.identity => res.request.compression = .none,	605	.identity => res.request.compression = .none,
606	.compress, .@"x-compress" => return error.CompressionNotSupported,	606	.compress, .@"x-compress" => return error.CompressionNotSupported,
607	.deflate => res.request.compression = .{	607	.deflate => res.request.compression = .{
608	.deflate = std.compress.zlib.decompressStream(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,	608	.deflate = std.compress.zlib.decompressor(res.transferReader()),
609	},	609	},
610	.gzip, .@"x-gzip" => res.request.compression = .{	610	.gzip, .@"x-gzip" => res.request.compression = .{
611	.gzip = std.compress.gzip.decompress(res.allocator, res.transferReader()) catch return error.CompressionInitializationFailed,	611	.gzip = std.compress.gzip.decompressor(res.transferReader()),
612	},	612	},
613	.zstd => res.request.compression = .{	613	.zstd => res.request.compression = .{
614	.zstd = std.compress.zstd.decompressStream(res.allocator, res.transferReader()),	614	.zstd = std.compress.zstd.decompressStream(res.allocator, res.transferReader()),

...	@@ -1115,8 +1115,7 @@ fn indexPackFirstPass(	...	@@ -1115,8 +1115,7 @@ fn indexPackFirstPass(
1115	const entry_header = try EntryHeader.read(entry_crc32_reader.reader());	1115	const entry_header = try EntryHeader.read(entry_crc32_reader.reader());
1116	switch (entry_header) {	1116	switch (entry_header) {
1117	inline .commit, .tree, .blob, .tag => \|object, tag\| {	1117	inline .commit, .tree, .blob, .tag => \|object, tag\| {
1118	var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());	1118	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1119	defer entry_decompress_stream.deinit();
1120	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());	1119	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1121	var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));	1120	var entry_hashed_writer = hashedWriter(std.io.null_writer, Sha1.init(.{}));
1122	const entry_writer = entry_hashed_writer.writer();	1121	const entry_writer = entry_hashed_writer.writer();
...	@@ -1135,8 +1134,7 @@ fn indexPackFirstPass(	...	@@ -1135,8 +1134,7 @@ fn indexPackFirstPass(
1135	});	1134	});
1136	},	1135	},
1137	inline .ofs_delta, .ref_delta => \|delta\| {	1136	inline .ofs_delta, .ref_delta => \|delta\| {
1138	var entry_decompress_stream = try std.compress.zlib.decompressStream(allocator, entry_crc32_reader.reader());	1137	var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1139	defer entry_decompress_stream.deinit();
1140	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());	1138	var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1141	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();	1139	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1142	try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);	1140	try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
...	@@ -1257,8 +1255,7 @@ fn resolveDeltaChain(	...	@@ -1257,8 +1255,7 @@ fn resolveDeltaChain(
1257	fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {	1255	fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
1258	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;	1256	const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1259	var buffered_reader = std.io.bufferedReader(reader);	1257	var buffered_reader = std.io.bufferedReader(reader);
1260	var decompress_stream = try std.compress.zlib.decompressStream(allocator, buffered_reader.reader());	1258	var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader());
1261	defer decompress_stream.deinit();
1262	const data = try allocator.alloc(u8, alloc_size);	1259	const data = try allocator.alloc(u8, alloc_size);
1263	errdefer allocator.free(data);	1260	errdefer allocator.free(data);
1264	try decompress_stream.reader().readNoEof(data);	1261	try decompress_stream.reader().readNoEof(data);

Merge pull request #18923 from ianic/add_flate