Commit 490f067de82c8fd53029ddde0c73a242f4c04758 | zig

build.zig+15-2

...	@@ -93,12 +93,25 @@ pub fn build(b: *Builder) !void {	...	@@ -93,12 +93,25 @@ pub fn build(b: *Builder) !void {
93	.install_dir = .lib,	93	.install_dir = .lib,
94	.install_subdir = "zig",	94	.install_subdir = "zig",
95	.exclude_extensions = &[_][]const u8{	95	.exclude_extensions = &[_][]const u8{
96	"README.md",	96	// exclude files from lib/std/compress/
		97	".gz",
97	".z.0",	98	".z.0",
98	".z.9",	99	".z.9",
99	".gz",
100	"rfc1951.txt",	100	"rfc1951.txt",
		101	"rfc1952.txt",
		102	// exclude files from lib/std/compress/deflate/testdata
		103	".expect",
		104	".expect-noinput",
		105	".golden",
		106	".input",
		107	"compress-e.txt",
		108	"compress-gettysburg.txt",
		109	"compress-pi.txt",
		110	"rfc1951.txt",
		111	// exclude files from lib/std/tz/
101	".tzif",	112	".tzif",
		113	// others
		114	"README.md",
102	},	115	},
103	.blank_extensions = &[_][]const u8{	116	.blank_extensions = &[_][]const u8{
104	"test.zig",	117	"test.zig",

lib/std/compress.zig+1

...	@@ -5,6 +5,7 @@ pub const gzip = @import("compress/gzip.zig");	...	@@ -5,6 +5,7 @@ pub const gzip = @import("compress/gzip.zig");
5	pub const zlib = @import("compress/zlib.zig");	5	pub const zlib = @import("compress/zlib.zig");
6		6
7	test {	7	test {
		8	_ = deflate;
8	_ = gzip;	9	_ = gzip;
9	_ = zlib;	10	_ = zlib;
10	}	11	}

lib/std/compress/deflate.zig+20-729

...	@@ -1,738 +1,29 @@	...	@@ -1,738 +1,29 @@
1	//	1	//! The deflate package is a translation of the Go code of the compress/flate package from
2	// Decompressor for DEFLATE data streams (RFC1951)	2	//! https://go.googlesource.com/go/+/refs/tags/go1.17/src/compress/flate/
3	//
4	// Heavily inspired by the simple decompressor puff.c by Mark Adler
5		3
6	const std = @import("std");	4	const deflate = @import("deflate/compressor.zig");
7	const io = std.io;	5	const inflate = @import("deflate/decompressor.zig");
8	const math = std.math;
9	const mem = std.mem;
10		6
11	const assert = std.debug.assert;	7	pub const Compression = deflate.Compression;
		8	pub const Compressor = deflate.Compressor;
		9	pub const Decompressor = inflate.Decompressor;
12		10
13	const MAXBITS = 15;	11	pub const compressor = deflate.compressor;
14	const MAXLCODES = 286;	12	pub const decompressor = inflate.decompressor;
15	const MAXDCODES = 30;
16	const MAXCODES = MAXLCODES + MAXDCODES;
17	const FIXLCODES = 288;
18		13
19	// The maximum length of a Huffman code's prefix we can decode using the fast	14	test {
20	// path. The factor 9 is inherited from Zlib, tweaking the value showed little	15	_ = @import("deflate/token.zig");
21	// or no changes in the profiler output.	16	_ = @import("deflate/bits_utils.zig");
22	const PREFIX_LUT_BITS = 9;	17	_ = @import("deflate/dict_decoder.zig");
23		18
24	const Huffman = struct {	19	_ = @import("deflate/huffman_code.zig");
25	const LUTEntry = packed struct { symbol: u16 align(4), len: u16 };	20	_ = @import("deflate/huffman_bit_writer.zig");
26		21
27	// Number of codes for each possible length	22	_ = @import("deflate/compressor.zig");
28	count: [MAXBITS + 1]u16,	23	_ = @import("deflate/compressor_test.zig");
29	// Mapping between codes and symbols
30	symbol: [MAXCODES]u16,
31		24
32	// The decoding process uses a trick explained by Mark Adler in [1].	25	_ = @import("deflate/deflate_fast.zig");
33	// We basically precompute for a fixed number of codes (0 <= x <= 2^N-1)	26	_ = @import("deflate/deflate_fast_test.zig");
34	// the symbol and the effective code length we'd get if the decoder was run
35	// on the given N-bit sequence.
36	// A code with length 0 means the sequence is not a valid prefix for this
37	// canonical Huffman code and we have to decode it using a slower method.
38	//
39	// [1] https://github.com/madler/zlib/blob/v1.2.11/doc/algorithm.txt#L58
40	prefix_lut: [1 << PREFIX_LUT_BITS]LUTEntry,
41	// The following info refer to the codes of length PREFIX_LUT_BITS+1 and are
42	// used to bootstrap the bit-by-bit reading method if the fast-path fails.
43	last_code: u16,
44	last_index: u16,
45		27
46	min_code_len: u16,	28	_ = @import("deflate/decompressor.zig");
47
48	const ConstructError = error{ Oversubscribed, IncompleteSet };
49
50	fn construct(self: *Huffman, code_length: []const u16) ConstructError!void {
51	for (self.count) \|*val\| {
52	val.* = 0;
53	}
54
55	self.min_code_len = math.maxInt(u16);
56	for (code_length) \|len\| {
57	if (len != 0 and len < self.min_code_len)
58	self.min_code_len = len;
59	self.count[len] += 1;
60	}
61
62	// All zero.
63	if (self.count[0] == code_length.len) {
64	self.min_code_len = 0;
65	return;
66	}
67
68	var left: isize = 1;
69	for (self.count[1..]) \|val\| {
70	// Each added bit doubles the amount of codes.
71	left *= 2;
72	// Make sure the number of codes with this length isn't too high.
73	left -= @as(isize, @bitCast(i16, val));
74	if (left < 0)
75	return error.Oversubscribed;
76	}
77
78	// Compute the offset of the first symbol represented by a code of a
79	// given length in the symbol table, together with the first canonical
80	// Huffman code for that length.
81	var offset: [MAXBITS + 1]u16 = undefined;
82	var codes: [MAXBITS + 1]u16 = undefined;
83	{
84	offset[1] = 0;
85	codes[1] = 0;
86	var len: usize = 1;
87	while (len < MAXBITS) : (len += 1) {
88	offset[len + 1] = offset[len] + self.count[len];
89	codes[len + 1] = (codes[len] + self.count[len]) << 1;
90	}
91	}
92
93	self.prefix_lut = mem.zeroes(@TypeOf(self.prefix_lut));
94
95	for (code_length) \|len, symbol\| {
96	if (len != 0) {
97	// Fill the symbol table.
98	// The symbols are assigned sequentially for each length.
99	self.symbol[offset[len]] = @truncate(u16, symbol);
100	// Track the last assigned offset.
101	offset[len] += 1;
102	}
103
104	if (len == 0 or len > PREFIX_LUT_BITS)
105	continue;
106
107	// Given a Huffman code of length N we transform it into an index
108	// into the lookup table by reversing its bits and filling the
109	// remaining bits (PREFIX_LUT_BITS - N) with every possible
110	// combination of bits to act as a wildcard.
111	const bits_to_fill = @intCast(u5, PREFIX_LUT_BITS - len);
112	const rev_code = bitReverse(u16, codes[len], len);
113
114	// Track the last used code, but only for lengths < PREFIX_LUT_BITS.
115	codes[len] += 1;
116
117	var j: usize = 0;
118	while (j < @as(usize, 1) << bits_to_fill) : (j += 1) {
119	const index = rev_code \| (j << @intCast(u5, len));
120	assert(self.prefix_lut[index].len == 0);
121	self.prefix_lut[index] = .{
122	.symbol = @truncate(u16, symbol),
123	.len = @truncate(u16, len),
124	};
125	}
126	}
127
128	self.last_code = codes[PREFIX_LUT_BITS + 1];
129	self.last_index = offset[PREFIX_LUT_BITS + 1] - self.count[PREFIX_LUT_BITS + 1];
130
131	if (left > 0)
132	return error.IncompleteSet;
133	}
134	};
135
136	// Reverse bit-by-bit a N-bit code.
137	fn bitReverse(comptime T: type, value: T, N: usize) T {
138	const r = @bitReverse(T, value);
139	return r >> @intCast(math.Log2Int(T), @typeInfo(T).Int.bits - N);
140	}
141
142	pub fn InflateStream(comptime ReaderType: type) type {
143	return struct {
144	const Self = @This();
145
146	pub const Error = ReaderType.Error \|\| error{
147	EndOfStream,
148	BadCounts,
149	InvalidBlockType,
150	InvalidDistance,
151	InvalidFixedCode,
152	InvalidLength,
153	InvalidStoredSize,
154	InvalidSymbol,
155	InvalidTree,
156	MissingEOBCode,
157	NoLastLength,
158	OutOfCodes,
159	};
160	pub const Reader = io.Reader(*Self, Error, read);
161
162	inner_reader: ReaderType,
163
164	// True if the decoder met the end of the compressed stream, no further
165	// data can be decompressed
166	seen_eos: bool,
167
168	state: union(enum) {
169	// Parse a compressed block header and set up the internal state for
170	// decompressing its contents.
171	DecodeBlockHeader: void,
172	// Decode all the symbols in a compressed block.
173	DecodeBlockData: void,
174	// Copy N bytes of uncompressed data from the underlying stream into
175	// the window.
176	Copy: usize,
177	// Copy 1 byte into the window.
178	CopyLit: u8,
179	// Copy L bytes from the window itself, starting from D bytes
180	// behind.
181	CopyFrom: struct { distance: u16, length: u16 },
182	},
183
184	// Sliding window for the LZ77 algorithm
185	window: struct {
186	const WSelf = @This();
187
188	// invariant: buffer length is always a power of 2
189	buf: []u8,
190	// invariant: ri <= wi
191	wi: usize = 0, // Write index
192	ri: usize = 0, // Read index
193	el: usize = 0, // Number of readable elements
194	total_written: usize = 0,
195
196	fn readable(self: *WSelf) usize {
197	return self.el;
198	}
199
200	fn writable(self: *WSelf) usize {
201	return self.buf.len - self.el;
202	}
203
204	// Insert a single byte into the window.
205	// Returns 1 if there's enough space for the new byte and 0
206	// otherwise.
207	fn append(self: *WSelf, value: u8) usize {
208	if (self.writable() < 1) return 0;
209	self.appendUnsafe(value);
210	return 1;
211	}
212
213	// Insert a single byte into the window.
214	// Assumes there's enough space.
215	inline fn appendUnsafe(self: *WSelf, value: u8) void {
216	self.buf[self.wi] = value;
217	self.wi = (self.wi + 1) & (self.buf.len - 1);
218	self.el += 1;
219	self.total_written += 1;
220	}
221
222	// Fill dest[] with data from the window, starting from the read
223	// position. This updates the read pointer.
224	// Returns the number of read bytes or 0 if there's nothing to read
225	// yet.
226	fn read(self: *WSelf, dest: []u8) usize {
227	const N = math.min(dest.len, self.readable());
228
229	if (N == 0) return 0;
230
231	if (self.ri + N < self.buf.len) {
232	// The data doesn't wrap around
233	mem.copy(u8, dest, self.buf[self.ri .. self.ri + N]);
234	} else {
235	// The data wraps around the buffer, split the copy
236	std.mem.copy(u8, dest, self.buf[self.ri..]);
237	// How much data we've copied from `ri` to the end
238	const r = self.buf.len - self.ri;
239	std.mem.copy(u8, dest[r..], self.buf[0 .. N - r]);
240	}
241
242	self.ri = (self.ri + N) & (self.buf.len - 1);
243	self.el -= N;
244
245	return N;
246	}
247
248	// Copy `length` bytes starting from `distance` bytes behind the
249	// write pointer.
250	// Be careful as the length may be greater than the distance, that's
251	// how the compressor encodes run-length encoded sequences.
252	fn copyFrom(self: *WSelf, distance: usize, length: usize) usize {
253	const N = math.min(length, self.writable());
254
255	if (N == 0) return 0;
256
257	// TODO: Profile and, if needed, replace with smarter juggling
258	// of the window memory for the non-overlapping case.
259	var i: usize = 0;
260	while (i < N) : (i += 1) {
261	const index = (self.wi -% distance) & (self.buf.len - 1);
262	self.appendUnsafe(self.buf[index]);
263	}
264
265	return N;
266	}
267	},
268
269	// Compressor-local Huffman tables used to decompress blocks with
270	// dynamic codes.
271	huffman_tables: [2]Huffman = undefined,
272
273	// Huffman tables used for decoding length/distance pairs.
274	hdist: *Huffman,
275	hlen: *Huffman,
276
277	// Temporary buffer for the bitstream.
278	// Bits 0..`bits_left` are filled with data, the remaining ones are zeros.
279	bits: u32,
280	bits_left: usize,
281
282	fn peekBits(self: *Self, bits: usize) !u32 {
283	while (self.bits_left < bits) {
284	const byte = try self.inner_reader.readByte();
285	self.bits \|= @as(u32, byte) << @intCast(u5, self.bits_left);
286	self.bits_left += 8;
287	}
288	const mask = (@as(u32, 1) << @intCast(u5, bits)) - 1;
289	return self.bits & mask;
290	}
291	fn readBits(self: *Self, bits: usize) !u32 {
292	const val = try self.peekBits(bits);
293	self.discardBits(bits);
294	return val;
295	}
296	fn discardBits(self: *Self, bits: usize) void {
297	self.bits >>= @intCast(u5, bits);
298	self.bits_left -= bits;
299	}
300
301	fn stored(self: *Self) !void {
302	// Discard the remaining bits, the length field is always
303	// byte-aligned (and so is the data).
304	self.discardBits(self.bits_left);
305
306	const length = try self.inner_reader.readIntLittle(u16);
307	const length_cpl = try self.inner_reader.readIntLittle(u16);
308
309	if (length != ~length_cpl)
310	return error.InvalidStoredSize;
311
312	self.state = .{ .Copy = length };
313	}
314
315	fn fixed(self: *Self) !void {
316	comptime var lencode: Huffman = undefined;
317	comptime var distcode: Huffman = undefined;
318
319	// The Huffman codes are specified in the RFC1951, section 3.2.6
320	comptime {
321	@setEvalBranchQuota(100000);
322
323	const len_lengths =
324	[_]u16{8} ** 144 ++
325	[_]u16{9} ** 112 ++
326	[_]u16{7} ** 24 ++
327	[_]u16{8} ** 8;
328	assert(len_lengths.len == FIXLCODES);
329	try lencode.construct(len_lengths[0..]);
330
331	const dist_lengths = [_]u16{5} ** MAXDCODES;
332	distcode.construct(dist_lengths[0..]) catch \|err\| switch (err) {
333	// This error is expected because we only compute distance codes
334	// 0-29, which is fine since "distance codes 30-31 will never actually
335	// occur in the compressed data" (from section 3.2.6 of RFC1951).
336	error.IncompleteSet => {},
337	else => return err,
338	};
339	}
340
341	self.hlen = &lencode;
342	self.hdist = &distcode;
343	self.state = .DecodeBlockData;
344	}
345
346	fn dynamic(self: *Self) !void {
347	// Number of length codes
348	const nlen = (try self.readBits(5)) + 257;
349	// Number of distance codes
350	const ndist = (try self.readBits(5)) + 1;
351	// Number of code length codes
352	const ncode = (try self.readBits(4)) + 4;
353
354	if (nlen > MAXLCODES or ndist > MAXDCODES)
355	return error.BadCounts;
356
357	// Permutation of code length codes
358	const ORDER = [19]u16{
359	16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4,
360	12, 3, 13, 2, 14, 1, 15,
361	};
362
363	// Build the Huffman table to decode the code length codes
364	var lencode: Huffman = undefined;
365	{
366	var lengths = std.mem.zeroes([19]u16);
367
368	// Read the code lengths, missing ones are left as zero
369	for (ORDER[0..ncode]) \|val\| {
370	lengths[val] = @intCast(u16, try self.readBits(3));
371	}
372
373	lencode.construct(lengths[0..]) catch return error.InvalidTree;
374	}
375
376	// Read the length/literal and distance code length tables.
377	// Zero the table by default so we can avoid explicitly writing out
378	// zeros for codes 17 and 18
379	var lengths = std.mem.zeroes([MAXCODES]u16);
380
381	var i: usize = 0;
382	while (i < nlen + ndist) {
383	const symbol = try self.decode(&lencode);
384
385	switch (symbol) {
386	0...15 => {
387	lengths[i] = symbol;
388	i += 1;
389	},
390	16 => {
391	// repeat last length 3..6 times
392	if (i == 0) return error.NoLastLength;
393
394	const last_length = lengths[i - 1];
395	const repeat = 3 + (try self.readBits(2));
396	const last_index = i + repeat;
397	if (last_index > lengths.len)
398	return error.InvalidLength;
399	while (i < last_index) : (i += 1) {
400	lengths[i] = last_length;
401	}
402	},
403	17 => {
404	// repeat zero 3..10 times
405	i += 3 + (try self.readBits(3));
406	},
407	18 => {
408	// repeat zero 11..138 times
409	i += 11 + (try self.readBits(7));
410	},
411	else => return error.InvalidSymbol,
412	}
413	}
414
415	if (i > nlen + ndist)
416	return error.InvalidLength;
417
418	// Check if the end of block code is present
419	if (lengths[256] == 0)
420	return error.MissingEOBCode;
421
422	self.huffman_tables[0].construct(lengths[0..nlen]) catch \|err\| switch (err) {
423	error.Oversubscribed => return error.InvalidTree,
424	error.IncompleteSet => {
425	// incomplete code ok only for single length 1 code
426	if (nlen != self.huffman_tables[0].count[0] + self.huffman_tables[0].count[1]) {
427	return error.InvalidTree;
428	}
429	},
430	};
431	self.huffman_tables[1].construct(lengths[nlen .. nlen + ndist]) catch \|err\| switch (err) {
432	error.Oversubscribed => return error.InvalidTree,
433	error.IncompleteSet => {
434	// incomplete code ok only for single length 1 code
435	if (ndist != self.huffman_tables[1].count[0] + self.huffman_tables[1].count[1]) {
436	return error.InvalidTree;
437	}
438	},
439	};
440
441	self.hlen = &self.huffman_tables[0];
442	self.hdist = &self.huffman_tables[1];
443	self.state = .DecodeBlockData;
444	}
445
446	fn codes(self: Self, lencode: Huffman, distcode: *Huffman) !bool {
447	// Size base for length codes 257..285
448	const LENS = [29]u16{
449	3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31,
450	35, 43, 51, 59, 67, 83, 99, 115, 131, 163, 195, 227, 258,
451	};
452	// Extra bits for length codes 257..285
453	const LEXT = [29]u16{
454	0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2,
455	3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0,
456	};
457	// Offset base for distance codes 0..29
458	const DISTS = [30]u16{
459	1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193,
460	257, 385, 513, 769, 1025, 1537, 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577,
461	};
462	// Extra bits for distance codes 0..29
463	const DEXT = [30]u16{
464	0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6,
465	7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
466	};
467
468	while (true) {
469	const symbol = try self.decode(lencode);
470
471	switch (symbol) {
472	0...255 => {
473	// Literal value
474	const c = @truncate(u8, symbol);
475	if (self.window.append(c) == 0) {
476	self.state = .{ .CopyLit = c };
477	return false;
478	}
479	},
480	256 => {
481	// End of block symbol
482	return true;
483	},
484	257...285 => {
485	// Length/distance pair
486	const length_symbol = symbol - 257;
487	const length = LENS[length_symbol] +
488	@intCast(u16, try self.readBits(LEXT[length_symbol]));
489
490	const distance_symbol = try self.decode(distcode);
491	const distance = DISTS[distance_symbol] +
492	@intCast(u16, try self.readBits(DEXT[distance_symbol]));
493
494	if (distance > self.window.buf.len or distance > self.window.total_written)
495	return error.InvalidDistance;
496
497	const written = self.window.copyFrom(distance, length);
498	if (written != length) {
499	self.state = .{
500	.CopyFrom = .{
501	.distance = distance,
502	.length = length - @truncate(u16, written),
503	},
504	};
505	return false;
506	}
507	},
508	else => return error.InvalidFixedCode,
509	}
510	}
511	}
512
513	fn decode(self: Self, h: Huffman) !u16 {
514	// Using u32 instead of u16 to reduce the number of casts needed.
515	var prefix: u32 = 0;
516
517	// Fast path, read some bits and hope they're the prefix of some code.
518	// We can't read PREFIX_LUT_BITS as we don't want to read past the
519	// deflate stream end, use an incremental approach instead.
520	var code_len = h.min_code_len;
521	if (code_len == 0)
522	return error.OutOfCodes;
523	while (true) {
524	_ = try self.peekBits(code_len);
525	// Small optimization win, use as many bits as possible in the
526	// table lookup.
527	prefix = self.bits & ((1 << PREFIX_LUT_BITS) - 1);
528
529	const lut_entry = &h.prefix_lut[prefix];
530	// The code is longer than PREFIX_LUT_BITS!
531	if (lut_entry.len == 0)
532	break;
533	// If the code lenght doesn't increase we found a match.
534	if (lut_entry.len <= code_len) {
535	self.discardBits(code_len);
536	return lut_entry.symbol;
537	}
538
539	code_len = lut_entry.len;
540	}
541
542	// The sequence we've read is not a prefix of any code of length <=
543	// PREFIX_LUT_BITS, keep decoding it using a slower method.
544	prefix = try self.readBits(PREFIX_LUT_BITS);
545
546	// Speed up the decoding by starting from the first code length
547	// that's not covered by the table.
548	var len: usize = PREFIX_LUT_BITS + 1;
549	var first: usize = h.last_code;
550	var index: usize = h.last_index;
551
552	// Reverse the prefix so that the LSB becomes the MSB and make space
553	// for the next bit.
554	var code = bitReverse(u32, prefix, PREFIX_LUT_BITS + 1);
555
556	while (len <= MAXBITS) : (len += 1) {
557	code \|= try self.readBits(1);
558	const count = h.count[len];
559	if (code < first + count) {
560	return h.symbol[index + (code - first)];
561	}
562	index += count;
563	first += count;
564	first <<= 1;
565	code <<= 1;
566	}
567
568	return error.OutOfCodes;
569	}
570
571	fn step(self: *Self) !void {
572	while (true) {
573	switch (self.state) {
574	.DecodeBlockHeader => {
575	// The compressed stream is done.
576	if (self.seen_eos) return;
577
578	const last = @intCast(u1, try self.readBits(1));
579	const kind = @intCast(u2, try self.readBits(2));
580
581	self.seen_eos = last != 0;
582
583	// The next state depends on the block type.
584	switch (kind) {
585	0 => try self.stored(),
586	1 => try self.fixed(),
587	2 => try self.dynamic(),
588	3 => return error.InvalidBlockType,
589	}
590	},
591	.DecodeBlockData => {
592	if (!try self.codes(self.hlen, self.hdist)) {
593	return;
594	}
595
596	self.state = .DecodeBlockHeader;
597	},
598	.Copy => \|*length\| {
599	const N = math.min(self.window.writable(), length.*);
600
601	// TODO: This loop can be more efficient. On the other
602	// hand uncompressed blocks are not that common so...
603	var i: usize = 0;
604	while (i < N) : (i += 1) {
605	var tmp: [1]u8 = undefined;
606	if ((try self.inner_reader.read(&tmp)) != 1) {
607	// Unexpected end of stream, keep this error
608	// consistent with the use of readBitsNoEof.
609	return error.EndOfStream;
610	}
611	self.window.appendUnsafe(tmp[0]);
612	}
613
614	if (N != length.*) {
615	length.* -= N;
616	return;
617	}
618
619	self.state = .DecodeBlockHeader;
620	},
621	.CopyLit => \|c\| {
622	if (self.window.append(c) == 0) {
623	return;
624	}
625
626	self.state = .DecodeBlockData;
627	},
628	.CopyFrom => \|*info\| {
629	const written = self.window.copyFrom(info.distance, info.length);
630	if (written != info.length) {
631	info.length -= @truncate(u16, written);
632	return;
633	}
634
635	self.state = .DecodeBlockData;
636	},
637	}
638	}
639	}
640
641	fn init(source: ReaderType, window_slice: []u8) Self {
642	assert(math.isPowerOfTwo(window_slice.len));
643
644	return Self{
645	.inner_reader = source,
646	.window = .{ .buf = window_slice },
647	.seen_eos = false,
648	.state = .DecodeBlockHeader,
649	.hdist = undefined,
650	.hlen = undefined,
651	.bits = 0,
652	.bits_left = 0,
653	};
654	}
655
656	// Implements the io.Reader interface
657	pub fn read(self: *Self, buffer: []u8) Error!usize {
658	if (buffer.len == 0)
659	return 0;
660
661	// Try reading as much as possible from the window
662	var read_amt: usize = self.window.read(buffer);
663	while (read_amt < buffer.len) {
664	// Run the state machine, we can detect the "effective" end of
665	// stream condition by checking if any progress was made.
666	// Why "effective"? Because even though `seen_eos` is true we
667	// may still have to finish processing other decoding steps.
668	try self.step();
669	// No progress was made
670	if (self.window.readable() == 0)
671	break;
672
673	read_amt += self.window.read(buffer[read_amt..]);
674	}
675
676	return read_amt;
677	}
678
679	pub fn reader(self: *Self) Reader {
680	return .{ .context = self };
681	}
682	};
683	}
684
685	pub fn inflateStream(reader: anytype, window_slice: []u8) InflateStream(@TypeOf(reader)) {
686	return InflateStream(@TypeOf(reader)).init(reader, window_slice);
687	}
688
689	test "lengths overflow" {
690	// malformed final dynamic block, tries to write 321 code lengths (MAXCODES is 316)
691	// f dy hlit hdist hclen 16 17 18 0 (18) x138 (18) x138 (18) x39 (16) x6
692	// 1 10 11101 11101 0000 010 010 010 010 (11) 1111111 (11) 1111111 (11) 0011100 (01) 11
693	const stream = [_]u8{ 0b11101101, 0b00011101, 0b00100100, 0b11101001, 0b11111111, 0b11111111, 0b00111001, 0b00001110 };
694	try std.testing.expectError(error.InvalidLength, testInflate(stream[0..]));
695	}
696
697	test "empty distance alphabet" {
698	// dynamic block with empty distance alphabet is valid if only literals and end of data symbol are used
699	// f dy hlit hdist hclen 16 17 18 0 8 7 9 6 10 5 11 4 12 3 13 2 14 1 15 (18) x128 (18) x128 (1) ( 0) (256)
700	// 1 10 00000 00000 1111 000 000 010 010 000 000 000 000 000 000 000 000 000 000 000 000 000 001 000 (11) 1110101 (11) 1110101 (0) (10) (0)
701	const stream = [_]u8{ 0b00000101, 0b11100000, 0b00000001, 0b00001001, 0b00000000, 0b00000000, 0b00000000, 0b00000000, 0b00010000, 0b01011100, 0b10111111, 0b00101110 };
702	try testInflate(stream[0..]);
703	}
704
705	test "distance past beginning of output stream" {
706	// f fx ('A') ('B') ('C') <len=4, dist=4> (end)
707	// 1 01 (01110001) (01110010) (01110011) (0000010) (00011) (0000000)
708	const stream = [_]u8{ 0b01110011, 0b01110100, 0b01110010, 0b00000110, 0b01100001, 0b00000000 };
709	try std.testing.expectError(error.InvalidDistance, testInflate(stream[0..]));
710	}
711
712	test "inflateStream fuzzing" {
713	// see https://github.com/ziglang/zig/issues/9842
714	try std.testing.expectError(error.EndOfStream, testInflate("\x95\x90=o\xc20\x10\x86\xf30"));
715	try std.testing.expectError(error.OutOfCodes, testInflate("\x950\x00\x0000000"));
716
717	// Huffman.construct errors
718	// lencode
719	try std.testing.expectError(error.InvalidTree, testInflate("\x950000"));
720	try std.testing.expectError(error.InvalidTree, testInflate("\x05000"));
721	// hlen
722	try std.testing.expectError(error.InvalidTree, testInflate("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00"));
723	// hdist
724	try std.testing.expectError(error.InvalidTree, testInflate("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf."));
725
726	// Huffman.construct -> error.IncompleteSet returns that shouldn't give error.InvalidTree
727	// (like the "empty distance alphabet" test but for ndist instead of nlen)
728	try std.testing.expectError(error.EndOfStream, testInflate("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce"));
729	try testInflate("\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0");
730	}
731
732	fn testInflate(data: []const u8) !void {
733	var window: [0x8000]u8 = undefined;
734	const reader = std.io.fixedBufferStream(data).reader();
735	var inflate = inflateStream(reader, &window);
736	var inflated = try inflate.reader().readAllAlloc(std.testing.allocator, std.math.maxInt(usize));
737	defer std.testing.allocator.free(inflated);
738	}	29	}

lib/std/compress/deflate/bits_utils.zig created+34

...	@@ -0,0 +1,34 @@
1	const math = @import("std").math;
2
3	// Reverse bit-by-bit a N-bit code.
4	pub fn bitReverse(comptime T: type, value: T, N: usize) T {
5	const r = @bitReverse(T, value);
6	return r >> @intCast(math.Log2Int(T), @typeInfo(T).Int.bits - N);
7	}
8
9	test "bitReverse" {
10	const std = @import("std");
11	const expect = std.testing.expect;
12
13	const ReverseBitsTest = struct {
14	in: u16,
15	bit_count: u5,
16	out: u16,
17	};
18
19	var reverse_bits_tests = [_]ReverseBitsTest{
20	.{ .in = 1, .bit_count = 1, .out = 1 },
21	.{ .in = 1, .bit_count = 2, .out = 2 },
22	.{ .in = 1, .bit_count = 3, .out = 4 },
23	.{ .in = 1, .bit_count = 4, .out = 8 },
24	.{ .in = 1, .bit_count = 5, .out = 16 },
25	.{ .in = 17, .bit_count = 5, .out = 17 },
26	.{ .in = 257, .bit_count = 9, .out = 257 },
27	.{ .in = 29, .bit_count = 5, .out = 23 },
28	};
29
30	for (reverse_bits_tests) \|h\| {
31	var v = bitReverse(u16, h.in, h.bit_count);
32	try expect(v == h.out);
33	}
34	}

lib/std/compress/deflate/compressor.zig created+1111

...	@@ -0,0 +1,1111 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const fmt = std.fmt;
4	const io = std.io;
5	const math = std.math;
6	const mem = std.mem;
7
8	const Allocator = std.mem.Allocator;
9
10	const deflate_const = @import("deflate_const.zig");
11	const fast = @import("deflate_fast.zig");
12	const hm_bw = @import("huffman_bit_writer.zig");
13	const mu = @import("mem_utils.zig");
14	const token = @import("token.zig");
15
16	pub const Compression = enum(i5) {
17	/// huffman_only disables Lempel-Ziv match searching and only performs Huffman
18	/// entropy encoding. This mode is useful in compressing data that has
19	/// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
20	/// that lacks an entropy encoder. Compression gains are achieved when
21	/// certain bytes in the input stream occur more frequently than others.
22	///
23	/// Note that huffman_only produces a compressed output that is
24	/// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
25	/// continue to be able to decompress this output.
26	huffman_only = -2,
27	/// Same as level_6
28	default_compression = -1,
29	/// Does not attempt any compression; only adds the necessary DEFLATE framing.
30	no_compression = 0,
31	/// Prioritizes speed over output size, based on Snappy's LZ77-style encoder
32	best_speed = 1,
33	level_2 = 2,
34	level_3 = 3,
35	level_4 = 4,
36	level_5 = 5,
37	level_6 = 6,
38	level_7 = 7,
39	level_8 = 8,
40	/// Prioritizes smaller output size over speed
41	best_compression = 9,
42	};
43
44	const log_window_size = 15;
45	const window_size = 1 << log_window_size;
46	const window_mask = window_size - 1;
47
48	// The LZ77 step produces a sequence of literal tokens and <length, offset>
49	// pair tokens. The offset is also known as distance. The underlying wire
50	// format limits the range of lengths and offsets. For example, there are
51	// 256 legitimate lengths: those in the range [3, 258]. This package's
52	// compressor uses a higher minimum match length, enabling optimizations
53	// such as finding matches via 32-bit loads and compares.
54	const base_match_length = deflate_const.base_match_length; // The smallest match length per the RFC section 3.2.5
55	const min_match_length = 4; // The smallest match length that the compressor actually emits
56	const max_match_length = deflate_const.max_match_length;
57	const base_match_offset = deflate_const.base_match_offset; // The smallest match offset
58	const max_match_offset = deflate_const.max_match_offset; // The largest match offset
59
60	// The maximum number of tokens we put into a single flate block, just to
61	// stop things from getting too large.
62	const max_flate_block_tokens = 1 << 14;
63	const max_store_block_size = deflate_const.max_store_block_size;
64	const hash_bits = 17; // After 17 performance degrades
65	const hash_size = 1 << hash_bits;
66	const hash_mask = (1 << hash_bits) - 1;
67	const max_hash_offset = 1 << 24;
68
69	const skip_never = math.maxInt(u32);
70
71	const CompressionLevel = struct {
72	good: u16,
73	lazy: u16,
74	nice: u16,
75	chain: u16,
76	fast_skip_hashshing: u32,
77	};
78
79	fn levels(compression: Compression) CompressionLevel {
80	switch (compression) {
81	.no_compression,
82	.best_speed, // best_speed uses a custom algorithm; see deflate_fast.zig
83	.huffman_only,
84	=> return .{
85	.good = 0,
86	.lazy = 0,
87	.nice = 0,
88	.chain = 0,
89	.fast_skip_hashshing = 0,
90	},
91	// For levels 2-3 we don't bother trying with lazy matches.
92	.level_2 => return .{
93	.good = 4,
94	.lazy = 0,
95	.nice = 16,
96	.chain = 8,
97	.fast_skip_hashshing = 5,
98	},
99	.level_3 => return .{
100	.good = 4,
101	.lazy = 0,
102	.nice = 32,
103	.chain = 32,
104	.fast_skip_hashshing = 6,
105	},
106
107	// Levels 4-9 use increasingly more lazy matching and increasingly stringent conditions for
108	// "good enough".
109	.level_4 => return .{
110	.good = 4,
111	.lazy = 4,
112	.nice = 16,
113	.chain = 16,
114	.fast_skip_hashshing = skip_never,
115	},
116	.level_5 => return .{
117	.good = 8,
118	.lazy = 16,
119	.nice = 32,
120	.chain = 32,
121	.fast_skip_hashshing = skip_never,
122	},
123	.default_compression,
124	.level_6,
125	=> return .{
126	.good = 8,
127	.lazy = 16,
128	.nice = 128,
129	.chain = 128,
130	.fast_skip_hashshing = skip_never,
131	},
132	.level_7 => return .{
133	.good = 8,
134	.lazy = 32,
135	.nice = 128,
136	.chain = 256,
137	.fast_skip_hashshing = skip_never,
138	},
139	.level_8 => return .{
140	.good = 32,
141	.lazy = 128,
142	.nice = 258,
143	.chain = 1024,
144	.fast_skip_hashshing = skip_never,
145	},
146	.best_compression => return .{
147	.good = 32,
148	.lazy = 258,
149	.nice = 258,
150	.chain = 4096,
151	.fast_skip_hashshing = skip_never,
152	},
153	}
154	}
155
156	// matchLen returns the number of matching bytes in a and b
157	// up to length 'max'. Both slices must be at least 'max'
158	// bytes in size.
159	fn matchLen(a: []u8, b: []u8, max: u32) u32 {
160	var bounded_a = a[0..max];
161	var bounded_b = b[0..max];
162	for (bounded_a) \|av, i\| {
163	if (bounded_b[i] != av) {
164	return @intCast(u32, i);
165	}
166	}
167	return max;
168	}
169
170	const hash_mul = 0x1e35a7bd;
171
172	// hash4 returns a hash representation of the first 4 bytes
173	// of the supplied slice.
174	// The caller must ensure that b.len >= 4.
175	fn hash4(b: []u8) u32 {
176	return ((@as(u32, b[3]) \|
177	@as(u32, b[2]) << 8 \|
178	@as(u32, b[1]) << 16 \|
179	@as(u32, b[0]) << 24) *% hash_mul) >> (32 - hash_bits);
180	}
181
182	// bulkHash4 will compute hashes using the same
183	// algorithm as hash4
184	fn bulkHash4(b: []u8, dst: []u32) u32 {
185	if (b.len < min_match_length) {
186	return 0;
187	}
188	var hb =
189	@as(u32, b[3]) \|
190	@as(u32, b[2]) << 8 \|
191	@as(u32, b[1]) << 16 \|
192	@as(u32, b[0]) << 24;
193
194	dst[0] = (hb *% hash_mul) >> (32 - hash_bits);
195	var end = b.len - min_match_length + 1;
196	var i: u32 = 1;
197	while (i < end) : (i += 1) {
198	hb = (hb << 8) \| @as(u32, b[i + 3]);
199	dst[i] = (hb *% hash_mul) >> (32 - hash_bits);
200	}
201
202	return hb;
203	}
204
205	const CompressorOptions = struct {
206	level: Compression = .default_compression,
207	dictionary: ?[]const u8 = null,
208	};
209
210	/// Returns a new Compressor compressing data at the given level.
211	/// Following zlib, levels range from 1 (best_speed) to 9 (best_compression);
212	/// higher levels typically run slower but compress more. Level 0
213	/// (no_compression) does not attempt any compression; it only adds the
214	/// necessary DEFLATE framing.
215	/// Level -1 (default_compression) uses the default compression level.
216	/// Level -2 (huffman_only) will use Huffman compression only, giving
217	/// a very fast compression for all types of input, but sacrificing considerable
218	/// compression efficiency.
219	///
220	/// `dictionary` is optional and initializes the new `Compressor` with a preset dictionary.
221	/// The returned Compressor behaves as if the dictionary had been written to it without producing
222	/// any compressed output. The compressed data written to hm_bw can only be decompressed by a
223	/// Decompressor initialized with the same dictionary.
224	///
225	/// The compressed data will be passed to the provided `writer`, see `writer()` and `write()`.
226	pub fn compressor(
227	allocator: Allocator,
228	writer: anytype,
229	options: CompressorOptions,
230	) !Compressor(@TypeOf(writer)) {
231	return Compressor(@TypeOf(writer)).init(allocator, writer, options);
232	}
233
234	pub fn Compressor(comptime WriterType: anytype) type {
235	return struct {
236	const Self = @This();
237
238	/// A Writer takes data written to it and writes the compressed
239	/// form of that data to an underlying writer.
240	pub const Writer = io.Writer(*Self, Error, write);
241
242	/// Returns a Writer that takes data written to it and writes the compressed
243	/// form of that data to an underlying writer.
244	pub fn writer(self: *Self) Writer {
245	return .{ .context = self };
246	}
247
248	pub const Error = WriterType.Error;
249
250	allocator: Allocator,
251
252	compression: Compression,
253	compression_level: CompressionLevel,
254
255	// Inner writer wrapped in a HuffmanBitWriter
256	hm_bw: hm_bw.HuffmanBitWriter(WriterType) = undefined,
257	bulk_hasher: fn ([]u8, []u32) u32,
258
259	sync: bool, // requesting flush
260	best_speed_enc: *fast.DeflateFast, // Encoder for best_speed
261
262	// Input hash chains
263	// hash_head[hashValue] contains the largest inputIndex with the specified hash value
264	// If hash_head[hashValue] is within the current window, then
265	// hash_prev[hash_head[hashValue] & window_mask] contains the previous index
266	// with the same hash value.
267	chain_head: u32,
268	hash_head: []u32, // [hash_size]u32,
269	hash_prev: []u32, // [window_size]u32,
270	hash_offset: u32,
271
272	// input window: unprocessed data is window[index..window_end]
273	index: u32,
274	window: []u8,
275	window_end: usize,
276	block_start: usize, // window index where current tokens start
277	byte_available: bool, // if true, still need to process window[index-1].
278
279	// queued output tokens
280	tokens: []token.Token,
281	tokens_count: u16,
282
283	// deflate state
284	length: u32,
285	offset: u32,
286	hash: u32,
287	max_insert_index: usize,
288	err: bool,
289
290	// hash_match must be able to contain hashes for the maximum match length.
291	hash_match: []u32, // [max_match_length - 1]u32,
292
293	// dictionary
294	dictionary: ?[]const u8,
295
296	fn fillDeflate(self: *Self, b: []const u8) u32 {
297	if (self.index >= 2 * window_size - (min_match_length + max_match_length)) {
298	// shift the window by window_size
299	mem.copy(u8, self.window, self.window[window_size .. 2 * window_size]);
300	self.index -= window_size;
301	self.window_end -= window_size;
302	if (self.block_start >= window_size) {
303	self.block_start -= window_size;
304	} else {
305	self.block_start = math.maxInt(u32);
306	}
307	self.hash_offset += window_size;
308	if (self.hash_offset > max_hash_offset) {
309	var delta = self.hash_offset - 1;
310	self.hash_offset -= delta;
311	self.chain_head -\|= delta;
312
313	// Iterate over slices instead of arrays to avoid copying
314	// the entire table onto the stack (https://golang.org/issue/18625).
315	for (self.hash_prev) \|v, i\| {
316	if (v > delta) {
317	self.hash_prev[i] = @intCast(u32, v - delta);
318	} else {
319	self.hash_prev[i] = 0;
320	}
321	}
322	for (self.hash_head) \|v, i\| {
323	if (v > delta) {
324	self.hash_head[i] = @intCast(u32, v - delta);
325	} else {
326	self.hash_head[i] = 0;
327	}
328	}
329	}
330	}
331	var n = mu.copy(self.window[self.window_end..], b);
332	self.window_end += n;
333	return @intCast(u32, n);
334	}
335
336	fn writeBlock(self: *Self, tokens: []token.Token, index: usize) !void {
337	if (index > 0) {
338	var window: ?[]u8 = null;
339	if (self.block_start <= index) {
340	window = self.window[self.block_start..index];
341	}
342	self.block_start = index;
343	try self.hm_bw.writeBlock(tokens, false, window);
344	return;
345	}
346	return;
347	}
348
349	// fillWindow will fill the current window with the supplied
350	// dictionary and calculate all hashes.
351	// This is much faster than doing a full encode.
352	// Should only be used after a reset.
353	fn fillWindow(self: *Self, in_b: []const u8) void {
354	var b = in_b;
355	// Do not fill window if we are in store-only mode (look at the fill() function to see
356	// Compressions which use fillStore() instead of fillDeflate()).
357	if (self.compression == .no_compression or
358	self.compression == .huffman_only or
359	self.compression == .best_speed)
360	{
361	return;
362	}
363
364	// fillWindow() must not be called with stale data
365	assert(self.index == 0 and self.window_end == 0);
366
367	// If we are given too much, cut it.
368	if (b.len > window_size) {
369	b = b[b.len - window_size ..];
370	}
371	// Add all to window.
372	mem.copy(u8, self.window, b);
373	var n = b.len;
374
375	// Calculate 256 hashes at the time (more L1 cache hits)
376	var loops = (n + 256 - min_match_length) / 256;
377	var j: usize = 0;
378	while (j < loops) : (j += 1) {
379	var index = j * 256;
380	var end = index + 256 + min_match_length - 1;
381	if (end > n) {
382	end = n;
383	}
384	var to_check = self.window[index..end];
385	var dst_size = to_check.len - min_match_length + 1;
386
387	if (dst_size <= 0) {
388	continue;
389	}
390
391	var dst = self.hash_match[0..dst_size];
392	_ = self.bulk_hasher(to_check, dst);
393	var new_h: u32 = 0;
394	for (dst) \|val, i\| {
395	var di = i + index;
396	new_h = val;
397	var hh = &self.hash_head[new_h & hash_mask];
398	// Get previous value with the same hash.
399	// Our chain should point to the previous value.
400	self.hash_prev[di & window_mask] = hh.*;
401	// Set the head of the hash chain to us.
402	hh.* = @intCast(u32, di + self.hash_offset);
403	}
404	self.hash = new_h;
405	}
406	// Update window information.
407	self.window_end = n;
408	self.index = @intCast(u32, n);
409	}
410
411	const Match = struct {
412	length: u32,
413	offset: u32,
414	ok: bool,
415	};
416
417	// Try to find a match starting at pos whose length is greater than prev_length.
418	// We only look at self.compression_level.chain possibilities before giving up.
419	fn findMatch(
420	self: *Self,
421	pos: u32,
422	prev_head: u32,
423	prev_length: u32,
424	lookahead: u32,
425	) Match {
426	var length: u32 = 0;
427	var offset: u32 = 0;
428	var ok: bool = false;
429
430	var min_match_look: u32 = max_match_length;
431	if (lookahead < min_match_look) {
432	min_match_look = lookahead;
433	}
434
435	var win = self.window[0 .. pos + min_match_look];
436
437	// We quit when we get a match that's at least nice long
438	var nice = win.len - pos;
439	if (self.compression_level.nice < nice) {
440	nice = self.compression_level.nice;
441	}
442
443	// If we've got a match that's good enough, only look in 1/4 the chain.
444	var tries = self.compression_level.chain;
445	length = prev_length;
446	if (length >= self.compression_level.good) {
447	tries >>= 2;
448	}
449
450	var w_end = win[pos + length];
451	var w_pos = win[pos..];
452	var min_index = pos -\| window_size;
453
454	var i = prev_head;
455	while (tries > 0) : (tries -= 1) {
456	if (w_end == win[i + length]) {
457	var n = matchLen(win[i..], w_pos, min_match_look);
458
459	if (n > length and (n > min_match_length or pos - i <= 4096)) {
460	length = n;
461	offset = pos - i;
462	ok = true;
463	if (n >= nice) {
464	// The match is good enough that we don't try to find a better one.
465	break;
466	}
467	w_end = win[pos + n];
468	}
469	}
470	if (i == min_index) {
471	// hash_prev[i & window_mask] has already been overwritten, so stop now.
472	break;
473	}
474
475	if (@intCast(u32, self.hash_prev[i & window_mask]) < self.hash_offset) {
476	break;
477	}
478
479	i = @intCast(u32, self.hash_prev[i & window_mask]) - self.hash_offset;
480	if (i < min_index) {
481	break;
482	}
483	}
484
485	return Match{ .length = length, .offset = offset, .ok = ok };
486	}
487
488	fn writeStoredBlock(self: *Self, buf: []u8) !void {
489	try self.hm_bw.writeStoredHeader(buf.len, false);
490	try self.hm_bw.writeBytes(buf);
491	}
492
493	// encSpeed will compress and store the currently added data,
494	// if enough has been accumulated or we at the end of the stream.
495	fn encSpeed(self: *Self) !void {
496	// We only compress if we have max_store_block_size.
497	if (self.window_end < max_store_block_size) {
498	if (!self.sync) {
499	return;
500	}
501
502	// Handle small sizes.
503	if (self.window_end < 128) {
504	switch (self.window_end) {
505	0 => return,
506	1...16 => {
507	try self.writeStoredBlock(self.window[0..self.window_end]);
508	},
509	else => {
510	try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
511	self.err = self.hm_bw.err;
512	},
513	}
514	self.window_end = 0;
515	self.best_speed_enc.reset();
516	return;
517	}
518	}
519	// Encode the block.
520	self.tokens_count = 0;
521	self.best_speed_enc.encode(
522	self.tokens,
523	&self.tokens_count,
524	self.window[0..self.window_end],
525	);
526
527	// If we removed less than 1/16th, Huffman compress the block.
528	if (self.tokens_count > self.window_end - (self.window_end >> 4)) {
529	try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
530	} else {
531	try self.hm_bw.writeBlockDynamic(
532	self.tokens[0..self.tokens_count],
533	false,
534	self.window[0..self.window_end],
535	);
536	}
537	self.err = self.hm_bw.err;
538	self.window_end = 0;
539	}
540
541	fn initDeflate(self: *Self) !void {
542	self.window = try self.allocator.alloc(u8, 2 * window_size);
543	self.hash_offset = 1;
544	self.tokens = try self.allocator.alloc(token.Token, max_flate_block_tokens);
545	self.tokens_count = 0;
546	mem.set(token.Token, self.tokens, 0);
547	self.length = min_match_length - 1;
548	self.offset = 0;
549	self.byte_available = false;
550	self.index = 0;
551	self.hash = 0;
552	self.chain_head = 0;
553	self.bulk_hasher = bulkHash4;
554	}
555
556	fn deflate(self: *Self) !void {
557	if (self.window_end - self.index < min_match_length + max_match_length and !self.sync) {
558	return;
559	}
560
561	self.max_insert_index = self.window_end -\| (min_match_length - 1);
562	if (self.index < self.max_insert_index) {
563	self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
564	}
565
566	while (true) {
567	assert(self.index <= self.window_end);
568
569	var lookahead = self.window_end -\| self.index;
570	if (lookahead < min_match_length + max_match_length) {
571	if (!self.sync) {
572	break;
573	}
574	assert(self.index <= self.window_end);
575
576	if (lookahead == 0) {
577	// Flush current output block if any.
578	if (self.byte_available) {
579	// There is still one pending token that needs to be flushed
580	self.tokens[self.tokens_count] = token.literalToken(@intCast(u32, self.window[self.index - 1]));
581	self.tokens_count += 1;
582	self.byte_available = false;
583	}
584	if (self.tokens.len > 0) {
585	try self.writeBlock(self.tokens[0..self.tokens_count], self.index);
586	self.tokens_count = 0;
587	}
588	break;
589	}
590	}
591	if (self.index < self.max_insert_index) {
592	// Update the hash
593	self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
594	var hh = &self.hash_head[self.hash & hash_mask];
595	self.chain_head = @intCast(u32, hh.*);
596	self.hash_prev[self.index & window_mask] = @intCast(u32, self.chain_head);
597	hh.* = @intCast(u32, self.index + self.hash_offset);
598	}
599	var prev_length = self.length;
600	var prev_offset = self.offset;
601	self.length = min_match_length - 1;
602	self.offset = 0;
603	var min_index = self.index -\| window_size;
604
605	if (self.hash_offset <= self.chain_head and
606	self.chain_head - self.hash_offset >= min_index and
607	(self.compression_level.fast_skip_hashshing != skip_never and
608	lookahead > min_match_length - 1 or
609	self.compression_level.fast_skip_hashshing == skip_never and
610	lookahead > prev_length and
611	prev_length < self.compression_level.lazy))
612	{
613	{
614	var fmatch = self.findMatch(
615	self.index,
616	self.chain_head -\| self.hash_offset,
617	min_match_length - 1,
618	@intCast(u32, lookahead),
619	);
620	if (fmatch.ok) {
621	self.length = fmatch.length;
622	self.offset = fmatch.offset;
623	}
624	}
625	}
626	if (self.compression_level.fast_skip_hashshing != skip_never and
627	self.length >= min_match_length or
628	self.compression_level.fast_skip_hashshing == skip_never and
629	prev_length >= min_match_length and
630	self.length <= prev_length)
631	{
632	// There was a match at the previous step, and the current match is
633	// not better. Output the previous match.
634	if (self.compression_level.fast_skip_hashshing != skip_never) {
635	self.tokens[self.tokens_count] = token.matchToken(@intCast(u32, self.length - base_match_length), @intCast(u32, self.offset - base_match_offset));
636	self.tokens_count += 1;
637	} else {
638	self.tokens[self.tokens_count] = token.matchToken(
639	@intCast(u32, prev_length - base_match_length),
640	@intCast(u32, prev_offset -\| base_match_offset),
641	);
642	self.tokens_count += 1;
643	}
644	// Insert in the hash table all strings up to the end of the match.
645	// index and index-1 are already inserted. If there is not enough
646	// lookahead, the last two strings are not inserted into the hash
647	// table.
648	if (self.length <= self.compression_level.fast_skip_hashshing) {
649	var newIndex: u32 = 0;
650	if (self.compression_level.fast_skip_hashshing != skip_never) {
651	newIndex = self.index + self.length;
652	} else {
653	newIndex = self.index + prev_length - 1;
654	}
655	var index = self.index;
656	index += 1;
657	while (index < newIndex) : (index += 1) {
658	if (index < self.max_insert_index) {
659	self.hash = hash4(self.window[index .. index + min_match_length]);
660	// Get previous value with the same hash.
661	// Our chain should point to the previous value.
662	var hh = &self.hash_head[self.hash & hash_mask];
663	self.hash_prev[index & window_mask] = hh.*;
664	// Set the head of the hash chain to us.
665	hh.* = @intCast(u32, index + self.hash_offset);
666	}
667	}
668	self.index = index;
669
670	if (self.compression_level.fast_skip_hashshing == skip_never) {
671	self.byte_available = false;
672	self.length = min_match_length - 1;
673	}
674	} else {
675	// For matches this long, we don't bother inserting each individual
676	// item into the table.
677	self.index += self.length;
678	if (self.index < self.max_insert_index) {
679	self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
680	}
681	}
682	if (self.tokens_count == max_flate_block_tokens) {
683	// The block includes the current character
684	try self.writeBlock(self.tokens[0..self.tokens_count], self.index);
685	self.tokens_count = 0;
686	}
687	} else {
688	if (self.compression_level.fast_skip_hashshing != skip_never or self.byte_available) {
689	var i = self.index -\| 1;
690	if (self.compression_level.fast_skip_hashshing != skip_never) {
691	i = self.index;
692	}
693	self.tokens[self.tokens_count] = token.literalToken(@intCast(u32, self.window[i]));
694	self.tokens_count += 1;
695	if (self.tokens_count == max_flate_block_tokens) {
696	try self.writeBlock(self.tokens[0..self.tokens_count], i + 1);
697	self.tokens_count = 0;
698	}
699	}
700	self.index += 1;
701	if (self.compression_level.fast_skip_hashshing == skip_never) {
702	self.byte_available = true;
703	}
704	}
705	}
706	}
707
708	fn fillStore(self: *Self, b: []const u8) u32 {
709	var n = mu.copy(self.window[self.window_end..], b);
710	self.window_end += n;
711	return @intCast(u32, n);
712	}
713
714	fn store(self: *Self) !void {
715	if (self.window_end > 0 and (self.window_end == max_store_block_size or self.sync)) {
716	try self.writeStoredBlock(self.window[0..self.window_end]);
717	self.window_end = 0;
718	}
719	}
720
721	// storeHuff compresses and stores the currently added data
722	// when the self.window is full or we are at the end of the stream.
723	fn storeHuff(self: *Self) !void {
724	if (self.window_end < self.window.len and !self.sync or self.window_end == 0) {
725	return;
726	}
727	try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
728	self.err = self.hm_bw.err;
729	self.window_end = 0;
730	}
731
732	pub fn bytesWritten(self: *Self) usize {
733	return self.hm_bw.bytes_written;
734	}
735
736	/// Writes the compressed form of `input` to the underlying writer.
737	pub fn write(self: *Self, input: []const u8) !usize {
738	var buf = input;
739
740	// writes data to hm_bw, which will eventually write the
741	// compressed form of data to its underlying writer.
742	while (buf.len > 0) {
743	try self.step();
744	var filled = self.fill(buf);
745	buf = buf[filled..];
746	}
747
748	return input.len;
749	}
750
751	/// Flushes any pending data to the underlying writer.
752	/// It is useful mainly in compressed network protocols, to ensure that
753	/// a remote reader has enough data to reconstruct a packet.
754	/// Flush does not return until the data has been written.
755	/// Calling `flush()` when there is no pending data still causes the Writer
756	/// to emit a sync marker of at least 4 bytes.
757	/// If the underlying writer returns an error, `flush()` returns that error.
758	///
759	/// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
760	pub fn flush(self: *Self) !void {
761	self.sync = true;
762	try self.step();
763	try self.hm_bw.writeStoredHeader(0, false);
764	try self.hm_bw.flush();
765	self.sync = false;
766	return;
767	}
768
769	fn step(self: *Self) !void {
770	switch (self.compression) {
771	.no_compression => return self.store(),
772	.huffman_only => return self.storeHuff(),
773	.best_speed => return self.encSpeed(),
774	.default_compression,
775	.level_2,
776	.level_3,
777	.level_4,
778	.level_5,
779	.level_6,
780	.level_7,
781	.level_8,
782	.best_compression,
783	=> return self.deflate(),
784	}
785	}
786
787	fn fill(self: *Self, b: []const u8) u32 {
788	switch (self.compression) {
789	.no_compression => return self.fillStore(b),
790	.huffman_only => return self.fillStore(b),
791	.best_speed => return self.fillStore(b),
792	.default_compression,
793	.level_2,
794	.level_3,
795	.level_4,
796	.level_5,
797	.level_6,
798	.level_7,
799	.level_8,
800	.best_compression,
801	=> return self.fillDeflate(b),
802	}
803	}
804
805	fn init(
806	allocator: Allocator,
807	in_writer: WriterType,
808	options: CompressorOptions,
809	) !Self {
810	var s = Self{
811	.allocator = undefined,
812	.compression = undefined,
813	.compression_level = undefined,
814	.hm_bw = undefined, // HuffmanBitWriter
815	.bulk_hasher = undefined,
816	.sync = false,
817	.best_speed_enc = undefined, // Best speed encoder
818	.chain_head = 0,
819	.hash_head = undefined,
820	.hash_prev = undefined, // previous hash
821	.hash_offset = 0,
822	.index = 0,
823	.window = undefined,
824	.window_end = 0,
825	.block_start = 0,
826	.byte_available = false,
827	.tokens = undefined,
828	.tokens_count = 0,
829	.length = 0,
830	.offset = 0,
831	.hash = 0,
832	.max_insert_index = 0,
833	.err = false, // Error
834	.hash_match = undefined,
835	.dictionary = options.dictionary,
836	};
837
838	s.hm_bw = try hm_bw.huffmanBitWriter(allocator, in_writer);
839	s.allocator = allocator;
840
841	s.hash_head = try allocator.alloc(u32, hash_size);
842	s.hash_prev = try allocator.alloc(u32, window_size);
843	s.hash_match = try allocator.alloc(u32, max_match_length - 1);
844	mem.set(u32, s.hash_head, 0);
845	mem.set(u32, s.hash_prev, 0);
846	mem.set(u32, s.hash_match, 0);
847
848	switch (options.level) {
849	.no_compression => {
850	s.compression = options.level;
851	s.compression_level = levels(options.level);
852	s.window = try allocator.alloc(u8, max_store_block_size);
853	s.tokens = try allocator.alloc(token.Token, 0);
854	},
855	.huffman_only => {
856	s.compression = options.level;
857	s.compression_level = levels(options.level);
858	s.window = try allocator.alloc(u8, max_store_block_size);
859	s.tokens = try allocator.alloc(token.Token, 0);
860	},
861	.best_speed => {
862	s.compression = options.level;
863	s.compression_level = levels(options.level);
864	s.window = try allocator.alloc(u8, max_store_block_size);
865	s.tokens = try allocator.alloc(token.Token, max_store_block_size);
866	s.best_speed_enc = try allocator.create(fast.DeflateFast);
867	s.best_speed_enc.* = fast.deflateFast();
868	try s.best_speed_enc.init(allocator);
869	},
870	.default_compression => {
871	s.compression = .level_6;
872	s.compression_level = levels(.level_6);
873	try s.initDeflate();
874	if (options.dictionary != null) {
875	s.fillWindow(options.dictionary.?);
876	}
877	},
878	.level_2,
879	.level_3,
880	.level_4,
881	.level_5,
882	.level_6,
883	.level_7,
884	.level_8,
885	.best_compression,
886	=> {
887	s.compression = options.level;
888	s.compression_level = levels(options.level);
889	try s.initDeflate();
890	if (options.dictionary != null) {
891	s.fillWindow(options.dictionary.?);
892	}
893	},
894	}
895	return s;
896	}
897
898	/// Release all allocated memory.
899	pub fn deinit(self: *Self) void {
900	self.hm_bw.deinit();
901	self.allocator.free(self.window);
902	self.allocator.free(self.tokens);
903	self.allocator.free(self.hash_head);
904	self.allocator.free(self.hash_prev);
905	self.allocator.free(self.hash_match);
906	if (self.compression == .best_speed) {
907	self.best_speed_enc.deinit();
908	self.allocator.destroy(self.best_speed_enc);
909	}
910	}
911
912	/// Reset discards the inner writer's state and replace the inner writer with new_writer.
913	/// new_writer must be of the same type as the previous writer.
914	pub fn reset(self: *Self, new_writer: WriterType) void {
915	self.hm_bw.reset(new_writer);
916	self.sync = false;
917	switch (self.compression) {
918	// Reset window
919	.no_compression => self.window_end = 0,
920	// Reset window, tokens, and encoder
921	.best_speed => {
922	self.window_end = 0;
923	self.tokens_count = 0;
924	self.best_speed_enc.reset();
925	},
926	// Reset everything and reinclude the dictionary if there is one
927	.huffman_only,
928	.default_compression,
929	.level_2,
930	.level_3,
931	.level_4,
932	.level_5,
933	.level_6,
934	.level_7,
935	.level_8,
936	.best_compression,
937	=> {
938	self.chain_head = 0;
939	mem.set(u32, self.hash_head, 0);
940	mem.set(u32, self.hash_prev, 0);
941	self.hash_offset = 1;
942	self.index = 0;
943	self.window_end = 0;
944	self.block_start = 0;
945	self.byte_available = false;
946	self.tokens_count = 0;
947	self.length = min_match_length - 1;
948	self.offset = 0;
949	self.hash = 0;
950	self.max_insert_index = 0;
951
952	if (self.dictionary != null) {
953	self.fillWindow(self.dictionary.?);
954	}
955	},
956	}
957	}
958
959	/// Writes any pending data to the underlying writer.
960	pub fn close(self: *Self) !void {
961	self.sync = true;
962	try self.step();
963	try self.hm_bw.writeStoredHeader(0, true);
964	try self.hm_bw.flush();
965	return;
966	}
967	};
968	}
969
970	// tests
971
972	const expect = std.testing.expect;
973	const testing = std.testing;
974
975	const ArrayList = std.ArrayList;
976
977	const DeflateTest = struct {
978	in: []const u8,
979	level: Compression,
980	out: []const u8,
981	};
982
983	var deflate_tests = [_]DeflateTest{
984	// Level 0
985	.{
986	.in = &[_]u8{},
987	.level = .no_compression,
988	.out = &[_]u8{ 1, 0, 0, 255, 255 },
989	},
990
991	// Level -1
992	.{
993	.in = &[_]u8{0x11},
994	.level = .default_compression,
995	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
996	},
997	.{
998	.in = &[_]u8{0x11},
999	.level = .level_6,
1000	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1001	},
1002
1003	// Level 4
1004	.{
1005	.in = &[_]u8{0x11},
1006	.level = .level_4,
1007	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1008	},
1009
1010	// Level 0
1011	.{
1012	.in = &[_]u8{0x11},
1013	.level = .no_compression,
1014	.out = &[_]u8{ 0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255 },
1015	},
1016	.{
1017	.in = &[_]u8{ 0x11, 0x12 },
1018	.level = .no_compression,
1019	.out = &[_]u8{ 0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255 },
1020	},
1021	.{
1022	.in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1023	.level = .no_compression,
1024	.out = &[_]u8{ 0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255 },
1025	},
1026
1027	// Level 2
1028	.{
1029	.in = &[_]u8{},
1030	.level = .level_2,
1031	.out = &[_]u8{ 1, 0, 0, 255, 255 },
1032	},
1033	.{
1034	.in = &[_]u8{0x11},
1035	.level = .level_2,
1036	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1037	},
1038	.{
1039	.in = &[_]u8{ 0x11, 0x12 },
1040	.level = .level_2,
1041	.out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 },
1042	},
1043	.{
1044	.in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1045	.level = .level_2,
1046	.out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 },
1047	},
1048
1049	// Level 9
1050	.{
1051	.in = &[_]u8{},
1052	.level = .best_compression,
1053	.out = &[_]u8{ 1, 0, 0, 255, 255 },
1054	},
1055	.{
1056	.in = &[_]u8{0x11},
1057	.level = .best_compression,
1058	.out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1059	},
1060	.{
1061	.in = &[_]u8{ 0x11, 0x12 },
1062	.level = .best_compression,
1063	.out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 },
1064	},
1065	.{
1066	.in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1067	.level = .best_compression,
1068	.out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 },
1069	},
1070	};
1071
1072	test "deflate" {
1073	for (deflate_tests) \|dt\| {
1074	var output = ArrayList(u8).init(testing.allocator);
1075	defer output.deinit();
1076
1077	var comp = try compressor(testing.allocator, output.writer(), .{ .level = dt.level });
1078	_ = try comp.write(dt.in);
1079	try comp.close();
1080	comp.deinit();
1081
1082	try expect(mem.eql(u8, output.items, dt.out));
1083	}
1084	}
1085
1086	test "bulkHash4" {
1087	for (deflate_tests) \|x\| {
1088	if (x.out.len < min_match_length) {
1089	continue;
1090	}
1091	// double the test data
1092	var out = try testing.allocator.alloc(u8, x.out.len * 2);
1093	defer testing.allocator.free(out);
1094	mem.copy(u8, out[0..x.out.len], x.out);
1095	mem.copy(u8, out[x.out.len..], x.out);
1096
1097	var j: usize = 4;
1098	while (j < out.len) : (j += 1) {
1099	var y = out[0..j];
1100
1101	var dst = try testing.allocator.alloc(u32, y.len - min_match_length + 1);
1102	defer testing.allocator.free(dst);
1103
1104	_ = bulkHash4(y, dst);
1105	for (dst) \|got, i\| {
1106	var want = hash4(y[i..]);
1107	try expect(got == want);
1108	}
1109	}
1110	}
1111	}

lib/std/compress/deflate/compressor_test.zig created+560

...	@@ -0,0 +1,560 @@
1	const std = @import("std");
2	const builtin = @import("builtin");
3	const expect = std.testing.expect;
4	const fifo = std.fifo;
5	const io = std.io;
6	const math = std.math;
7	const mem = std.mem;
8	const testing = std.testing;
9
10	const ArrayList = std.ArrayList;
11
12	const deflate = @import("compressor.zig");
13	const inflate = @import("decompressor.zig");
14
15	const compressor = deflate.compressor;
16	const decompressor = inflate.decompressor;
17	const huffman_only = deflate.huffman_only;
18
19	fn testSync(level: deflate.Compression, input: []const u8) !void {
20	if (input.len == 0) {
21	return;
22	}
23
24	var divided_buf = fifo
25	.LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
26	.init(testing.allocator);
27	defer divided_buf.deinit();
28	var whole_buf = std.ArrayList(u8).init(testing.allocator);
29	defer whole_buf.deinit();
30
31	var multi_writer = io.multiWriter(.{
32	divided_buf.writer(),
33	whole_buf.writer(),
34	}).writer();
35
36	var comp = try compressor(
37	testing.allocator,
38	multi_writer,
39	.{ .level = level },
40	);
41	defer comp.deinit();
42
43	{
44	var decomp = try decompressor(
45	testing.allocator,
46	divided_buf.reader(),
47	null,
48	);
49	defer decomp.deinit();
50
51	// Write first half of the input and flush()
52	var half: usize = (input.len + 1) / 2;
53	var half_len: usize = half - 0;
54	{
55	_ = try comp.writer().writeAll(input[0..half]);
56
57	// Flush
58	try comp.flush();
59
60	// Read back
61	var decompressed = try testing.allocator.alloc(u8, half_len);
62	defer testing.allocator.free(decompressed);
63
64	var read = try decomp.reader().readAll(decompressed); // read at least half
65	try expect(read == half_len);
66	try expect(mem.eql(u8, input[0..half], decompressed));
67	}
68
69	// Write last half of the input and close()
70	half_len = input.len - half;
71	{
72	_ = try comp.writer().writeAll(input[half..]);
73
74	// Close
75	try comp.close();
76
77	// Read back
78	var decompressed = try testing.allocator.alloc(u8, half_len);
79	defer testing.allocator.free(decompressed);
80
81	var read = try decomp.reader().readAll(decompressed);
82	try expect(read == half_len);
83	try expect(mem.eql(u8, input[half..], decompressed));
84
85	// Extra read
86	var final: [10]u8 = undefined;
87	read = try decomp.reader().readAll(&final);
88	try expect(read == 0); // expect ended stream to return 0 bytes
89
90	_ = decomp.close();
91	}
92	}
93
94	_ = try comp.writer().writeAll(input);
95	try comp.close();
96
97	// stream should work for ordinary reader too (reading whole_buf in one go)
98	var whole_buf_reader = io.fixedBufferStream(whole_buf.items).reader();
99	var decomp = try decompressor(testing.allocator, whole_buf_reader, null);
100	defer decomp.deinit();
101
102	var decompressed = try testing.allocator.alloc(u8, input.len);
103	defer testing.allocator.free(decompressed);
104
105	_ = try decomp.reader().readAll(decompressed);
106	_ = decomp.close();
107
108	try expect(mem.eql(u8, input, decompressed));
109	}
110
111	fn testToFromWithLevelAndLimit(level: deflate.Compression, input: []const u8, limit: u32) !void {
112	var compressed = std.ArrayList(u8).init(testing.allocator);
113	defer compressed.deinit();
114
115	var comp = try compressor(testing.allocator, compressed.writer(), .{ .level = level });
116	defer comp.deinit();
117
118	try comp.writer().writeAll(input);
119	try comp.close();
120
121	if (limit > 0) {
122	try expect(compressed.items.len <= limit);
123	}
124
125	var decomp = try decompressor(
126	testing.allocator,
127	io.fixedBufferStream(compressed.items).reader(),
128	null,
129	);
130	defer decomp.deinit();
131
132	var decompressed = try testing.allocator.alloc(u8, input.len);
133	defer testing.allocator.free(decompressed);
134
135	var read: usize = try decomp.reader().readAll(decompressed);
136	try expect(read == input.len);
137	try expect(mem.eql(u8, input, decompressed));
138
139	try testSync(level, input);
140	}
141
142	fn testToFromWithLimit(input: []const u8, limit: [11]u32) !void {
143	try testToFromWithLevelAndLimit(.no_compression, input, limit[0]);
144	try testToFromWithLevelAndLimit(.best_speed, input, limit[1]);
145	try testToFromWithLevelAndLimit(.level_2, input, limit[2]);
146	try testToFromWithLevelAndLimit(.level_3, input, limit[3]);
147	try testToFromWithLevelAndLimit(.level_4, input, limit[4]);
148	try testToFromWithLevelAndLimit(.level_5, input, limit[5]);
149	try testToFromWithLevelAndLimit(.level_6, input, limit[6]);
150	try testToFromWithLevelAndLimit(.level_7, input, limit[7]);
151	try testToFromWithLevelAndLimit(.level_8, input, limit[8]);
152	try testToFromWithLevelAndLimit(.best_compression, input, limit[9]);
153	try testToFromWithLevelAndLimit(.huffman_only, input, limit[10]);
154	}
155
156	test "deflate/inflate" {
157	var limits = [_]u32{0} ** 11;
158
159	var test0 = [_]u8{};
160	var test1 = [_]u8{0x11};
161	var test2 = [_]u8{ 0x11, 0x12 };
162	var test3 = [_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 };
163	var test4 = [_]u8{ 0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13 };
164
165	try testToFromWithLimit(&test0, limits);
166	try testToFromWithLimit(&test1, limits);
167	try testToFromWithLimit(&test2, limits);
168	try testToFromWithLimit(&test3, limits);
169	try testToFromWithLimit(&test4, limits);
170
171	var large_data_chunk = try testing.allocator.alloc(u8, 100_000);
172	defer testing.allocator.free(large_data_chunk);
173	// fill with random data
174	for (large_data_chunk) \|_, i\| {
175	var mul: u8 = @truncate(u8, i);
176	_ = @mulWithOverflow(u8, mul, mul, &mul);
177	large_data_chunk[i] = mul;
178	}
179	try testToFromWithLimit(large_data_chunk, limits);
180	}
181
182	test "very long sparse chunk" {
183	// A SparseReader returns a stream consisting of 0s ending with 65,536 (1<<16) 1s.
184	// This tests missing hash references in a very large input.
185	const SparseReader = struct {
186	l: usize, // length
187	cur: usize, // current position
188
189	const Self = @This();
190	const Error = error{};
191
192	pub const Reader = io.Reader(*Self, Error, read);
193
194	pub fn reader(self: *Self) Reader {
195	return .{ .context = self };
196	}
197
198	fn read(s: *Self, b: []u8) Error!usize {
199	var n: usize = 0; // amount read
200
201	if (s.cur >= s.l) {
202	return 0;
203	}
204	n = b.len;
205	var cur = s.cur + n;
206	if (cur > s.l) {
207	n -= cur - s.l;
208	cur = s.l;
209	}
210	for (b[0..n]) \|_, i\| {
211	if (s.cur + i >= s.l -\| (1 << 16)) {
212	b[i] = 1;
213	} else {
214	b[i] = 0;
215	}
216	}
217	s.cur = cur;
218	return n;
219	}
220	};
221
222	var comp = try compressor(
223	testing.allocator,
224	io.null_writer,
225	.{ .level = .best_speed },
226	);
227	defer comp.deinit();
228	var writer = comp.writer();
229
230	var sparse = SparseReader{ .l = 0x23e8, .cur = 0 };
231	var reader = sparse.reader();
232
233	var read: usize = 1;
234	var written: usize = 0;
235	while (read > 0) {
236	var buf: [1 << 15]u8 = undefined; // 32,768 bytes buffer
237	read = try reader.read(&buf);
238	written += try writer.write(buf[0..read]);
239	}
240	try expect(written == 0x23e8);
241	}
242
243	test "compressor reset" {
244	for (std.enums.values(deflate.Compression)) \|c\| {
245	try testWriterReset(c, null);
246	try testWriterReset(c, "dict");
247	try testWriterReset(c, "hello");
248	}
249	}
250
251	fn testWriterReset(level: deflate.Compression, dict: ?[]const u8) !void {
252	const filler = struct {
253	fn writeData(c: anytype) !void {
254	const msg = "all your base are belong to us";
255	try c.writer().writeAll(msg);
256	try c.flush();
257
258	const hello = "hello world";
259	var i: usize = 0;
260	while (i < 1024) : (i += 1) {
261	try c.writer().writeAll(hello);
262	}
263
264	i = 0;
265	while (i < 65000) : (i += 1) {
266	try c.writer().writeAll("x");
267	}
268	}
269	};
270
271	var buf1 = ArrayList(u8).init(testing.allocator);
272	defer buf1.deinit();
273	var buf2 = ArrayList(u8).init(testing.allocator);
274	defer buf2.deinit();
275
276	var comp = try compressor(
277	testing.allocator,
278	buf1.writer(),
279	.{ .level = level, .dictionary = dict },
280	);
281	defer comp.deinit();
282
283	try filler.writeData(&comp);
284	try comp.close();
285
286	comp.reset(buf2.writer());
287	try filler.writeData(&comp);
288	try comp.close();
289
290	try expect(mem.eql(u8, buf1.items, buf2.items));
291	}
292
293	test "decompressor dictionary" {
294	const dict = "hello world"; // dictionary
295	const text = "hello again world";
296
297	var compressed = fifo
298	.LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
299	.init(testing.allocator);
300	defer compressed.deinit();
301
302	var comp = try compressor(
303	testing.allocator,
304	compressed.writer(),
305	.{
306	.level = .level_5,
307	.dictionary = null, // no dictionary
308	},
309	);
310	defer comp.deinit();
311
312	// imitate a compressor with a dictionary
313	try comp.writer().writeAll(dict);
314	try comp.flush();
315	compressed.discard(compressed.readableLength()); // empty the output
316	try comp.writer().writeAll(text);
317	try comp.close();
318
319	var decompressed = try testing.allocator.alloc(u8, text.len);
320	defer testing.allocator.free(decompressed);
321
322	var decomp = try decompressor(
323	testing.allocator,
324	compressed.reader(),
325	dict,
326	);
327	defer decomp.deinit();
328
329	_ = try decomp.reader().readAll(decompressed);
330	try expect(mem.eql(u8, decompressed, "hello again world"));
331	}
332
333	test "compressor dictionary" {
334	const dict = "hello world";
335	const text = "hello again world";
336
337	var compressed_nd = fifo
338	.LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
339	.init(testing.allocator); // compressed with no dictionary
340	defer compressed_nd.deinit();
341
342	var compressed_d = ArrayList(u8).init(testing.allocator); // compressed with a dictionary
343	defer compressed_d.deinit();
344
345	// imitate a compressor with a dictionary
346	var comp_nd = try compressor(
347	testing.allocator,
348	compressed_nd.writer(),
349	.{
350	.level = .level_5,
351	.dictionary = null, // no dictionary
352	},
353	);
354	defer comp_nd.deinit();
355	try comp_nd.writer().writeAll(dict);
356	try comp_nd.flush();
357	compressed_nd.discard(compressed_nd.readableLength()); // empty the output
358	try comp_nd.writer().writeAll(text);
359	try comp_nd.close();
360
361	// use a compressor with a dictionary
362	var comp_d = try compressor(
363	testing.allocator,
364	compressed_d.writer(),
365	.{
366	.level = .level_5,
367	.dictionary = dict, // with a dictionary
368	},
369	);
370	defer comp_d.deinit();
371	try comp_d.writer().writeAll(text);
372	try comp_d.close();
373
374	try expect(mem.eql(u8, compressed_nd.readableSlice(0), compressed_d.items));
375	}
376
377	// Update the hash for best_speed only if d.index < d.maxInsertIndex
378	// See https://golang.org/issue/2508
379	test "Go non-regression test for 2508" {
380	var comp = try compressor(
381	testing.allocator,
382	io.null_writer,
383	.{ .level = .best_speed },
384	);
385	defer comp.deinit();
386
387	var buf = [_]u8{0} ** 1024;
388
389	var i: usize = 0;
390	while (i < 131_072) : (i += 1) {
391	try comp.writer().writeAll(&buf);
392	try comp.close();
393	}
394	}
395
396	test "deflate/inflate string" {
397	// Skip wasi because it does not support std.fs.openDirAbsolute()
398	if (builtin.os.tag == .wasi) return error.SkipZigTest;
399
400	const current_dir = try std.fs.openDirAbsolute(std.fs.path.dirname(@src().file).?, .{});
401	const testdata_dir = try current_dir.openDir("testdata", .{});
402
403	const StringTest = struct {
404	filename: []const u8,
405	limit: [11]u32,
406	};
407
408	var deflate_inflate_string_tests = [_]StringTest{
409	.{
410	.filename = "compress-e.txt",
411	.limit = [11]u32{
412	100_018, // no_compression
413	50_650, // best_speed
414	50_960, // 2
415	51_150, // 3
416	50_930, // 4
417	50_790, // 5
418	50_790, // 6
419	50_790, // 7
420	50_790, // 8
421	50_790, // best_compression
422	43_683, // huffman_only
423	},
424	},
425	.{
426	.filename = "rfc1951.txt",
427	.limit = [11]u32{
428	36_954, // no_compression
429	12_952, // best_speed
430	12_228, // 2
431	12_016, // 3
432	11_466, // 4
433	11_191, // 5
434	11_129, // 6
435	11_120, // 7
436	11_112, // 8
437	11_109, // best_compression
438	20_273, // huffman_only
439	},
440	},
441	};
442
443	for (deflate_inflate_string_tests) \|t\| {
444	const golden_file = try testdata_dir.openFile(t.filename, .{ .read = true });
445	defer golden_file.close();
446	var golden = try golden_file.reader().readAllAlloc(testing.allocator, math.maxInt(usize));
447	defer testing.allocator.free(golden);
448
449	try testToFromWithLimit(golden, t.limit);
450	}
451	}
452
453	test "inflate reset" {
454	const strings = [_][]const u8{
455	"lorem ipsum izzle fo rizzle",
456	"the quick brown fox jumped over",
457	};
458
459	var compressed_strings = [_]ArrayList(u8){
460	ArrayList(u8).init(testing.allocator),
461	ArrayList(u8).init(testing.allocator),
462	};
463	defer compressed_strings[0].deinit();
464	defer compressed_strings[1].deinit();
465
466	for (strings) \|s, i\| {
467	var comp = try compressor(
468	testing.allocator,
469	compressed_strings[i].writer(),
470	.{ .level = .level_6 },
471	);
472	defer comp.deinit();
473
474	try comp.writer().writeAll(s);
475	try comp.close();
476	}
477
478	var decomp = try decompressor(
479	testing.allocator,
480	io.fixedBufferStream(compressed_strings[0].items).reader(),
481	null,
482	);
483	defer decomp.deinit();
484
485	var decompressed_0: []u8 = try decomp.reader()
486	.readAllAlloc(testing.allocator, math.maxInt(usize));
487	defer testing.allocator.free(decompressed_0);
488
489	try decomp.reset(
490	io.fixedBufferStream(compressed_strings[1].items).reader(),
491	null,
492	);
493
494	var decompressed_1: []u8 = try decomp.reader()
495	.readAllAlloc(testing.allocator, math.maxInt(usize));
496	defer testing.allocator.free(decompressed_1);
497
498	_ = decomp.close();
499
500	try expect(strings[0].len == decompressed_0.len);
501	try expect(strings[1].len == decompressed_1.len);
502
503	try expect(mem.eql(u8, strings[0], decompressed_0));
504	try expect(mem.eql(u8, strings[1], decompressed_1));
505	}
506
507	test "inflate reset dictionary" {
508	const dict = "the lorem fox";
509	const strings = [_][]const u8{
510	"lorem ipsum izzle fo rizzle",
511	"the quick brown fox jumped over",
512	};
513
514	var compressed_strings = [_]ArrayList(u8){
515	ArrayList(u8).init(testing.allocator),
516	ArrayList(u8).init(testing.allocator),
517	};
518	defer compressed_strings[0].deinit();
519	defer compressed_strings[1].deinit();
520
521	for (strings) \|s, i\| {
522	var comp = try compressor(
523	testing.allocator,
524	compressed_strings[i].writer(),
525	.{ .level = .level_6 },
526	);
527	defer comp.deinit();
528
529	try comp.writer().writeAll(s);
530	try comp.close();
531	}
532
533	var decomp = try decompressor(
534	testing.allocator,
535	io.fixedBufferStream(compressed_strings[0].items).reader(),
536	dict,
537	);
538	defer decomp.deinit();
539
540	var decompressed_0: []u8 = try decomp.reader()
541	.readAllAlloc(testing.allocator, math.maxInt(usize));
542	defer testing.allocator.free(decompressed_0);
543
544	try decomp.reset(
545	io.fixedBufferStream(compressed_strings[1].items).reader(),
546	dict,
547	);
548
549	var decompressed_1: []u8 = try decomp.reader()
550	.readAllAlloc(testing.allocator, math.maxInt(usize));
551	defer testing.allocator.free(decompressed_1);
552
553	_ = decomp.close();
554
555	try expect(strings[0].len == decompressed_0.len);
556	try expect(strings[1].len == decompressed_1.len);
557
558	try expect(mem.eql(u8, strings[0], decompressed_0));
559	try expect(mem.eql(u8, strings[1], decompressed_1));
560	}

lib/std/compress/deflate/decompressor.zig created+1090

...	@@ -0,0 +1,1090 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5
6	const Allocator = std.mem.Allocator;
7	const ArrayList = std.ArrayList;
8
9	const bu = @import("bits_utils.zig");
10	const ddec = @import("dict_decoder.zig");
11	const deflate_const = @import("deflate_const.zig");
12	const mu = @import("mem_utils.zig");
13
14	const max_match_offset = deflate_const.max_match_offset;
15	const end_block_marker = deflate_const.end_block_marker;
16
17	const max_code_len = 16; // max length of Huffman code
18	// The next three numbers come from the RFC section 3.2.7, with the
19	// additional proviso in section 3.2.5 which implies that distance codes
20	// 30 and 31 should never occur in compressed data.
21	const max_num_lit = 286;
22	const max_num_dist = 30;
23	const num_codes = 19; // number of codes in Huffman meta-code
24
25	var corrupt_input_error_offset: u64 = undefined;
26
27	const InflateError = error{
28	CorruptInput, // A CorruptInput error reports the presence of corrupt input at a given offset.
29	BadInternalState, // An BadInternalState reports an error in the flate code itself.
30	BadReaderState, // An error was encountered while accessing the inner reader
31	UnexpectedEndOfStream,
32	EndOfStreamWithNoError,
33	};
34
35	// The data structure for decoding Huffman tables is based on that of
36	// zlib. There is a lookup table of a fixed bit width (huffman_chunk_bits),
37	// For codes smaller than the table width, there are multiple entries
38	// (each combination of trailing bits has the same value). For codes
39	// larger than the table width, the table contains a link to an overflow
40	// table. The width of each entry in the link table is the maximum code
41	// size minus the chunk width.
42	//
43	// Note that you can do a lookup in the table even without all bits
44	// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
45	// have the property that shorter codes come before longer ones, the
46	// bit length estimate in the result is a lower bound on the actual
47	// number of bits.
48	//
49	// See the following:
50	// https://github.com/madler/zlib/raw/master/doc/algorithm.txt
51
52	// chunk & 15 is number of bits
53	// chunk >> 4 is value, including table link
54
55	const huffman_chunk_bits = 9;
56	const huffman_num_chunks = 1 << huffman_chunk_bits; // 512
57	const huffman_count_mask = 15; // 0b1111
58	const huffman_value_shift = 4;
59
60	const HuffmanDecoder = struct {
61	const Self = @This();
62
63	allocator: Allocator = undefined,
64
65	min: u32 = 0, // the minimum code length
66	chunks: [huffman_num_chunks]u16 = [1]u16{0} ** huffman_num_chunks, // chunks as described above
67	links: [][]u16 = undefined, // overflow links
68	link_mask: u32 = 0, // mask the width of the link table
69	initialized: bool = false,
70	sub_chunks: ArrayList(u32) = undefined,
71
72	// Initialize Huffman decoding tables from array of code lengths.
73	// Following this function, self is guaranteed to be initialized into a complete
74	// tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
75	// degenerate case where the tree has only a single symbol with length 1. Empty
76	// trees are permitted.
77	fn init(self: *Self, allocator: Allocator, lengths: []u32) !bool {
78
79	// Sanity enables additional runtime tests during Huffman
80	// table construction. It's intended to be used during
81	// development and debugging
82	const sanity = false;
83
84	if (self.min != 0) {
85	self.* = HuffmanDecoder{};
86	}
87
88	self.allocator = allocator;
89
90	// Count number of codes of each length,
91	// compute min and max length.
92	var count: [max_code_len]u32 = [1]u32{0} ** max_code_len;
93	var min: u32 = 0;
94	var max: u32 = 0;
95	for (lengths) \|n\| {
96	if (n == 0) {
97	continue;
98	}
99	if (min == 0) {
100	min = n;
101	}
102	min = @minimum(n, min);
103	max = @maximum(n, max);
104	count[n] += 1;
105	}
106
107	// Empty tree. The decompressor.huffSym function will fail later if the tree
108	// is used. Technically, an empty tree is only valid for the HDIST tree and
109	// not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
110	// is guaranteed to fail since it will attempt to use the tree to decode the
111	// codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
112	// guaranteed to fail later since the compressed data section must be
113	// composed of at least one symbol (the end-of-block marker).
114	if (max == 0) {
115	return true;
116	}
117
118	var next_code: [max_code_len]u32 = [1]u32{0} ** max_code_len;
119	var code: u32 = 0;
120	{
121	var i = min;
122	while (i <= max) : (i += 1) {
123	code <<= 1;
124	next_code[i] = code;
125	code += count[i];
126	}
127	}
128
129	// Check that the coding is complete (i.e., that we've
130	// assigned all 2-to-the-max possible bit sequences).
131	// Exception: To be compatible with zlib, we also need to
132	// accept degenerate single-code codings. See also
133	// TestDegenerateHuffmanCoding.
134	if (code != @as(u32, 1) << @intCast(u5, max) and !(code == 1 and max == 1)) {
135	return false;
136	}
137
138	self.min = min;
139	if (max > huffman_chunk_bits) {
140	var num_links = @as(u32, 1) << @intCast(u5, max - huffman_chunk_bits);
141	self.link_mask = @intCast(u32, num_links - 1);
142
143	// create link tables
144	var link = next_code[huffman_chunk_bits + 1] >> 1;
145	self.links = try self.allocator.alloc([]u16, huffman_num_chunks - link);
146	self.sub_chunks = ArrayList(u32).init(self.allocator);
147	self.initialized = true;
148	var j = @intCast(u32, link);
149	while (j < huffman_num_chunks) : (j += 1) {
150	var reverse = @intCast(u32, bu.bitReverse(u16, @intCast(u16, j), 16));
151	reverse >>= @intCast(u32, 16 - huffman_chunk_bits);
152	var off = j - @intCast(u32, link);
153	if (sanity) {
154	// check we are not overwriting an existing chunk
155	assert(self.chunks[reverse] == 0);
156	}
157	self.chunks[reverse] = @intCast(u16, off << huffman_value_shift \| (huffman_chunk_bits + 1));
158	self.links[off] = try self.allocator.alloc(u16, num_links);
159	if (sanity) {
160	// initialize to a known invalid chunk code (0) to see if we overwrite
161	// this value later on
162	mem.set(u16, self.links[off], 0);
163	}
164	try self.sub_chunks.append(off);
165	}
166	}
167
168	for (lengths) \|n, li\| {
169	if (n == 0) {
170	continue;
171	}
172	var ncode = next_code[n];
173	next_code[n] += 1;
174	var chunk = @intCast(u16, (li << huffman_value_shift) \| n);
175	var reverse = @intCast(u16, bu.bitReverse(u16, @intCast(u16, ncode), 16));
176	reverse >>= @intCast(u4, 16 - n);
177	if (n <= huffman_chunk_bits) {
178	var off = reverse;
179	while (off < self.chunks.len) : (off += @as(u16, 1) << @intCast(u4, n)) {
180	// We should never need to overwrite
181	// an existing chunk. Also, 0 is
182	// never a valid chunk, because the
183	// lower 4 "count" bits should be
184	// between 1 and 15.
185	if (sanity) {
186	assert(self.chunks[off] == 0);
187	}
188	self.chunks[off] = chunk;
189	}
190	} else {
191	var j = reverse & (huffman_num_chunks - 1);
192	if (sanity) {
193	// Expect an indirect chunk
194	assert(self.chunks[j] & huffman_count_mask == huffman_chunk_bits + 1);
195	// Longer codes should have been
196	// associated with a link table above.
197	}
198	var value = self.chunks[j] >> huffman_value_shift;
199	var link_tab = self.links[value];
200	reverse >>= huffman_chunk_bits;
201	var off = reverse;
202	while (off < link_tab.len) : (off += @as(u16, 1) << @intCast(u4, n - huffman_chunk_bits)) {
203	if (sanity) {
204	// check we are not overwriting an existing chunk
205	assert(link_tab[off] == 0);
206	}
207	link_tab[off] = @intCast(u16, chunk);
208	}
209	}
210	}
211
212	if (sanity) {
213	// Above we've sanity checked that we never overwrote
214	// an existing entry. Here we additionally check that
215	// we filled the tables completely.
216	for (self.chunks) \|chunk, i\| {
217	// As an exception, in the degenerate
218	// single-code case, we allow odd
219	// chunks to be missing.
220	if (code == 1 and i % 2 == 1) {
221	continue;
222	}
223
224	// Assert we are not missing a chunk.
225	// All chunks should have been written once
226	// thus losing their initial value of 0
227	assert(chunk != 0);
228	}
229
230	if (self.initialized) {
231	for (self.links) \|link_tab\| {
232	for (link_tab) \|chunk\| {
233	// Assert we are not missing a chunk.
234	assert(chunk != 0);
235	}
236	}
237	}
238	}
239
240	return true;
241	}
242
243	/// Release all allocated memory.
244	pub fn deinit(self: *Self) void {
245	if (self.initialized and self.links.len > 0) {
246	for (self.sub_chunks.items) \|off\| {
247	self.allocator.free(self.links[off]);
248	}
249	self.allocator.free(self.links);
250	self.sub_chunks.deinit();
251	self.initialized = false;
252	}
253	}
254	};
255
256	var fixed_huffman_decoder: ?HuffmanDecoder = null;
257
258	fn fixedHuffmanDecoderInit(allocator: Allocator) !HuffmanDecoder {
259	if (fixed_huffman_decoder != null) {
260	return fixed_huffman_decoder.?;
261	}
262
263	// These come from the RFC section 3.2.6.
264	var bits: [288]u32 = undefined;
265	var i: u32 = 0;
266	while (i < 144) : (i += 1) {
267	bits[i] = 8;
268	}
269	while (i < 256) : (i += 1) {
270	bits[i] = 9;
271	}
272	while (i < 280) : (i += 1) {
273	bits[i] = 7;
274	}
275	while (i < 288) : (i += 1) {
276	bits[i] = 8;
277	}
278
279	fixed_huffman_decoder = HuffmanDecoder{};
280	_ = try fixed_huffman_decoder.?.init(allocator, &bits);
281	return fixed_huffman_decoder.?;
282	}
283
284	const DecompressorState = enum {
285	init,
286	dict,
287	};
288
289	/// Returns a new Decompressor that can be used to read the uncompressed version of `reader`.
290	/// `dictionary` is optional and initializes the Decompressor with a preset dictionary.
291	/// The returned Decompressor behaves as if the uncompressed data stream started with the given
292	/// dictionary, which has already been read. Use the same `dictionary` as the compressor used to
293	/// compress the data.
294	/// This decompressor may use at most 300 KiB of heap memory from the provided allocator.
295	/// The uncompressed data will be written into the provided buffer, see `reader()` and `read()`.
296	pub fn decompressor(allocator: Allocator, reader: anytype, dictionary: ?[]const u8) !Decompressor(@TypeOf(reader)) {
297	return Decompressor(@TypeOf(reader)).init(allocator, reader, dictionary);
298	}
299
300	pub fn Decompressor(comptime ReaderType: type) type {
301	return struct {
302	const Self = @This();
303
304	pub const Error =
305	ReaderType.Error \|\|
306	error{EndOfStream} \|\|
307	InflateError \|\|
308	Allocator.Error;
309	pub const Reader = io.Reader(*Self, Error, read);
310
311	allocator: Allocator,
312
313	// Input source.
314	inner_reader: ReaderType,
315	roffset: u64,
316
317	// Input bits, in top of b.
318	b: u32,
319	nb: u32,
320
321	// Huffman decoders for literal/length, distance.
322	hd1: HuffmanDecoder,
323	hd2: HuffmanDecoder,
324
325	// Length arrays used to define Huffman codes.
326	bits: *[max_num_lit + max_num_dist]u32,
327	codebits: *[num_codes]u32,
328
329	// Output history, buffer.
330	dict: ddec.DictDecoder,
331
332	// Temporary buffer (avoids repeated allocation).
333	buf: [4]u8,
334
335	// Next step in the decompression,
336	// and decompression state.
337	step: fn (*Self) Error!void,
338	step_state: DecompressorState,
339	final: bool,
340	err: ?Error,
341	to_read: []u8,
342	// Huffman states for the lit/length values
343	hl: ?*HuffmanDecoder,
344	// Huffman states for the distance values.
345	hd: ?*HuffmanDecoder,
346	copy_len: u32,
347	copy_dist: u32,
348
349	/// Returns a Reader that reads compressed data from an underlying reader and outputs
350	/// uncompressed data.
351	pub fn reader(self: *Self) Reader {
352	return .{ .context = self };
353	}
354
355	fn init(allocator: Allocator, in_reader: ReaderType, dict: ?[]const u8) !Self {
356	fixed_huffman_decoder = try fixedHuffmanDecoderInit(allocator);
357
358	var bits = try allocator.create([max_num_lit + max_num_dist]u32);
359	var codebits = try allocator.create([num_codes]u32);
360
361	var dd = ddec.DictDecoder{};
362	try dd.init(allocator, max_match_offset, dict);
363
364	return Self{
365	.allocator = allocator,
366
367	// Input source.
368	.inner_reader = in_reader,
369	.roffset = 0,
370
371	// Input bits, in top of b.
372	.b = 0,
373	.nb = 0,
374
375	// Huffman decoders for literal/length, distance.
376	.hd1 = HuffmanDecoder{},
377	.hd2 = HuffmanDecoder{},
378
379	// Length arrays used to define Huffman codes.
380	.bits = bits,
381	.codebits = codebits,
382
383	// Output history, buffer.
384	.dict = dd,
385
386	// Temporary buffer (avoids repeated allocation).
387	.buf = [_]u8{0} ** 4,
388
389	// Next step in the decompression and decompression state.
390	.step = nextBlock,
391	.step_state = .init,
392	.final = false,
393	.err = null,
394	.to_read = &[0]u8{},
395	.hl = null,
396	.hd = null,
397	.copy_len = 0,
398	.copy_dist = 0,
399	};
400	}
401
402	/// Release all allocated memory.
403	pub fn deinit(self: *Self) void {
404	self.hd2.deinit();
405	self.hd1.deinit();
406	self.dict.deinit();
407	self.allocator.destroy(self.codebits);
408	self.allocator.destroy(self.bits);
409	}
410
411	fn nextBlock(self: *Self) Error!void {
412	while (self.nb < 1 + 2) {
413	self.moreBits() catch \|e\| {
414	self.err = e;
415	return e;
416	};
417	}
418	self.final = self.b & 1 == 1;
419	self.b >>= 1;
420	var typ = self.b & 3;
421	self.b >>= 2;
422	self.nb -= 1 + 2;
423	switch (typ) {
424	0 => try self.dataBlock(),
425	1 => {
426	// compressed, fixed Huffman tables
427	self.hl = &fixed_huffman_decoder.?;
428	self.hd = null;
429	try self.huffmanBlock();
430	},
431	2 => {
432	// compressed, dynamic Huffman tables
433	self.hd2.deinit();
434	self.hd1.deinit();
435	try self.readHuffman();
436	self.hl = &self.hd1;
437	self.hd = &self.hd2;
438	try self.huffmanBlock();
439	},
440	else => {
441	// 3 is reserved.
442	corrupt_input_error_offset = self.roffset;
443	self.err = InflateError.CorruptInput;
444	return InflateError.CorruptInput;
445	},
446	}
447	}
448
449	/// Reads compressed data from the underlying reader and outputs uncompressed data into
450	/// `output`.
451	pub fn read(self: *Self, output: []u8) Error!usize {
452	while (true) {
453	if (self.to_read.len > 0) {
454	var n = mu.copy(output, self.to_read);
455	self.to_read = self.to_read[n..];
456	if (self.to_read.len == 0 and
457	self.err != null)
458	{
459	if (self.err.? == InflateError.EndOfStreamWithNoError) {
460	return n;
461	}
462	return self.err.?;
463	}
464	return n;
465	}
466	if (self.err != null) {
467	if (self.err.? == InflateError.EndOfStreamWithNoError) {
468	return 0;
469	}
470	return self.err.?;
471	}
472	self.step(self) catch \|e\| {
473	self.err = e;
474	if (self.to_read.len == 0) {
475	self.to_read = self.dict.readFlush(); // Flush what's left in case of error
476	}
477	};
478	}
479	}
480
481	pub fn close(self: *Self) ?Error {
482	if (self.err == Error.EndOfStreamWithNoError) {
483	return null;
484	}
485	return self.err;
486	}
487
488	// RFC 1951 section 3.2.7.
489	// Compression with dynamic Huffman codes
490
491	const code_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
492
493	fn readHuffman(self: *Self) Error!void {
494	// HLIT[5], HDIST[5], HCLEN[4].
495	while (self.nb < 5 + 5 + 4) {
496	try self.moreBits();
497	}
498	var nlit = @intCast(u32, self.b & 0x1F) + 257;
499	if (nlit > max_num_lit) {
500	corrupt_input_error_offset = self.roffset;
501	self.err = InflateError.CorruptInput;
502	return InflateError.CorruptInput;
503	}
504	self.b >>= 5;
505	var ndist = @intCast(u32, self.b & 0x1F) + 1;
506	if (ndist > max_num_dist) {
507	corrupt_input_error_offset = self.roffset;
508	self.err = InflateError.CorruptInput;
509	return InflateError.CorruptInput;
510	}
511	self.b >>= 5;
512	var nclen = @intCast(u32, self.b & 0xF) + 4;
513	// num_codes is 19, so nclen is always valid.
514	self.b >>= 4;
515	self.nb -= 5 + 5 + 4;
516
517	// (HCLEN+4)*3 bits: code lengths in the magic code_order order.
518	var i: u32 = 0;
519	while (i < nclen) : (i += 1) {
520	while (self.nb < 3) {
521	try self.moreBits();
522	}
523	self.codebits[code_order[i]] = @intCast(u32, self.b & 0x7);
524	self.b >>= 3;
525	self.nb -= 3;
526	}
527	i = nclen;
528	while (i < code_order.len) : (i += 1) {
529	self.codebits[code_order[i]] = 0;
530	}
531	if (!try self.hd1.init(self.allocator, self.codebits[0..])) {
532	corrupt_input_error_offset = self.roffset;
533	self.err = InflateError.CorruptInput;
534	return InflateError.CorruptInput;
535	}
536
537	// HLIT + 257 code lengths, HDIST + 1 code lengths,
538	// using the code length Huffman code.
539	i = 0;
540	var n = nlit + ndist;
541	while (i < n) {
542	var x = try self.huffSym(&self.hd1);
543	if (x < 16) {
544	// Actual length.
545	self.bits[i] = x;
546	i += 1;
547	continue;
548	}
549	// Repeat previous length or zero.
550	var rep: u32 = 0;
551	var nb: u32 = 0;
552	var b: u32 = 0;
553	switch (x) {
554	16 => {
555	rep = 3;
556	nb = 2;
557	if (i == 0) {
558	corrupt_input_error_offset = self.roffset;
559	self.err = InflateError.CorruptInput;
560	return InflateError.CorruptInput;
561	}
562	b = self.bits[i - 1];
563	},
564	17 => {
565	rep = 3;
566	nb = 3;
567	b = 0;
568	},
569	18 => {
570	rep = 11;
571	nb = 7;
572	b = 0;
573	},
574	else => return error.BadInternalState, // unexpected length code
575	}
576	while (self.nb < nb) {
577	try self.moreBits();
578	}
579	rep += @intCast(u32, self.b & (@as(u32, 1) << @intCast(u5, nb)) - 1);
580	self.b >>= @intCast(u5, nb);
581	self.nb -= nb;
582	if (i + rep > n) {
583	corrupt_input_error_offset = self.roffset;
584	self.err = InflateError.CorruptInput;
585	return InflateError.CorruptInput;
586	}
587	var j: u32 = 0;
588	while (j < rep) : (j += 1) {
589	self.bits[i] = b;
590	i += 1;
591	}
592	}
593
594	if (!try self.hd1.init(self.allocator, self.bits[0..nlit]) or
595	!try self.hd2.init(self.allocator, self.bits[nlit .. nlit + ndist]))
596	{
597	corrupt_input_error_offset = self.roffset;
598	self.err = InflateError.CorruptInput;
599	return InflateError.CorruptInput;
600	}
601
602	// As an optimization, we can initialize the min bits to read at a time
603	// for the HLIT tree to the length of the EOB marker since we know that
604	// every block must terminate with one. This preserves the property that
605	// we never read any extra bytes after the end of the DEFLATE stream.
606	if (self.hd1.min < self.bits[end_block_marker]) {
607	self.hd1.min = self.bits[end_block_marker];
608	}
609
610	return;
611	}
612
613	// Decode a single Huffman block.
614	// hl and hd are the Huffman states for the lit/length values
615	// and the distance values, respectively. If hd == null, using the
616	// fixed distance encoding associated with fixed Huffman blocks.
617	fn huffmanBlock(self: *Self) Error!void {
618	while (true) {
619	switch (self.step_state) {
620	.init => {
621	// Read literal and/or (length, distance) according to RFC section 3.2.3.
622	var v = try self.huffSym(self.hl.?);
623	var n: u32 = 0; // number of bits extra
624	var length: u32 = 0;
625	switch (v) {
626	0...255 => {
627	self.dict.writeByte(@intCast(u8, v));
628	if (self.dict.availWrite() == 0) {
629	self.to_read = self.dict.readFlush();
630	self.step = huffmanBlock;
631	self.step_state = .init;
632	return;
633	}
634	self.step_state = .init;
635	continue;
636	},
637	256 => {
638	self.finishBlock();
639	return;
640	},
641	// otherwise, reference to older data
642	257...264 => {
643	length = v - (257 - 3);
644	n = 0;
645	},
646	265...268 => {
647	length = v * 2 - (265 * 2 - 11);
648	n = 1;
649	},
650	269...272 => {
651	length = v * 4 - (269 * 4 - 19);
652	n = 2;
653	},
654	273...276 => {
655	length = v * 8 - (273 * 8 - 35);
656	n = 3;
657	},
658	277...280 => {
659	length = v * 16 - (277 * 16 - 67);
660	n = 4;
661	},
662	281...284 => {
663	length = v * 32 - (281 * 32 - 131);
664	n = 5;
665	},
666	max_num_lit - 1 => { // 285
667	length = 258;
668	n = 0;
669	},
670	else => {
671	corrupt_input_error_offset = self.roffset;
672	self.err = InflateError.CorruptInput;
673	return InflateError.CorruptInput;
674	},
675	}
676	if (n > 0) {
677	while (self.nb < n) {
678	try self.moreBits();
679	}
680	length += @intCast(u32, self.b) & ((@as(u32, 1) << @intCast(u5, n)) - 1);
681	self.b >>= @intCast(u5, n);
682	self.nb -= n;
683	}
684
685	var dist: u32 = 0;
686	if (self.hd == null) {
687	while (self.nb < 5) {
688	try self.moreBits();
689	}
690	dist = @intCast(
691	u32,
692	bu.bitReverse(u8, @intCast(u8, (self.b & 0x1F) << 3), 8),
693	);
694	self.b >>= 5;
695	self.nb -= 5;
696	} else {
697	dist = try self.huffSym(self.hd.?);
698	}
699
700	switch (dist) {
701	0...3 => dist += 1,
702	4...max_num_dist - 1 => { // 4...29
703	var nb = @intCast(u32, dist - 2) >> 1;
704	// have 1 bit in bottom of dist, need nb more.
705	var extra = (dist & 1) << @intCast(u5, nb);
706	while (self.nb < nb) {
707	try self.moreBits();
708	}
709	extra \|= @intCast(u32, self.b & (@as(u32, 1) << @intCast(u5, nb)) - 1);
710	self.b >>= @intCast(u5, nb);
711	self.nb -= nb;
712	dist = (@as(u32, 1) << @intCast(u5, nb + 1)) + 1 + extra;
713	},
714	else => {
715	corrupt_input_error_offset = self.roffset;
716	self.err = InflateError.CorruptInput;
717	return InflateError.CorruptInput;
718	},
719	}
720
721	// No check on length; encoding can be prescient.
722	if (dist > self.dict.histSize()) {
723	corrupt_input_error_offset = self.roffset;
724	self.err = InflateError.CorruptInput;
725	return InflateError.CorruptInput;
726	}
727
728	self.copy_len = length;
729	self.copy_dist = dist;
730	self.step_state = .dict;
731	},
732
733	.dict => {
734	// Perform a backwards copy according to RFC section 3.2.3.
735	var cnt = self.dict.tryWriteCopy(self.copy_dist, self.copy_len);
736	if (cnt == 0) {
737	cnt = self.dict.writeCopy(self.copy_dist, self.copy_len);
738	}
739	self.copy_len -= cnt;
740
741	if (self.dict.availWrite() == 0 or self.copy_len > 0) {
742	self.to_read = self.dict.readFlush();
743	self.step = huffmanBlock; // We need to continue this work
744	self.step_state = .dict;
745	return;
746	}
747	self.step_state = .init;
748	},
749	}
750	}
751	}
752
753	// Copy a single uncompressed data block from input to output.
754	fn dataBlock(self: *Self) Error!void {
755	// Uncompressed.
756	// Discard current half-byte.
757	self.nb = 0;
758	self.b = 0;
759
760	// Length then ones-complement of length.
761	var nr: u32 = 4;
762	self.inner_reader.readNoEof(self.buf[0..nr]) catch {
763	self.err = InflateError.UnexpectedEndOfStream;
764	return InflateError.UnexpectedEndOfStream;
765	};
766	self.roffset += @intCast(u64, nr);
767	var n = @intCast(u32, self.buf[0]) \| @intCast(u32, self.buf[1]) << 8;
768	var nn = @intCast(u32, self.buf[2]) \| @intCast(u32, self.buf[3]) << 8;
769	if (@intCast(u16, nn) != @truncate(u16, ~n)) {
770	corrupt_input_error_offset = self.roffset;
771	self.err = InflateError.CorruptInput;
772	return InflateError.CorruptInput;
773	}
774
775	if (n == 0) {
776	self.to_read = self.dict.readFlush();
777	self.finishBlock();
778	return;
779	}
780
781	self.copy_len = n;
782	try self.copyData();
783	}
784
785	// copyData copies self.copy_len bytes from the underlying reader into self.hist.
786	// It pauses for reads when self.hist is full.
787	fn copyData(self: *Self) Error!void {
788	var buf = self.dict.writeSlice();
789	if (buf.len > self.copy_len) {
790	buf = buf[0..self.copy_len];
791	}
792
793	var cnt = try self.inner_reader.read(buf);
794	if (cnt < buf.len) {
795	self.err = InflateError.UnexpectedEndOfStream;
796	}
797	self.roffset += @intCast(u64, cnt);
798	self.copy_len -= @intCast(u32, cnt);
799	self.dict.writeMark(@intCast(u32, cnt));
800	if (self.err != null) {
801	return InflateError.UnexpectedEndOfStream;
802	}
803
804	if (self.dict.availWrite() == 0 or self.copy_len > 0) {
805	self.to_read = self.dict.readFlush();
806	self.step = copyData;
807	return;
808	}
809	self.finishBlock();
810	}
811
812	fn finishBlock(self: *Self) void {
813	if (self.final) {
814	if (self.dict.availRead() > 0) {
815	self.to_read = self.dict.readFlush();
816	}
817	self.err = InflateError.EndOfStreamWithNoError;
818	}
819	self.step = nextBlock;
820	}
821
822	fn moreBits(self: *Self) InflateError!void {
823	var c = self.inner_reader.readByte() catch \|e\| {
824	if (e == error.EndOfStream) {
825	return InflateError.UnexpectedEndOfStream;
826	}
827	return InflateError.BadReaderState;
828	};
829	self.roffset += 1;
830	self.b \|= @as(u32, c) << @intCast(u5, self.nb);
831	self.nb += 8;
832	return;
833	}
834
835	// Read the next Huffman-encoded symbol according to h.
836	fn huffSym(self: Self, h: HuffmanDecoder) InflateError!u32 {
837	// Since a HuffmanDecoder can be empty or be composed of a degenerate tree
838	// with single element, huffSym must error on these two edge cases. In both
839	// cases, the chunks slice will be 0 for the invalid sequence, leading it
840	// satisfy the n == 0 check below.
841	var n: u32 = h.min;
842	// Optimization. Go compiler isn't smart enough to keep self.b, self.nb in registers,
843	// but is smart enough to keep local variables in registers, so use nb and b,
844	// inline call to moreBits and reassign b, nb back to self on return.
845	var nb = self.nb;
846	var b = self.b;
847	while (true) {
848	while (nb < n) {
849	var c = self.inner_reader.readByte() catch \|e\| {
850	self.b = b;
851	self.nb = nb;
852	if (e == error.EndOfStream) {
853	return error.UnexpectedEndOfStream;
854	}
855	return InflateError.BadReaderState;
856	};
857	self.roffset += 1;
858	b \|= @intCast(u32, c) << @intCast(u5, nb & 31);
859	nb += 8;
860	}
861	var chunk = h.chunks[b & (huffman_num_chunks - 1)];
862	n = @intCast(u32, chunk & huffman_count_mask);
863	if (n > huffman_chunk_bits) {
864	chunk = h.links[chunk >> huffman_value_shift][(b >> huffman_chunk_bits) & h.link_mask];
865	n = @intCast(u32, chunk & huffman_count_mask);
866	}
867	if (n <= nb) {
868	if (n == 0) {
869	self.b = b;
870	self.nb = nb;
871	corrupt_input_error_offset = self.roffset;
872	self.err = InflateError.CorruptInput;
873	return InflateError.CorruptInput;
874	}
875	self.b = b >> @intCast(u5, n & 31);
876	self.nb = nb - n;
877	return @intCast(u32, chunk >> huffman_value_shift);
878	}
879	}
880	}
881
882	/// Replaces the inner reader and dictionary with new_reader and new_dict.
883	/// new_reader must be of the same type as the reader being replaced.
884	pub fn reset(s: *Self, new_reader: ReaderType, new_dict: ?[]const u8) !void {
885	s.inner_reader = new_reader;
886	s.step = nextBlock;
887	s.err = null;
888
889	s.dict.deinit();
890	try s.dict.init(s.allocator, max_match_offset, new_dict);
891
892	return;
893	}
894	};
895	}
896
897	// tests
898	const expect = std.testing.expect;
899	const expectError = std.testing.expectError;
900	const io = std.io;
901	const testing = std.testing;
902
903	test "truncated input" {
904	const TruncatedTest = struct {
905	input: []const u8,
906	output: []const u8,
907	};
908
909	const tests = [_]TruncatedTest{
910	.{ .input = "\x00", .output = "" },
911	.{ .input = "\x00\x0c", .output = "" },
912	.{ .input = "\x00\x0c\x00", .output = "" },
913	.{ .input = "\x00\x0c\x00\xf3\xff", .output = "" },
914	.{ .input = "\x00\x0c\x00\xf3\xffhello", .output = "hello" },
915	.{ .input = "\x00\x0c\x00\xf3\xffhello, world", .output = "hello, world" },
916	.{ .input = "\x02", .output = "" },
917	.{ .input = "\xf2H\xcd", .output = "He" },
918	.{ .input = "\xf2H͙0a\u{0084}\t", .output = "Hel\x90\x90\x90\x90\x90" },
919	.{ .input = "\xf2H͙0a\u{0084}\t\x00", .output = "Hel\x90\x90\x90\x90\x90" },
920	};
921
922	for (tests) \|t\| {
923	var r = io.fixedBufferStream(t.input).reader();
924	var z = try decompressor(testing.allocator, r, null);
925	defer z.deinit();
926	var zr = z.reader();
927
928	var output = [1]u8{0} ** 12;
929	try expectError(error.UnexpectedEndOfStream, zr.readAll(&output));
930	try expect(mem.eql(u8, output[0..t.output.len], t.output));
931	}
932	}
933
934	test "Go non-regression test for 9842" {
935	// See https://golang.org/issue/9842
936
937	const Test = struct {
938	err: ?anyerror,
939	input: []const u8,
940	};
941
942	const tests = [_]Test{
943	.{ .err = error.UnexpectedEndOfStream, .input = ("\x95\x90=o\xc20\x10\x86\xf30") },
944	.{ .err = error.CorruptInput, .input = ("\x950\x00\x0000000") },
945
946	// Huffman.construct errors
947
948	// lencode
949	.{ .err = error.CorruptInput, .input = ("\x950000") },
950	.{ .err = error.CorruptInput, .input = ("\x05000") },
951	// hlen
952	.{ .err = error.CorruptInput, .input = ("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00") },
953	// hdist
954	.{ .err = error.CorruptInput, .input = ("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf.") },
955
956	// like the "empty distance alphabet" test but for ndist instead of nlen
957	.{ .err = error.CorruptInput, .input = ("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce") },
958	.{ .err = null, .input = "\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0" },
959	};
960
961	for (tests) \|t\| {
962	const reader = std.io.fixedBufferStream(t.input).reader();
963	var decomp = try decompressor(testing.allocator, reader, null);
964	defer decomp.deinit();
965
966	var output: [10]u8 = undefined;
967	if (t.err != null) {
968	try expectError(t.err.?, decomp.reader().read(&output));
969	} else {
970	_ = try decomp.reader().read(&output);
971	}
972	}
973	}
974
975	test "inflate A Tale of Two Cities (1859) intro" {
976	const compressed = [_]u8{
977	0x74, 0xeb, 0xcd, 0x0d, 0x80, 0x20, 0x0c, 0x47, 0x71, 0xdc, 0x9d, 0xa2, 0x03, 0xb8, 0x88,
978	0x63, 0xf0, 0xf1, 0x47, 0x9a, 0x00, 0x35, 0xb4, 0x86, 0xf5, 0x0d, 0x27, 0x63, 0x82, 0xe7,
979	0xdf, 0x7b, 0x87, 0xd1, 0x70, 0x4a, 0x96, 0x41, 0x1e, 0x6a, 0x24, 0x89, 0x8c, 0x2b, 0x74,
980	0xdf, 0xf8, 0x95, 0x21, 0xfd, 0x8f, 0xdc, 0x89, 0x09, 0x83, 0x35, 0x4a, 0x5d, 0x49, 0x12,
981	0x29, 0xac, 0xb9, 0x41, 0xbf, 0x23, 0x2e, 0x09, 0x79, 0x06, 0x1e, 0x85, 0x91, 0xd6, 0xc6,
982	0x2d, 0x74, 0xc4, 0xfb, 0xa1, 0x7b, 0x0f, 0x52, 0x20, 0x84, 0x61, 0x28, 0x0c, 0x63, 0xdf,
983	0x53, 0xf4, 0x00, 0x1e, 0xc3, 0xa5, 0x97, 0x88, 0xf4, 0xd9, 0x04, 0xa5, 0x2d, 0x49, 0x54,
984	0xbc, 0xfd, 0x90, 0xa5, 0x0c, 0xae, 0xbf, 0x3f, 0x84, 0x77, 0x88, 0x3f, 0xaf, 0xc0, 0x40,
985	0xd6, 0x5b, 0x14, 0x8b, 0x54, 0xf6, 0x0f, 0x9b, 0x49, 0xf7, 0xbf, 0xbf, 0x36, 0x54, 0x5a,
986	0x0d, 0xe6, 0x3e, 0xf0, 0x9e, 0x29, 0xcd, 0xa1, 0x41, 0x05, 0x36, 0x48, 0x74, 0x4a, 0xe9,
987	0x46, 0x66, 0x2a, 0x19, 0x17, 0xf4, 0x71, 0x8e, 0xcb, 0x15, 0x5b, 0x57, 0xe4, 0xf3, 0xc7,
988	0xe7, 0x1e, 0x9d, 0x50, 0x08, 0xc3, 0x50, 0x18, 0xc6, 0x2a, 0x19, 0xa0, 0xdd, 0xc3, 0x35,
989	0x82, 0x3d, 0x6a, 0xb0, 0x34, 0x92, 0x16, 0x8b, 0xdb, 0x1b, 0xeb, 0x7d, 0xbc, 0xf8, 0x16,
990	0xf8, 0xc2, 0xe1, 0xaf, 0x81, 0x7e, 0x58, 0xf4, 0x9f, 0x74, 0xf8, 0xcd, 0x39, 0xd3, 0xaa,
991	0x0f, 0x26, 0x31, 0xcc, 0x8d, 0x9a, 0xd2, 0x04, 0x3e, 0x51, 0xbe, 0x7e, 0xbc, 0xc5, 0x27,
992	0x3d, 0xa5, 0xf3, 0x15, 0x63, 0x94, 0x42, 0x75, 0x53, 0x6b, 0x61, 0xc8, 0x01, 0x13, 0x4d,
993	0x23, 0xba, 0x2a, 0x2d, 0x6c, 0x94, 0x65, 0xc7, 0x4b, 0x86, 0x9b, 0x25, 0x3e, 0xba, 0x01,
994	0x10, 0x84, 0x81, 0x28, 0x80, 0x55, 0x1c, 0xc0, 0xa5, 0xaa, 0x36, 0xa6, 0x09, 0xa8, 0xa1,
995	0x85, 0xf9, 0x7d, 0x45, 0xbf, 0x80, 0xe4, 0xd1, 0xbb, 0xde, 0xb9, 0x5e, 0xf1, 0x23, 0x89,
996	0x4b, 0x00, 0xd5, 0x59, 0x84, 0x85, 0xe3, 0xd4, 0xdc, 0xb2, 0x66, 0xe9, 0xc1, 0x44, 0x0b,
997	0x1e, 0x84, 0xec, 0xe6, 0xa1, 0xc7, 0x42, 0x6a, 0x09, 0x6d, 0x9a, 0x5e, 0x70, 0xa2, 0x36,
998	0x94, 0x29, 0x2c, 0x85, 0x3f, 0x24, 0x39, 0xf3, 0xae, 0xc3, 0xca, 0xca, 0xaf, 0x2f, 0xce,
999	0x8e, 0x58, 0x91, 0x00, 0x25, 0xb5, 0xb3, 0xe9, 0xd4, 0xda, 0xef, 0xfa, 0x48, 0x7b, 0x3b,
1000	0xe2, 0x63, 0x12, 0x00, 0x00, 0x20, 0x04, 0x80, 0x70, 0x36, 0x8c, 0xbd, 0x04, 0x71, 0xff,
1001	0xf6, 0x0f, 0x66, 0x38, 0xcf, 0xa1, 0x39, 0x11, 0x0f,
1002	};
1003
1004	const expected =
1005	\\It was the best of times,
1006	\\it was the worst of times,
1007	\\it was the age of wisdom,
1008	\\it was the age of foolishness,
1009	\\it was the epoch of belief,
1010	\\it was the epoch of incredulity,
1011	\\it was the season of Light,
1012	\\it was the season of Darkness,
1013	\\it was the spring of hope,
1014	\\it was the winter of despair,
1015	\\
1016	\\we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way---in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only.
1017	\\
1018	;
1019
1020	const reader = std.io.fixedBufferStream(&compressed).reader();
1021	var decomp = try decompressor(testing.allocator, reader, null);
1022	defer decomp.deinit();
1023
1024	var got: [700]u8 = undefined;
1025	var got_len = try decomp.reader().read(&got);
1026	try expect(got_len == 616);
1027	try expect(mem.eql(u8, got[0..expected.len], expected));
1028	}
1029
1030	test "lengths overflow" {
1031	// malformed final dynamic block, tries to write 321 code lengths (MAXCODES is 316)
1032	// f dy hlit hdist hclen 16 17 18 0 (18) x138 (18) x138 (18) x39 (16) x6
1033	// 1 10 11101 11101 0000 010 010 010 010 (11) 1111111 (11) 1111111 (11) 0011100 (01) 11
1034	const stream = [_]u8{
1035	0b11101101, 0b00011101, 0b00100100, 0b11101001, 0b11111111, 0b11111111, 0b00111001,
1036	0b00001110,
1037	};
1038	try expectError(error.CorruptInput, decompress(stream[0..]));
1039	}
1040
1041	test "empty distance alphabet" {
1042	// dynamic block with empty distance alphabet is valid if only literals and end of data symbol are used
1043	// f dy hlit hdist hclen 16 17 18 0 8 7 9 6 10 5 11 4 12 3 13 2 14 1 15 (18) x128 (18) x128 (1) ( 0) (256)
1044	// 1 10 00000 00000 1111 000 000 010 010 000 000 000 000 000 000 000 000 000 000 000 000 000 001 000 (11) 1110101 (11) 1110101 (0) (10) (0)
1045	const stream = [_]u8{
1046	0b00000101, 0b11100000, 0b00000001, 0b00001001, 0b00000000, 0b00000000,
1047	0b00000000, 0b00000000, 0b00010000, 0b01011100, 0b10111111, 0b00101110,
1048	};
1049	try decompress(stream[0..]);
1050	}
1051
1052	test "distance past beginning of output stream" {
1053	// f fx ('A') ('B') ('C') <len=4, dist=4> (end)
1054	// 1 01 (01110001) (01110010) (01110011) (0000010) (00011) (0000000)
1055	const stream = [_]u8{ 0b01110011, 0b01110100, 0b01110010, 0b00000110, 0b01100001, 0b00000000 };
1056	try std.testing.expectError(error.CorruptInput, decompress(stream[0..]));
1057	}
1058
1059	test "fuzzing" {
1060	const compressed = [_]u8{
1061	0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05, 0xfc, 0x30, 0x0b, 0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05,
1062	} ++ [_]u8{0xe1} 15 ++ [_]u8{0x30} ++ [_]u8{0xe1} 1481;
1063	try expectError(error.UnexpectedEndOfStream, decompress(&compressed));
1064
1065	// see https://github.com/ziglang/zig/issues/9842
1066	try expectError(error.UnexpectedEndOfStream, decompress("\x95\x90=o\xc20\x10\x86\xf30"));
1067	try expectError(error.CorruptInput, decompress("\x950\x00\x0000000"));
1068
1069	// Huffman errors
1070	// lencode
1071	try expectError(error.CorruptInput, decompress("\x950000"));
1072	try expectError(error.CorruptInput, decompress("\x05000"));
1073	// hlen
1074	try expectError(error.CorruptInput, decompress("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00"));
1075	// hdist
1076	try expectError(error.CorruptInput, decompress("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf."));
1077
1078	// like the "empty distance alphabet" test but for ndist instead of nlen
1079	try expectError(error.CorruptInput, decompress("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce"));
1080	try decompress("\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0");
1081	}
1082
1083	fn decompress(input: []const u8) !void {
1084	const allocator = testing.allocator;
1085	const reader = std.io.fixedBufferStream(input).reader();
1086	var decomp = try decompressor(allocator, reader, null);
1087	defer decomp.deinit();
1088	var output = try decomp.reader().readAllAlloc(allocator, math.maxInt(usize));
1089	defer std.testing.allocator.free(output);
1090	}

lib/std/compress/deflate/deflate_const.zig created+28

...	@@ -0,0 +1,28 @@
1	// Deflate
2
3	// Biggest block size for uncompressed block.
4	pub const max_store_block_size = 65535;
5	// The special code used to mark the end of a block.
6	pub const end_block_marker = 256;
7
8	// LZ77
9
10	// The smallest match length per the RFC section 3.2.5
11	pub const base_match_length = 3;
12	// The smallest match offset.
13	pub const base_match_offset = 1;
14	// The largest match length.
15	pub const max_match_length = 258;
16	// The largest match offset.
17	pub const max_match_offset = 1 << 15;
18
19	// Huffman Codes
20
21	// The largest offset code.
22	pub const offset_code_count = 30;
23	// Max number of frequencies used for a Huffman Code
24	// Possible lengths are codegenCodeCount (19), offset_code_count (30) and max_num_lit (286).
25	// The largest of these is max_num_lit.
26	pub const max_num_frequencies = max_num_lit;
27	// Maximum number of literals.
28	pub const max_num_lit = 286;

lib/std/compress/deflate/deflate_fast.zig created+721

...	@@ -0,0 +1,721 @@
1	// This encoding algorithm, which prioritizes speed over output size, is
2	// based on Snappy's LZ77-style encoder: github.com/golang/snappy
3
4	const std = @import("std");
5	const math = std.math;
6	const mem = std.mem;
7
8	const Allocator = std.mem.Allocator;
9
10	const deflate_const = @import("deflate_const.zig");
11	const deflate = @import("compressor.zig");
12	const token = @import("token.zig");
13
14	const base_match_length = deflate_const.base_match_length;
15	const base_match_offset = deflate_const.base_match_offset;
16	const max_match_length = deflate_const.max_match_length;
17	const max_match_offset = deflate_const.max_match_offset;
18	const max_store_block_size = deflate_const.max_store_block_size;
19
20	const table_bits = 14; // Bits used in the table.
21	const table_mask = table_size - 1; // Mask for table indices. Redundant, but can eliminate bounds checks.
22	const table_shift = 32 - table_bits; // Right-shift to get the table_bits most significant bits of a uint32.
23	const table_size = 1 << table_bits; // Size of the table.
24
25	// Reset the buffer offset when reaching this.
26	// Offsets are stored between blocks as i32 values.
27	// Since the offset we are checking against is at the beginning
28	// of the buffer, we need to subtract the current and input
29	// buffer to not risk overflowing the i32.
30	const buffer_reset = math.maxInt(i32) - max_store_block_size * 2;
31
32	fn load32(b: []u8, i: i32) u32 {
33	var s = b[@intCast(usize, i) .. @intCast(usize, i) + 4];
34	return @intCast(u32, s[0]) \|
35	@intCast(u32, s[1]) << 8 \|
36	@intCast(u32, s[2]) << 16 \|
37	@intCast(u32, s[3]) << 24;
38	}
39
40	fn load64(b: []u8, i: i32) u64 {
41	var s = b[@intCast(usize, i)..@intCast(usize, i + 8)];
42	return @intCast(u64, s[0]) \|
43	@intCast(u64, s[1]) << 8 \|
44	@intCast(u64, s[2]) << 16 \|
45	@intCast(u64, s[3]) << 24 \|
46	@intCast(u64, s[4]) << 32 \|
47	@intCast(u64, s[5]) << 40 \|
48	@intCast(u64, s[6]) << 48 \|
49	@intCast(u64, s[7]) << 56;
50	}
51
52	fn hash(u: u32) u32 {
53	return (u *% 0x1e35a7bd) >> table_shift;
54	}
55
56	// These constants are defined by the Snappy implementation so that its
57	// assembly implementation can fast-path some 16-bytes-at-a-time copies.
58	// They aren't necessary in the pure Go implementation, and may not be
59	// necessary in Zig, but using the same thresholds doesn't really hurt.
60	const input_margin = 16 - 1;
61	const min_non_literal_block_size = 1 + 1 + input_margin;
62
63	const TableEntry = struct {
64	val: u32, // Value at destination
65	offset: i32,
66	};
67
68	pub fn deflateFast() DeflateFast {
69	return DeflateFast{
70	.table = [_]TableEntry{.{ .val = 0, .offset = 0 }} ** table_size,
71	.prev = undefined,
72	.prev_len = 0,
73	.cur = max_store_block_size,
74	.allocator = undefined,
75	};
76	}
77
78	// DeflateFast maintains the table for matches,
79	// and the previous byte block for cross block matching.
80	pub const DeflateFast = struct {
81	table: [table_size]TableEntry,
82	prev: []u8, // Previous block, zero length if unknown.
83	prev_len: u32, // Previous block length
84	cur: i32, // Current match offset.
85	allocator: Allocator,
86
87	const Self = @This();
88
89	pub fn init(self: *Self, allocator: Allocator) !void {
90	self.allocator = allocator;
91	self.prev = try allocator.alloc(u8, max_store_block_size);
92	self.prev_len = 0;
93	}
94
95	pub fn deinit(self: *Self) void {
96	self.allocator.free(self.prev);
97	self.prev_len = 0;
98	}
99
100	// Encodes a block given in `src` and appends tokens to `dst` and returns the result.
101	pub fn encode(self: Self, dst: []token.Token, tokens_count: u16, src: []u8) void {
102
103	// Ensure that self.cur doesn't wrap.
104	if (self.cur >= buffer_reset) {
105	self.shiftOffsets();
106	}
107
108	// This check isn't in the Snappy implementation, but there, the caller
109	// instead of the callee handles this case.
110	if (src.len < min_non_literal_block_size) {
111	self.cur += max_store_block_size;
112	self.prev_len = 0;
113	emitLiteral(dst, tokens_count, src);
114	return;
115	}
116
117	// s_limit is when to stop looking for offset/length copies. The input_margin
118	// lets us use a fast path for emitLiteral in the main loop, while we are
119	// looking for copies.
120	var s_limit = @intCast(i32, src.len - input_margin);
121
122	// next_emit is where in src the next emitLiteral should start from.
123	var next_emit: i32 = 0;
124	var s: i32 = 0;
125	var cv: u32 = load32(src, s);
126	var next_hash: u32 = hash(cv);
127
128	outer: while (true) {
129	// Copied from the C++ snappy implementation:
130	//
131	// Heuristic match skipping: If 32 bytes are scanned with no matches
132	// found, start looking only at every other byte. If 32 more bytes are
133	// scanned (or skipped), look at every third byte, etc.. When a match
134	// is found, immediately go back to looking at every byte. This is a
135	// small loss (~5% performance, ~0.1% density) for compressible data
136	// due to more bookkeeping, but for non-compressible data (such as
137	// JPEG) it's a huge win since the compressor quickly "realizes" the
138	// data is incompressible and doesn't bother looking for matches
139	// everywhere.
140	//
141	// The "skip" variable keeps track of how many bytes there are since
142	// the last match; dividing it by 32 (ie. right-shifting by five) gives
143	// the number of bytes to move ahead for each iteration.
144	var skip: i32 = 32;
145
146	var next_s: i32 = s;
147	var candidate: TableEntry = undefined;
148	while (true) {
149	s = next_s;
150	var bytes_between_hash_lookups = skip >> 5;
151	next_s = s + bytes_between_hash_lookups;
152	skip += bytes_between_hash_lookups;
153	if (next_s > s_limit) {
154	break :outer;
155	}
156	candidate = self.table[next_hash & table_mask];
157	var now = load32(src, next_s);
158	self.table[next_hash & table_mask] = .{ .offset = s + self.cur, .val = cv };
159	next_hash = hash(now);
160
161	var offset = s - (candidate.offset - self.cur);
162	if (offset > max_match_offset or cv != candidate.val) {
163	// Out of range or not matched.
164	cv = now;
165	continue;
166	}
167	break;
168	}
169
170	// A 4-byte match has been found. We'll later see if more than 4 bytes
171	// match. But, prior to the match, src[next_emit..s] are unmatched. Emit
172	// them as literal bytes.
173	emitLiteral(dst, tokens_count, src[@intCast(usize, next_emit)..@intCast(usize, s)]);
174
175	// Call emitCopy, and then see if another emitCopy could be our next
176	// move. Repeat until we find no match for the input immediately after
177	// what was consumed by the last emitCopy call.
178	//
179	// If we exit this loop normally then we need to call emitLiteral next,
180	// though we don't yet know how big the literal will be. We handle that
181	// by proceeding to the next iteration of the main loop. We also can
182	// exit this loop via goto if we get close to exhausting the input.
183	while (true) {
184	// Invariant: we have a 4-byte match at s, and no need to emit any
185	// literal bytes prior to s.
186
187	// Extend the 4-byte match as long as possible.
188	//
189	s += 4;
190	var t = candidate.offset - self.cur + 4;
191	var l = self.matchLen(s, t, src);
192
193	// matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
194	dst[tokens_count.*] = token.matchToken(
195	@intCast(u32, l + 4 - base_match_length),
196	@intCast(u32, s - t - base_match_offset),
197	);
198	tokens_count.* += 1;
199	s += l;
200	next_emit = s;
201	if (s >= s_limit) {
202	break :outer;
203	}
204
205	// We could immediately start working at s now, but to improve
206	// compression we first update the hash table at s-1 and at s. If
207	// another emitCopy is not our next move, also calculate next_hash
208	// at s+1. At least on amd64 architecture, these three hash calculations
209	// are faster as one load64 call (with some shifts) instead of
210	// three load32 calls.
211	var x = load64(src, s - 1);
212	var prev_hash = hash(@truncate(u32, x));
213	self.table[prev_hash & table_mask] = TableEntry{
214	.offset = self.cur + s - 1,
215	.val = @truncate(u32, x),
216	};
217	x >>= 8;
218	var curr_hash = hash(@truncate(u32, x));
219	candidate = self.table[curr_hash & table_mask];
220	self.table[curr_hash & table_mask] = TableEntry{
221	.offset = self.cur + s,
222	.val = @truncate(u32, x),
223	};
224
225	var offset = s - (candidate.offset - self.cur);
226	if (offset > max_match_offset or @truncate(u32, x) != candidate.val) {
227	cv = @truncate(u32, x >> 8);
228	next_hash = hash(cv);
229	s += 1;
230	break;
231	}
232	}
233	}
234
235	if (@intCast(u32, next_emit) < src.len) {
236	emitLiteral(dst, tokens_count, src[@intCast(usize, next_emit)..]);
237	}
238	self.cur += @intCast(i32, src.len);
239	self.prev_len = @intCast(u32, src.len);
240	mem.copy(u8, self.prev[0..self.prev_len], src);
241	return;
242	}
243
244	fn emitLiteral(dst: []token.Token, tokens_count: *u16, lit: []u8) void {
245	for (lit) \|v\| {
246	dst[tokens_count.*] = token.literalToken(@intCast(u32, v));
247	tokens_count.* += 1;
248	}
249	return;
250	}
251
252	// matchLen returns the match length between src[s..] and src[t..].
253	// t can be negative to indicate the match is starting in self.prev.
254	// We assume that src[s-4 .. s] and src[t-4 .. t] already match.
255	fn matchLen(self: *Self, s: i32, t: i32, src: []u8) i32 {
256	var s1 = @intCast(u32, s) + max_match_length - 4;
257	if (s1 > src.len) {
258	s1 = @intCast(u32, src.len);
259	}
260
261	// If we are inside the current block
262	if (t >= 0) {
263	var b = src[@intCast(usize, t)..];
264	var a = src[@intCast(usize, s)..@intCast(usize, s1)];
265	b = b[0..a.len];
266	// Extend the match to be as long as possible.
267	for (a) \|_, i\| {
268	if (a[i] != b[i]) {
269	return @intCast(i32, i);
270	}
271	}
272	return @intCast(i32, a.len);
273	}
274
275	// We found a match in the previous block.
276	var tp = @intCast(i32, self.prev_len) + t;
277	if (tp < 0) {
278	return 0;
279	}
280
281	// Extend the match to be as long as possible.
282	var a = src[@intCast(usize, s)..@intCast(usize, s1)];
283	var b = self.prev[@intCast(usize, tp)..@intCast(usize, self.prev_len)];
284	if (b.len > a.len) {
285	b = b[0..a.len];
286	}
287	a = a[0..b.len];
288	for (b) \|_, i\| {
289	if (a[i] != b[i]) {
290	return @intCast(i32, i);
291	}
292	}
293
294	// If we reached our limit, we matched everything we are
295	// allowed to in the previous block and we return.
296	var n = @intCast(i32, b.len);
297	if (@intCast(u32, s + n) == s1) {
298	return n;
299	}
300
301	// Continue looking for more matches in the current block.
302	a = src[@intCast(usize, s + n)..@intCast(usize, s1)];
303	b = src[0..a.len];
304	for (a) \|_, i\| {
305	if (a[i] != b[i]) {
306	return @intCast(i32, i) + n;
307	}
308	}
309	return @intCast(i32, a.len) + n;
310	}
311
312	// Reset resets the encoding history.
313	// This ensures that no matches are made to the previous block.
314	pub fn reset(self: *Self) void {
315	self.prev_len = 0;
316	// Bump the offset, so all matches will fail distance check.
317	// Nothing should be >= self.cur in the table.
318	self.cur += max_match_offset;
319
320	// Protect against self.cur wraparound.
321	if (self.cur >= buffer_reset) {
322	self.shiftOffsets();
323	}
324	}
325
326	// shiftOffsets will shift down all match offset.
327	// This is only called in rare situations to prevent integer overflow.
328	//
329	// See https://golang.org/issue/18636 and https://golang.org/issues/34121.
330	fn shiftOffsets(self: *Self) void {
331	if (self.prev_len == 0) {
332	// We have no history; just clear the table.
333	for (self.table) \|_, i\| {
334	self.table[i] = TableEntry{ .val = 0, .offset = 0 };
335	}
336	self.cur = max_match_offset + 1;
337	return;
338	}
339
340	// Shift down everything in the table that isn't already too far away.
341	for (self.table) \|_, i\| {
342	var v = self.table[i].offset - self.cur + max_match_offset + 1;
343	if (v < 0) {
344	// We want to reset self.cur to max_match_offset + 1, so we need to shift
345	// all table entries down by (self.cur - (max_match_offset + 1)).
346	// Because we ignore matches > max_match_offset, we can cap
347	// any negative offsets at 0.
348	v = 0;
349	}
350	self.table[i].offset = v;
351	}
352	self.cur = max_match_offset + 1;
353	}
354	};
355
356	test "best speed match 1/3" {
357	const expect = std.testing.expect;
358
359	{
360	var previous = [_]u8{ 0, 0, 0, 1, 2 };
361	var e = DeflateFast{
362	.prev = &previous,
363	.prev_len = previous.len,
364	.table = undefined,
365	.allocator = undefined,
366	.cur = 0,
367	};
368	var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 };
369	var got: i32 = e.matchLen(3, -3, &current);
370	try expect(got == 6);
371	}
372	{
373	var previous = [_]u8{ 0, 0, 0, 1, 2 };
374	var e = DeflateFast{
375	.prev = &previous,
376	.prev_len = previous.len,
377	.table = undefined,
378	.allocator = undefined,
379	.cur = 0,
380	};
381	var current = [_]u8{ 2, 4, 5, 0, 1, 2, 3, 4, 5 };
382	var got: i32 = e.matchLen(3, -3, &current);
383	try expect(got == 3);
384	}
385	{
386	var previous = [_]u8{ 0, 0, 0, 1, 1 };
387	var e = DeflateFast{
388	.prev = &previous,
389	.prev_len = previous.len,
390	.table = undefined,
391	.allocator = undefined,
392	.cur = 0,
393	};
394	var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 };
395	var got: i32 = e.matchLen(3, -3, &current);
396	try expect(got == 2);
397	}
398	{
399	var previous = [_]u8{ 0, 0, 0, 1, 2 };
400	var e = DeflateFast{
401	.prev = &previous,
402	.prev_len = previous.len,
403	.table = undefined,
404	.allocator = undefined,
405	.cur = 0,
406	};
407	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
408	var got: i32 = e.matchLen(0, -1, &current);
409	try expect(got == 4);
410	}
411	{
412	var previous = [_]u8{ 0, 0, 0, 1, 2, 3, 4, 5, 2, 2 };
413	var e = DeflateFast{
414	.prev = &previous,
415	.prev_len = previous.len,
416	.table = undefined,
417	.allocator = undefined,
418	.cur = 0,
419	};
420	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
421	var got: i32 = e.matchLen(4, -7, &current);
422	try expect(got == 5);
423	}
424	{
425	var previous = [_]u8{ 9, 9, 9, 9, 9 };
426	var e = DeflateFast{
427	.prev = &previous,
428	.prev_len = previous.len,
429	.table = undefined,
430	.allocator = undefined,
431	.cur = 0,
432	};
433	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
434	var got: i32 = e.matchLen(0, -1, &current);
435	try expect(got == 0);
436	}
437	{
438	var previous = [_]u8{ 9, 9, 9, 9, 9 };
439	var e = DeflateFast{
440	.prev = &previous,
441	.prev_len = previous.len,
442	.table = undefined,
443	.allocator = undefined,
444	.cur = 0,
445	};
446	var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
447	var got: i32 = e.matchLen(1, 0, &current);
448	try expect(got == 0);
449	}
450	}
451
452	test "best speed match 2/3" {
453	const expect = std.testing.expect;
454
455	{
456	var previous = [_]u8{};
457	var e = DeflateFast{
458	.prev = &previous,
459	.prev_len = previous.len,
460	.table = undefined,
461	.allocator = undefined,
462	.cur = 0,
463	};
464	var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
465	var got: i32 = e.matchLen(1, -5, &current);
466	try expect(got == 0);
467	}
468	{
469	var previous = [_]u8{};
470	var e = DeflateFast{
471	.prev = &previous,
472	.prev_len = previous.len,
473	.table = undefined,
474	.allocator = undefined,
475	.cur = 0,
476	};
477	var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
478	var got: i32 = e.matchLen(1, -1, &current);
479	try expect(got == 0);
480	}
481	{
482	var previous = [_]u8{};
483	var e = DeflateFast{
484	.prev = &previous,
485	.prev_len = previous.len,
486	.table = undefined,
487	.allocator = undefined,
488	.cur = 0,
489	};
490	var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
491	var got: i32 = e.matchLen(1, 0, &current);
492	try expect(got == 3);
493	}
494	{
495	var previous = [_]u8{ 3, 4, 5 };
496	var e = DeflateFast{
497	.prev = &previous,
498	.prev_len = previous.len,
499	.table = undefined,
500	.allocator = undefined,
501	.cur = 0,
502	};
503	var current = [_]u8{ 3, 4, 5 };
504	var got: i32 = e.matchLen(0, -3, &current);
505	try expect(got == 3);
506	}
507	}
508
509	test "best speed match 2/2" {
510	const testing = std.testing;
511	const expect = testing.expect;
512
513	const Case = struct {
514	previous: u32,
515	current: u32,
516	s: i32,
517	t: i32,
518	expected: i32,
519	};
520
521	const cases = [_]Case{
522	.{
523	.previous = 1000,
524	.current = 1000,
525	.s = 0,
526	.t = -1000,
527	.expected = max_match_length - 4,
528	},
529	.{
530	.previous = 200,
531	.s = 0,
532	.t = -200,
533	.current = 500,
534	.expected = max_match_length - 4,
535	},
536	.{
537	.previous = 200,
538	.s = 1,
539	.t = 0,
540	.current = 500,
541	.expected = max_match_length - 4,
542	},
543	.{
544	.previous = max_match_length - 4,
545	.s = 0,
546	.t = -(max_match_length - 4),
547	.current = 500,
548	.expected = max_match_length - 4,
549	},
550	.{
551	.previous = 200,
552	.s = 400,
553	.t = -200,
554	.current = 500,
555	.expected = 100,
556	},
557	.{
558	.previous = 10,
559	.s = 400,
560	.t = 200,
561	.current = 500,
562	.expected = 100,
563	},
564	};
565
566	for (cases) \|c\| {
567	var previous = try testing.allocator.alloc(u8, c.previous);
568	defer testing.allocator.free(previous);
569	mem.set(u8, previous, 0);
570
571	var current = try testing.allocator.alloc(u8, c.current);
572	defer testing.allocator.free(current);
573	mem.set(u8, current, 0);
574
575	var e = DeflateFast{
576	.prev = previous,
577	.prev_len = @intCast(u32, previous.len),
578	.table = undefined,
579	.allocator = undefined,
580	.cur = 0,
581	};
582	var got: i32 = e.matchLen(c.s, c.t, current);
583	try expect(got == c.expected);
584	}
585	}
586
587	test "best speed shift offsets" {
588	const testing = std.testing;
589	const expect = std.testing.expect;
590
591	// Test if shiftoffsets properly preserves matches and resets out-of-range matches
592	// seen in https://github.com/golang/go/issues/4142
593	var enc = deflateFast();
594	try enc.init(testing.allocator);
595	defer enc.deinit();
596
597	// test_data may not generate internal matches.
598	var test_data = [32]u8{
599	0xf5, 0x25, 0xf2, 0x55, 0xf6, 0xc1, 0x1f, 0x0b, 0x10, 0xa1,
600	0xd0, 0x77, 0x56, 0x38, 0xf1, 0x9c, 0x7f, 0x85, 0xc5, 0xbd,
601	0x16, 0x28, 0xd4, 0xf9, 0x03, 0xd4, 0xc0, 0xa1, 0x1e, 0x58,
602	0x5b, 0xc9,
603	};
604
605	var tokens = [_]token.Token{0} ** 32;
606	var tokens_count: u16 = 0;
607
608	// Encode the testdata with clean state.
609	// Second part should pick up matches from the first block.
610	tokens_count = 0;
611	enc.encode(&tokens, &tokens_count, &test_data);
612	var want_first_tokens = tokens_count;
613	tokens_count = 0;
614	enc.encode(&tokens, &tokens_count, &test_data);
615	var want_second_tokens = tokens_count;
616
617	try expect(want_first_tokens > want_second_tokens);
618
619	// Forward the current indicator to before wraparound.
620	enc.cur = buffer_reset - @intCast(i32, test_data.len);
621
622	// Part 1 before wrap, should match clean state.
623	tokens_count = 0;
624	enc.encode(&tokens, &tokens_count, &test_data);
625	var got = tokens_count;
626	try expect(want_first_tokens == got);
627
628	// Verify we are about to wrap.
629	try expect(enc.cur == buffer_reset);
630
631	// Part 2 should match clean state as well even if wrapped.
632	tokens_count = 0;
633	enc.encode(&tokens, &tokens_count, &test_data);
634	got = tokens_count;
635	try expect(want_second_tokens == got);
636
637	// Verify that we wrapped.
638	try expect(enc.cur < buffer_reset);
639
640	// Forward the current buffer, leaving the matches at the bottom.
641	enc.cur = buffer_reset;
642	enc.shiftOffsets();
643
644	// Ensure that no matches were picked up.
645	tokens_count = 0;
646	enc.encode(&tokens, &tokens_count, &test_data);
647	got = tokens_count;
648	try expect(want_first_tokens == got);
649	}
650
651	test "best speed reset" {
652	// test that encoding is consistent across a warparound of the table offset.
653	// See https://github.com/golang/go/issues/34121
654	const expect = std.testing.expect;
655	const fmt = std.fmt;
656	const testing = std.testing;
657
658	const ArrayList = std.ArrayList;
659
660	const input_size = 65536;
661	var input = try testing.allocator.alloc(u8, input_size);
662	defer testing.allocator.free(input);
663
664	var i: usize = 0;
665	while (i < input_size) : (i += 1) {
666	_ = try fmt.bufPrint(input, "asdfasdfasdfasdf{d}{d}fghfgujyut{d}yutyu\n", .{ i, i, i });
667	}
668	// This is specific to level 1 (best_speed).
669	const level = .best_speed;
670	const offset: usize = 1;
671
672	// We do an encode with a clean buffer to compare.
673	var want = ArrayList(u8).init(testing.allocator);
674	defer want.deinit();
675	var clean_comp = try deflate.compressor(
676	testing.allocator,
677	want.writer(),
678	.{ .level = level },
679	);
680	defer clean_comp.deinit();
681
682	// Write 3 times, close.
683	try clean_comp.writer().writeAll(input);
684	try clean_comp.writer().writeAll(input);
685	try clean_comp.writer().writeAll(input);
686	try clean_comp.close();
687
688	var o = offset;
689	while (o <= 256) : (o *= 2) {
690	var discard = ArrayList(u8).init(testing.allocator);
691	defer discard.deinit();
692
693	var comp = try deflate.compressor(
694	testing.allocator,
695	discard.writer(),
696	.{ .level = level },
697	);
698	defer comp.deinit();
699
700	// Reset until we are right before the wraparound.
701	// Each reset adds max_match_offset to the offset.
702	i = 0;
703	var limit = (buffer_reset - input.len - o - max_match_offset) / max_match_offset;
704	while (i < limit) : (i += 1) {
705	// skip ahead to where we are close to wrap around...
706	comp.reset(discard.writer());
707	}
708	var got = ArrayList(u8).init(testing.allocator);
709	defer got.deinit();
710	comp.reset(got.writer());
711
712	// Write 3 times, close.
713	try comp.writer().writeAll(input);
714	try comp.writer().writeAll(input);
715	try comp.writer().writeAll(input);
716	try comp.close();
717
718	// output must match at wraparound
719	try expect(mem.eql(u8, got.items, want.items));
720	}
721	}

lib/std/compress/deflate/deflate_fast_test.zig created+166

...	@@ -0,0 +1,166 @@
1	const std = @import("std");
2	const expect = std.testing.expect;
3	const io = std.io;
4	const mem = std.mem;
5	const testing = std.testing;
6
7	const ArrayList = std.ArrayList;
8
9	const deflate = @import("compressor.zig");
10	const inflate = @import("decompressor.zig");
11	const deflate_const = @import("deflate_const.zig");
12
13	test "best speed" {
14	// Tests that round-tripping through deflate and then inflate recovers the original input.
15	// The Write sizes are near the thresholds in the compressor.encSpeed method (0, 16, 128), as well
16	// as near `deflate_const.max_store_block_size` (65535).
17
18	var abcabc = try testing.allocator.alloc(u8, 131_072);
19	defer testing.allocator.free(abcabc);
20
21	for (abcabc) \|_, i\| {
22	abcabc[i] = @intCast(u8, i % 128);
23	}
24
25	var tc_01 = [_]u32{ 65536, 0 };
26	var tc_02 = [_]u32{ 65536, 1 };
27	var tc_03 = [_]u32{ 65536, 1, 256 };
28	var tc_04 = [_]u32{ 65536, 1, 65536 };
29	var tc_05 = [_]u32{ 65536, 14 };
30	var tc_06 = [_]u32{ 65536, 15 };
31	var tc_07 = [_]u32{ 65536, 16 };
32	var tc_08 = [_]u32{ 65536, 16, 256 };
33	var tc_09 = [_]u32{ 65536, 16, 65536 };
34	var tc_10 = [_]u32{ 65536, 127 };
35	var tc_11 = [_]u32{ 65536, 127 };
36	var tc_12 = [_]u32{ 65536, 128 };
37	var tc_13 = [_]u32{ 65536, 128, 256 };
38	var tc_14 = [_]u32{ 65536, 128, 65536 };
39	var tc_15 = [_]u32{ 65536, 129 };
40	var tc_16 = [_]u32{ 65536, 65536, 256 };
41	var tc_17 = [_]u32{ 65536, 65536, 65536 };
42	var test_cases = [_][]u32{
43	&tc_01, &tc_02, &tc_03, &tc_04, &tc_05, &tc_06, &tc_07, &tc_08, &tc_09, &tc_10,
44	&tc_11, &tc_12, &tc_13, &tc_14, &tc_15, &tc_16, &tc_17,
45	};
46
47	for (test_cases) \|tc\| {
48	var firsts = [_]u32{ 1, 65534, 65535, 65536, 65537, 131072 };
49
50	for (firsts) \|first_n\| {
51	tc[0] = first_n;
52
53	var to_flush = [_]bool{ false, true };
54	for (to_flush) \|flush\| {
55	var compressed = ArrayList(u8).init(testing.allocator);
56	defer compressed.deinit();
57
58	var want = ArrayList(u8).init(testing.allocator);
59	defer want.deinit();
60
61	var comp = try deflate.compressor(
62	testing.allocator,
63	compressed.writer(),
64	.{ .level = .best_speed },
65	);
66	defer comp.deinit();
67
68	for (tc) \|n\| {
69	try want.appendSlice(abcabc[0..n]);
70	try comp.writer().writeAll(abcabc[0..n]);
71	if (flush) {
72	try comp.flush();
73	}
74	}
75
76	try comp.close();
77
78	var decompressed = try testing.allocator.alloc(u8, want.items.len);
79	defer testing.allocator.free(decompressed);
80
81	var decomp = try inflate.decompressor(
82	testing.allocator,
83	io.fixedBufferStream(compressed.items).reader(),
84	null,
85	);
86	defer decomp.deinit();
87
88	var read = try decomp.reader().readAll(decompressed);
89	_ = decomp.close();
90
91	try expect(read == want.items.len);
92	try expect(mem.eql(u8, want.items, decompressed));
93	}
94	}
95	}
96	}
97
98	test "best speed max match offset" {
99	const abc = "abcdefgh";
100	const xyz = "stuvwxyz";
101	const input_margin = 16 - 1;
102
103	const match_before = [_]bool{ false, true };
104	for (match_before) \|do_match_before\| {
105	const extras = [_]u32{
106	0,
107	input_margin - 1,
108	input_margin,
109	input_margin + 1,
110	2 * input_margin,
111	};
112	for (extras) \|extra\| {
113	var offset_adj: i32 = -5;
114	while (offset_adj <= 5) : (offset_adj += 1) {
115	var offset = deflate_const.max_match_offset + offset_adj;
116
117	// Make src to be a []u8 of the form
118	// fmt("{s}{s}{s}{s}{s}", .{abc, zeros0, xyzMaybe, abc, zeros1})
119	// where:
120	// zeros0 is approximately max_match_offset zeros.
121	// xyzMaybe is either xyz or the empty string.
122	// zeros1 is between 0 and 30 zeros.
123	// The difference between the two abc's will be offset, which
124	// is max_match_offset plus or minus a small adjustment.
125	var src_len: usize = @intCast(usize, offset + abc.len + @intCast(i32, extra));
126	var src = try testing.allocator.alloc(u8, src_len);
127	defer testing.allocator.free(src);
128
129	mem.copy(u8, src, abc);
130	if (!do_match_before) {
131	var src_offset: usize = @intCast(usize, offset - xyz.len);
132	mem.copy(u8, src[src_offset..], xyz);
133	}
134	var src_offset: usize = @intCast(usize, offset);
135	mem.copy(u8, src[src_offset..], abc);
136
137	var compressed = ArrayList(u8).init(testing.allocator);
138	defer compressed.deinit();
139
140	var comp = try deflate.compressor(
141	testing.allocator,
142	compressed.writer(),
143	.{ .level = .best_speed },
144	);
145	defer comp.deinit();
146	try comp.writer().writeAll(src);
147	_ = try comp.close();
148
149	var decompressed = try testing.allocator.alloc(u8, src.len);
150	defer testing.allocator.free(decompressed);
151
152	var decomp = try inflate.decompressor(
153	testing.allocator,
154	io.fixedBufferStream(compressed.items).reader(),
155	null,
156	);
157	defer decomp.deinit();
158	var read = try decomp.reader().readAll(decompressed);
159	_ = decomp.close();
160
161	try expect(read == src.len);
162	try expect(mem.eql(u8, decompressed, src));
163	}
164	}
165	}
166	}

lib/std/compress/deflate/dict_decoder.zig created+420

...	@@ -0,0 +1,420 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const mem = std.mem;
4
5	const Allocator = std.mem.Allocator;
6
7	// Implements the LZ77 sliding dictionary as used in decompression.
8	// LZ77 decompresses data through sequences of two forms of commands:
9	//
10	// * Literal insertions: Runs of one or more symbols are inserted into the data
11	// stream as is. This is accomplished through the writeByte method for a
12	// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
13	// Any valid stream must start with a literal insertion if no preset dictionary
14	// is used.
15	//
16	// * Backward copies: Runs of one or more symbols are copied from previously
17	// emitted data. Backward copies come as the tuple (dist, length) where dist
18	// determines how far back in the stream to copy from and length determines how
19	// many bytes to copy. Note that it is valid for the length to be greater than
20	// the distance. Since LZ77 uses forward copies, that situation is used to
21	// perform a form of run-length encoding on repeated runs of symbols.
22	// The writeCopy and tryWriteCopy are used to implement this command.
23	//
24	// For performance reasons, this implementation performs little to no sanity
25	// checks about the arguments. As such, the invariants documented for each
26	// method call must be respected.
27	pub const DictDecoder = struct {
28	const Self = @This();
29
30	allocator: Allocator = undefined,
31
32	hist: []u8 = undefined, // Sliding window history
33
34	// Invariant: 0 <= rd_pos <= wr_pos <= hist.len
35	wr_pos: u32 = 0, // Current output position in buffer
36	rd_pos: u32 = 0, // Have emitted hist[0..rd_pos] already
37	full: bool = false, // Has a full window length been written yet?
38
39	// init initializes DictDecoder to have a sliding window dictionary of the given
40	// size. If a preset dict is provided, it will initialize the dictionary with
41	// the contents of dict.
42	pub fn init(self: *Self, allocator: Allocator, size: u32, dict: ?[]const u8) !void {
43	self.allocator = allocator;
44
45	self.hist = try allocator.alloc(u8, size);
46
47	self.wr_pos = 0;
48
49	if (dict != null) {
50	mem.copy(u8, self.hist, dict.?[dict.?.len -\| self.hist.len..]);
51	self.wr_pos = @intCast(u32, dict.?.len);
52	}
53
54	if (self.wr_pos == self.hist.len) {
55	self.wr_pos = 0;
56	self.full = true;
57	}
58	self.rd_pos = self.wr_pos;
59	}
60
61	pub fn deinit(self: *Self) void {
62	self.allocator.free(self.hist);
63	}
64
65	// Reports the total amount of historical data in the dictionary.
66	pub fn histSize(self: *Self) u32 {
67	if (self.full) {
68	return @intCast(u32, self.hist.len);
69	}
70	return self.wr_pos;
71	}
72
73	// Reports the number of bytes that can be flushed by readFlush.
74	pub fn availRead(self: *Self) u32 {
75	return self.wr_pos - self.rd_pos;
76	}
77
78	// Reports the available amount of output buffer space.
79	pub fn availWrite(self: *Self) u32 {
80	return @intCast(u32, self.hist.len - self.wr_pos);
81	}
82
83	// Returns a slice of the available buffer to write data to.
84	//
85	// This invariant will be kept: s.len <= availWrite()
86	pub fn writeSlice(self: *Self) []u8 {
87	return self.hist[self.wr_pos..];
88	}
89
90	// Advances the writer pointer by `count`.
91	//
92	// This invariant must be kept: 0 <= count <= availWrite()
93	pub fn writeMark(self: *Self, count: u32) void {
94	assert(0 <= count and count <= self.availWrite());
95	self.wr_pos += count;
96	}
97
98	// Writes a single byte to the dictionary.
99	//
100	// This invariant must be kept: 0 < availWrite()
101	pub fn writeByte(self: *Self, byte: u8) void {
102	self.hist[self.wr_pos] = byte;
103	self.wr_pos += 1;
104	}
105
106	fn copy(dst: []u8, src: []const u8) u32 {
107	if (src.len > dst.len) {
108	mem.copy(u8, dst, src[0..dst.len]);
109	return @intCast(u32, dst.len);
110	}
111	mem.copy(u8, dst, src);
112	return @intCast(u32, src.len);
113	}
114
115	// Copies a string at a given (dist, length) to the output.
116	// This returns the number of bytes copied and may be less than the requested
117	// length if the available space in the output buffer is too small.
118	//
119	// This invariant must be kept: 0 < dist <= histSize()
120	pub fn writeCopy(self: *Self, dist: u32, length: u32) u32 {
121	assert(0 < dist and dist <= self.histSize());
122	var dst_base = self.wr_pos;
123	var dst_pos = dst_base;
124	var src_pos: i32 = @intCast(i32, dst_pos) - @intCast(i32, dist);
125	var end_pos = dst_pos + length;
126	if (end_pos > self.hist.len) {
127	end_pos = @intCast(u32, self.hist.len);
128	}
129
130	// Copy non-overlapping section after destination position.
131	//
132	// This section is non-overlapping in that the copy length for this section
133	// is always less than or equal to the backwards distance. This can occur
134	// if a distance refers to data that wraps-around in the buffer.
135	// Thus, a backwards copy is performed here; that is, the exact bytes in
136	// the source prior to the copy is placed in the destination.
137	if (src_pos < 0) {
138	src_pos += @intCast(i32, self.hist.len);
139	dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@intCast(usize, src_pos)..]);
140	src_pos = 0;
141	}
142
143	// Copy possibly overlapping section before destination position.
144	//
145	// This section can overlap if the copy length for this section is larger
146	// than the backwards distance. This is allowed by LZ77 so that repeated
147	// strings can be succinctly represented using (dist, length) pairs.
148	// Thus, a forwards copy is performed here; that is, the bytes copied is
149	// possibly dependent on the resulting bytes in the destination as the copy
150	// progresses along. This is functionally equivalent to the following:
151	//
152	// var i = 0;
153	// while(i < end_pos - dst_pos) : (i+=1) {
154	// self.hist[dst_pos+i] = self.hist[src_pos+i];
155	// }
156	// dst_pos = end_pos;
157	//
158	while (dst_pos < end_pos) {
159	dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@intCast(usize, src_pos)..dst_pos]);
160	}
161
162	self.wr_pos = dst_pos;
163	return dst_pos - dst_base;
164	}
165
166	// Tries to copy a string at a given (distance, length) to the
167	// output. This specialized version is optimized for short distances.
168	//
169	// This method is designed to be inlined for performance reasons.
170	//
171	// This invariant must be kept: 0 < dist <= histSize()
172	pub fn tryWriteCopy(self: *Self, dist: u32, length: u32) u32 {
173	var dst_pos = self.wr_pos;
174	var end_pos = dst_pos + length;
175	if (dst_pos < dist or end_pos > self.hist.len) {
176	return 0;
177	}
178	var dst_base = dst_pos;
179	var src_pos = dst_pos - dist;
180
181	// Copy possibly overlapping section before destination position.
182	while (dst_pos < end_pos) {
183	dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[src_pos..dst_pos]);
184	}
185
186	self.wr_pos = dst_pos;
187	return dst_pos - dst_base;
188	}
189
190	// Returns a slice of the historical buffer that is ready to be
191	// emitted to the user. The data returned by readFlush must be fully consumed
192	// before calling any other DictDecoder methods.
193	pub fn readFlush(self: *Self) []u8 {
194	var to_read = self.hist[self.rd_pos..self.wr_pos];
195	self.rd_pos = self.wr_pos;
196	if (self.wr_pos == self.hist.len) {
197	self.wr_pos = 0;
198	self.rd_pos = 0;
199	self.full = true;
200	}
201	return to_read;
202	}
203	};
204
205	// tests
206
207	test "dictionary decoder" {
208	const ArrayList = std.ArrayList;
209	const expect = std.testing.expect;
210	const testing = std.testing;
211
212	const abc = "ABC\n";
213	const fox = "The quick brown fox jumped over the lazy dog!\n";
214	const poem: []const u8 =
215	\\The Road Not Taken
216	\\Robert Frost
217	\\
218	\\Two roads diverged in a yellow wood,
219	\\And sorry I could not travel both
220	\\And be one traveler, long I stood
221	\\And looked down one as far as I could
222	\\To where it bent in the undergrowth;
223	\\
224	\\Then took the other, as just as fair,
225	\\And having perhaps the better claim,
226	\\Because it was grassy and wanted wear;
227	\\Though as for that the passing there
228	\\Had worn them really about the same,
229	\\
230	\\And both that morning equally lay
231	\\In leaves no step had trodden black.
232	\\Oh, I kept the first for another day!
233	\\Yet knowing how way leads on to way,
234	\\I doubted if I should ever come back.
235	\\
236	\\I shall be telling this with a sigh
237	\\Somewhere ages and ages hence:
238	\\Two roads diverged in a wood, and I-
239	\\I took the one less traveled by,
240	\\And that has made all the difference.
241	\\
242	;
243
244	const uppercase: []const u8 =
245	\\THE ROAD NOT TAKEN
246	\\ROBERT FROST
247	\\
248	\\TWO ROADS DIVERGED IN A YELLOW WOOD,
249	\\AND SORRY I COULD NOT TRAVEL BOTH
250	\\AND BE ONE TRAVELER, LONG I STOOD
251	\\AND LOOKED DOWN ONE AS FAR AS I COULD
252	\\TO WHERE IT BENT IN THE UNDERGROWTH;
253	\\
254	\\THEN TOOK THE OTHER, AS JUST AS FAIR,
255	\\AND HAVING PERHAPS THE BETTER CLAIM,
256	\\BECAUSE IT WAS GRASSY AND WANTED WEAR;
257	\\THOUGH AS FOR THAT THE PASSING THERE
258	\\HAD WORN THEM REALLY ABOUT THE SAME,
259	\\
260	\\AND BOTH THAT MORNING EQUALLY LAY
261	\\IN LEAVES NO STEP HAD TRODDEN BLACK.
262	\\OH, I KEPT THE FIRST FOR ANOTHER DAY!
263	\\YET KNOWING HOW WAY LEADS ON TO WAY,
264	\\I DOUBTED IF I SHOULD EVER COME BACK.
265	\\
266	\\I SHALL BE TELLING THIS WITH A SIGH
267	\\SOMEWHERE AGES AND AGES HENCE:
268	\\TWO ROADS DIVERGED IN A WOOD, AND I-
269	\\I TOOK THE ONE LESS TRAVELED BY,
270	\\AND THAT HAS MADE ALL THE DIFFERENCE.
271	\\
272	;
273
274	const PoemRefs = struct {
275	dist: u32, // Backward distance (0 if this is an insertion)
276	length: u32, // Length of copy or insertion
277	};
278
279	var poem_refs = [_]PoemRefs{
280	.{ .dist = 0, .length = 38 }, .{ .dist = 33, .length = 3 }, .{ .dist = 0, .length = 48 },
281	.{ .dist = 79, .length = 3 }, .{ .dist = 0, .length = 11 }, .{ .dist = 34, .length = 5 },
282	.{ .dist = 0, .length = 6 }, .{ .dist = 23, .length = 7 }, .{ .dist = 0, .length = 8 },
283	.{ .dist = 50, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 69, .length = 3 },
284	.{ .dist = 34, .length = 5 }, .{ .dist = 0, .length = 4 }, .{ .dist = 97, .length = 3 },
285	.{ .dist = 0, .length = 4 }, .{ .dist = 43, .length = 5 }, .{ .dist = 0, .length = 6 },
286	.{ .dist = 7, .length = 4 }, .{ .dist = 88, .length = 7 }, .{ .dist = 0, .length = 12 },
287	.{ .dist = 80, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 141, .length = 4 },
288	.{ .dist = 0, .length = 1 }, .{ .dist = 196, .length = 3 }, .{ .dist = 0, .length = 3 },
289	.{ .dist = 157, .length = 3 }, .{ .dist = 0, .length = 6 }, .{ .dist = 181, .length = 3 },
290	.{ .dist = 0, .length = 2 }, .{ .dist = 23, .length = 3 }, .{ .dist = 77, .length = 3 },
291	.{ .dist = 28, .length = 5 }, .{ .dist = 128, .length = 3 }, .{ .dist = 110, .length = 4 },
292	.{ .dist = 70, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 85, .length = 6 },
293	.{ .dist = 0, .length = 2 }, .{ .dist = 182, .length = 6 }, .{ .dist = 0, .length = 4 },
294	.{ .dist = 133, .length = 3 }, .{ .dist = 0, .length = 7 }, .{ .dist = 47, .length = 5 },
295	.{ .dist = 0, .length = 20 }, .{ .dist = 112, .length = 5 }, .{ .dist = 0, .length = 1 },
296	.{ .dist = 58, .length = 3 }, .{ .dist = 0, .length = 8 }, .{ .dist = 59, .length = 3 },
297	.{ .dist = 0, .length = 4 }, .{ .dist = 173, .length = 3 }, .{ .dist = 0, .length = 5 },
298	.{ .dist = 114, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 92, .length = 5 },
299	.{ .dist = 0, .length = 2 }, .{ .dist = 71, .length = 3 }, .{ .dist = 0, .length = 2 },
300	.{ .dist = 76, .length = 5 }, .{ .dist = 0, .length = 1 }, .{ .dist = 46, .length = 3 },
301	.{ .dist = 96, .length = 4 }, .{ .dist = 130, .length = 4 }, .{ .dist = 0, .length = 3 },
302	.{ .dist = 360, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 178, .length = 5 },
303	.{ .dist = 0, .length = 7 }, .{ .dist = 75, .length = 3 }, .{ .dist = 0, .length = 3 },
304	.{ .dist = 45, .length = 6 }, .{ .dist = 0, .length = 6 }, .{ .dist = 299, .length = 6 },
305	.{ .dist = 180, .length = 3 }, .{ .dist = 70, .length = 6 }, .{ .dist = 0, .length = 1 },
306	.{ .dist = 48, .length = 3 }, .{ .dist = 66, .length = 4 }, .{ .dist = 0, .length = 3 },
307	.{ .dist = 47, .length = 5 }, .{ .dist = 0, .length = 9 }, .{ .dist = 325, .length = 3 },
308	.{ .dist = 0, .length = 1 }, .{ .dist = 359, .length = 3 }, .{ .dist = 318, .length = 3 },
309	.{ .dist = 0, .length = 2 }, .{ .dist = 199, .length = 3 }, .{ .dist = 0, .length = 1 },
310	.{ .dist = 344, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 248, .length = 3 },
311	.{ .dist = 0, .length = 10 }, .{ .dist = 310, .length = 3 }, .{ .dist = 0, .length = 3 },
312	.{ .dist = 93, .length = 6 }, .{ .dist = 0, .length = 3 }, .{ .dist = 252, .length = 3 },
313	.{ .dist = 157, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 273, .length = 5 },
314	.{ .dist = 0, .length = 14 }, .{ .dist = 99, .length = 4 }, .{ .dist = 0, .length = 1 },
315	.{ .dist = 464, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 92, .length = 4 },
316	.{ .dist = 495, .length = 3 }, .{ .dist = 0, .length = 1 }, .{ .dist = 322, .length = 4 },
317	.{ .dist = 16, .length = 4 }, .{ .dist = 0, .length = 3 }, .{ .dist = 402, .length = 3 },
318	.{ .dist = 0, .length = 2 }, .{ .dist = 237, .length = 4 }, .{ .dist = 0, .length = 2 },
319	.{ .dist = 432, .length = 4 }, .{ .dist = 0, .length = 1 }, .{ .dist = 483, .length = 5 },
320	.{ .dist = 0, .length = 2 }, .{ .dist = 294, .length = 4 }, .{ .dist = 0, .length = 2 },
321	.{ .dist = 306, .length = 3 }, .{ .dist = 113, .length = 5 }, .{ .dist = 0, .length = 1 },
322	.{ .dist = 26, .length = 4 }, .{ .dist = 164, .length = 3 }, .{ .dist = 488, .length = 4 },
323	.{ .dist = 0, .length = 1 }, .{ .dist = 542, .length = 3 }, .{ .dist = 248, .length = 6 },
324	.{ .dist = 0, .length = 5 }, .{ .dist = 205, .length = 3 }, .{ .dist = 0, .length = 8 },
325	.{ .dist = 48, .length = 3 }, .{ .dist = 449, .length = 6 }, .{ .dist = 0, .length = 2 },
326	.{ .dist = 192, .length = 3 }, .{ .dist = 328, .length = 4 }, .{ .dist = 9, .length = 5 },
327	.{ .dist = 433, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 622, .length = 25 },
328	.{ .dist = 615, .length = 5 }, .{ .dist = 46, .length = 5 }, .{ .dist = 0, .length = 2 },
329	.{ .dist = 104, .length = 3 }, .{ .dist = 475, .length = 10 }, .{ .dist = 549, .length = 3 },
330	.{ .dist = 0, .length = 4 }, .{ .dist = 597, .length = 8 }, .{ .dist = 314, .length = 3 },
331	.{ .dist = 0, .length = 1 }, .{ .dist = 473, .length = 6 }, .{ .dist = 317, .length = 5 },
332	.{ .dist = 0, .length = 1 }, .{ .dist = 400, .length = 3 }, .{ .dist = 0, .length = 3 },
333	.{ .dist = 109, .length = 3 }, .{ .dist = 151, .length = 3 }, .{ .dist = 48, .length = 4 },
334	.{ .dist = 0, .length = 4 }, .{ .dist = 125, .length = 3 }, .{ .dist = 108, .length = 3 },
335	.{ .dist = 0, .length = 2 },
336	};
337
338	var got_list = ArrayList(u8).init(testing.allocator);
339	defer got_list.deinit();
340	var got = got_list.writer();
341
342	var want_list = ArrayList(u8).init(testing.allocator);
343	defer want_list.deinit();
344	var want = want_list.writer();
345
346	var dd = DictDecoder{};
347	try dd.init(testing.allocator, 1 << 11, null);
348	defer dd.deinit();
349
350	const util = struct {
351	fn writeCopy(dst_dd: *DictDecoder, dst: anytype, dist: u32, length: u32) !void {
352	var len = length;
353	while (len > 0) {
354	var n = dst_dd.tryWriteCopy(dist, len);
355	if (n == 0) {
356	n = dst_dd.writeCopy(dist, len);
357	}
358
359	len -= n;
360	if (dst_dd.availWrite() == 0) {
361	_ = try dst.write(dst_dd.readFlush());
362	}
363	}
364	}
365	fn writeString(dst_dd: *DictDecoder, dst: anytype, str: []const u8) !void {
366	var string = str;
367	while (string.len > 0) {
368	var cnt = DictDecoder.copy(dst_dd.writeSlice(), string);
369	dst_dd.writeMark(cnt);
370	string = string[cnt..];
371	if (dst_dd.availWrite() == 0) {
372	_ = try dst.write(dst_dd.readFlush());
373	}
374	}
375	}
376	};
377
378	try util.writeString(&dd, got, ".");
379	_ = try want.write(".");
380
381	var str = poem;
382	for (poem_refs) \|ref, i\| {
383	_ = i;
384	if (ref.dist == 0) {
385	try util.writeString(&dd, got, str[0..ref.length]);
386	} else {
387	try util.writeCopy(&dd, got, ref.dist, ref.length);
388	}
389	str = str[ref.length..];
390	}
391	_ = try want.write(poem);
392
393	try util.writeCopy(&dd, got, dd.histSize(), 33);
394	_ = try want.write(want_list.items[0..33]);
395
396	try util.writeString(&dd, got, abc);
397	try util.writeCopy(&dd, got, abc.len, 59 * abc.len);
398	_ = try want.write(abc ** 60);
399
400	try util.writeString(&dd, got, fox);
401	try util.writeCopy(&dd, got, fox.len, 9 * fox.len);
402	_ = try want.write(fox ** 10);
403
404	try util.writeString(&dd, got, ".");
405	try util.writeCopy(&dd, got, 1, 9);
406	_ = try want.write("." ** 10);
407
408	try util.writeString(&dd, got, uppercase);
409	try util.writeCopy(&dd, got, uppercase.len, 7 * uppercase.len);
410	var i: u8 = 0;
411	while (i < 8) : (i += 1) {
412	_ = try want.write(uppercase);
413	}
414
415	try util.writeCopy(&dd, got, dd.histSize(), 10);
416	_ = try want.write(want_list.items[want_list.items.len - dd.histSize() ..][0..10]);
417
418	_ = try got.write(dd.readFlush());
419	try expect(mem.eql(u8, got_list.items, want_list.items));
420	}

lib/std/compress/deflate/huffman_bit_writer.zig created+1722

...	@@ -0,0 +1,1722 @@
1	const std = @import("std");
2	const builtin = @import("builtin");
3	const io = std.io;
4
5	const Allocator = std.mem.Allocator;
6
7	const deflate_const = @import("deflate_const.zig");
8	const hm_code = @import("huffman_code.zig");
9	const token = @import("token.zig");
10
11	// The first length code.
12	const length_codes_start = 257;
13
14	// The number of codegen codes.
15	const codegen_code_count = 19;
16	const bad_code = 255;
17
18	// buffer_flush_size indicates the buffer size
19	// after which bytes are flushed to the writer.
20	// Should preferably be a multiple of 6, since
21	// we accumulate 6 bytes between writes to the buffer.
22	const buffer_flush_size = 240;
23
24	// buffer_size is the actual output byte buffer size.
25	// It must have additional headroom for a flush
26	// which can contain up to 8 bytes.
27	const buffer_size = buffer_flush_size + 8;
28
29	// The number of extra bits needed by length code X - LENGTH_CODES_START.
30	var length_extra_bits = [_]u8{
31	0, 0, 0, // 257
32	0, 0, 0, 0, 0, 1, 1, 1, 1, 2, // 260
33	2, 2, 2, 3, 3, 3, 3, 4, 4, 4, // 270
34	4, 5, 5, 5, 5, 0, // 280
35	};
36
37	// The length indicated by length code X - LENGTH_CODES_START.
38	var length_base = [_]u32{
39	0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
40	12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
41	64, 80, 96, 112, 128, 160, 192, 224, 255,
42	};
43
44	// offset code word extra bits.
45	var offset_extra_bits = [_]i8{
46	0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
47	4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
48	9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
49	};
50
51	var offset_base = [_]u32{
52	0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
53	0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
54	0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
55	0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
56	0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
57	0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
58	};
59
60	// The odd order in which the codegen code sizes are written.
61	var codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
62
63	pub fn HuffmanBitWriter(comptime WriterType: type) type {
64	return struct {
65	const Self = @This();
66	pub const Error = WriterType.Error;
67
68	// writer is the underlying writer.
69	// Do not use it directly; use the write method, which ensures
70	// that Write errors are sticky.
71	inner_writer: WriterType,
72	bytes_written: usize,
73
74	// Data waiting to be written is bytes[0 .. nbytes]
75	// and then the low nbits of bits. Data is always written
76	// sequentially into the bytes array.
77	bits: u64,
78	nbits: u32, // number of bits
79	bytes: [buffer_size]u8,
80	codegen_freq: [codegen_code_count]u16,
81	nbytes: u32, // number of bytes
82	literal_freq: []u16,
83	offset_freq: []u16,
84	codegen: []u8,
85	literal_encoding: hm_code.HuffmanEncoder,
86	offset_encoding: hm_code.HuffmanEncoder,
87	codegen_encoding: hm_code.HuffmanEncoder,
88	err: bool = false,
89	fixed_literal_encoding: hm_code.HuffmanEncoder,
90	fixed_offset_encoding: hm_code.HuffmanEncoder,
91	allocator: Allocator,
92	huff_offset: hm_code.HuffmanEncoder,
93
94	pub fn reset(self: *Self, new_writer: WriterType) void {
95	self.inner_writer = new_writer;
96	self.bytes_written = 0;
97	self.bits = 0;
98	self.nbits = 0;
99	self.nbytes = 0;
100	self.err = false;
101	}
102
103	pub fn flush(self: *Self) Error!void {
104	if (self.err) {
105	self.nbits = 0;
106	return;
107	}
108	var n = self.nbytes;
109	while (self.nbits != 0) {
110	self.bytes[n] = @truncate(u8, self.bits);
111	self.bits >>= 8;
112	if (self.nbits > 8) { // Avoid underflow
113	self.nbits -= 8;
114	} else {
115	self.nbits = 0;
116	}
117	n += 1;
118	}
119	self.bits = 0;
120	try self.write(self.bytes[0..n]);
121	self.nbytes = 0;
122	}
123
124	fn write(self: *Self, b: []u8) Error!void {
125	if (self.err) {
126	return;
127	}
128	self.bytes_written += try self.inner_writer.write(b);
129	}
130
131	fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
132	if (self.err) {
133	return;
134	}
135	self.bits \|= @intCast(u64, b) << @intCast(u6, self.nbits);
136	self.nbits += nb;
137	if (self.nbits >= 48) {
138	var bits = self.bits;
139	self.bits >>= 48;
140	self.nbits -= 48;
141	var n = self.nbytes;
142	var bytes = self.bytes[n .. n + 6];
143	bytes[0] = @truncate(u8, bits);
144	bytes[1] = @truncate(u8, bits >> 8);
145	bytes[2] = @truncate(u8, bits >> 16);
146	bytes[3] = @truncate(u8, bits >> 24);
147	bytes[4] = @truncate(u8, bits >> 32);
148	bytes[5] = @truncate(u8, bits >> 40);
149	n += 6;
150	if (n >= buffer_flush_size) {
151	try self.write(self.bytes[0..n]);
152	n = 0;
153	}
154	self.nbytes = n;
155	}
156	}
157
158	pub fn writeBytes(self: *Self, bytes: []u8) Error!void {
159	if (self.err) {
160	return;
161	}
162	var n = self.nbytes;
163	if (self.nbits & 7 != 0) {
164	self.err = true; // unfinished bits
165	return;
166	}
167	while (self.nbits != 0) {
168	self.bytes[n] = @truncate(u8, self.bits);
169	self.bits >>= 8;
170	self.nbits -= 8;
171	n += 1;
172	}
173	if (n != 0) {
174	try self.write(self.bytes[0..n]);
175	}
176	self.nbytes = 0;
177	try self.write(bytes);
178	}
179
180	// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
181	// the literal and offset lengths arrays (which are concatenated into a single
182	// array). This method generates that run-length encoding.
183	//
184	// The result is written into the codegen array, and the frequencies
185	// of each code is written into the codegen_freq array.
186	// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
187	// information. Code bad_code is an end marker
188	//
189	// num_literals: The number of literals in literal_encoding
190	// num_offsets: The number of offsets in offset_encoding
191	// lit_enc: The literal encoder to use
192	// off_enc: The offset encoder to use
193	fn generateCodegen(
194	self: *Self,
195	num_literals: u32,
196	num_offsets: u32,
197	lit_enc: *hm_code.HuffmanEncoder,
198	off_enc: *hm_code.HuffmanEncoder,
199	) void {
200	for (self.codegen_freq) \|_, i\| {
201	self.codegen_freq[i] = 0;
202	}
203
204	// Note that we are using codegen both as a temporary variable for holding
205	// a copy of the frequencies, and as the place where we put the result.
206	// This is fine because the output is always shorter than the input used
207	// so far.
208	var codegen = self.codegen; // cache
209	// Copy the concatenated code sizes to codegen. Put a marker at the end.
210	var cgnl = codegen[0..num_literals];
211	for (cgnl) \|_, i\| {
212	cgnl[i] = @intCast(u8, lit_enc.codes[i].len);
213	}
214
215	cgnl = codegen[num_literals .. num_literals + num_offsets];
216	for (cgnl) \|_, i\| {
217	cgnl[i] = @intCast(u8, off_enc.codes[i].len);
218	}
219	codegen[num_literals + num_offsets] = bad_code;
220
221	var size = codegen[0];
222	var count: i32 = 1;
223	var out_index: u32 = 0;
224	var in_index: u32 = 1;
225	while (size != bad_code) : (in_index += 1) {
226	// INVARIANT: We have seen "count" copies of size that have not yet
227	// had output generated for them.
228	var next_size = codegen[in_index];
229	if (next_size == size) {
230	count += 1;
231	continue;
232	}
233	// We need to generate codegen indicating "count" of size.
234	if (size != 0) {
235	codegen[out_index] = size;
236	out_index += 1;
237	self.codegen_freq[size] += 1;
238	count -= 1;
239	while (count >= 3) {
240	var n: i32 = 6;
241	if (n > count) {
242	n = count;
243	}
244	codegen[out_index] = 16;
245	out_index += 1;
246	codegen[out_index] = @intCast(u8, n - 3);
247	out_index += 1;
248	self.codegen_freq[16] += 1;
249	count -= n;
250	}
251	} else {
252	while (count >= 11) {
253	var n: i32 = 138;
254	if (n > count) {
255	n = count;
256	}
257	codegen[out_index] = 18;
258	out_index += 1;
259	codegen[out_index] = @intCast(u8, n - 11);
260	out_index += 1;
261	self.codegen_freq[18] += 1;
262	count -= n;
263	}
264	if (count >= 3) {
265	// 3 <= count <= 10
266	codegen[out_index] = 17;
267	out_index += 1;
268	codegen[out_index] = @intCast(u8, count - 3);
269	out_index += 1;
270	self.codegen_freq[17] += 1;
271	count = 0;
272	}
273	}
274	count -= 1;
275	while (count >= 0) : (count -= 1) {
276	codegen[out_index] = size;
277	out_index += 1;
278	self.codegen_freq[size] += 1;
279	}
280	// Set up invariant for next time through the loop.
281	size = next_size;
282	count = 1;
283	}
284	// Marker indicating the end of the codegen.
285	codegen[out_index] = bad_code;
286	}
287
288	// dynamicSize returns the size of dynamically encoded data in bits.
289	fn dynamicSize(
290	self: *Self,
291	lit_enc: *hm_code.HuffmanEncoder, // literal encoder
292	off_enc: *hm_code.HuffmanEncoder, // offset encoder
293	extra_bits: u32,
294	) DynamicSize {
295	var num_codegens = self.codegen_freq.len;
296	while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
297	num_codegens -= 1;
298	}
299	var header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
300	self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
301	self.codegen_freq[16] * 2 +
302	self.codegen_freq[17] * 3 +
303	self.codegen_freq[18] * 7;
304	var size = header +
305	lit_enc.bitLength(self.literal_freq) +
306	off_enc.bitLength(self.offset_freq) +
307	extra_bits;
308
309	return DynamicSize{
310	.size = @intCast(u32, size),
311	.num_codegens = @intCast(u32, num_codegens),
312	};
313	}
314
315	// fixedSize returns the size of dynamically encoded data in bits.
316	fn fixedSize(self: *Self, extra_bits: u32) u32 {
317	return 3 +
318	self.fixed_literal_encoding.bitLength(self.literal_freq) +
319	self.fixed_offset_encoding.bitLength(self.offset_freq) +
320	extra_bits;
321	}
322
323	// storedSizeFits calculates the stored size, including header.
324	// The function returns the size in bits and whether the block
325	// fits inside a single block.
326	fn storedSizeFits(in: ?[]u8) StoredSize {
327	if (in == null) {
328	return .{ .size = 0, .storable = false };
329	}
330	if (in.?.len <= deflate_const.max_store_block_size) {
331	return .{ .size = @intCast(u32, (in.?.len + 5) * 8), .storable = true };
332	}
333	return .{ .size = 0, .storable = false };
334	}
335
336	fn writeCode(self: *Self, c: hm_code.HuffCode) Error!void {
337	if (self.err) {
338	return;
339	}
340	self.bits \|= @intCast(u64, c.code) << @intCast(u6, self.nbits);
341	self.nbits += @intCast(u32, c.len);
342	if (self.nbits >= 48) {
343	var bits = self.bits;
344	self.bits >>= 48;
345	self.nbits -= 48;
346	var n = self.nbytes;
347	var bytes = self.bytes[n .. n + 6];
348	bytes[0] = @truncate(u8, bits);
349	bytes[1] = @truncate(u8, bits >> 8);
350	bytes[2] = @truncate(u8, bits >> 16);
351	bytes[3] = @truncate(u8, bits >> 24);
352	bytes[4] = @truncate(u8, bits >> 32);
353	bytes[5] = @truncate(u8, bits >> 40);
354	n += 6;
355	if (n >= buffer_flush_size) {
356	try self.write(self.bytes[0..n]);
357	n = 0;
358	}
359	self.nbytes = n;
360	}
361	}
362
363	// Write the header of a dynamic Huffman block to the output stream.
364	//
365	// num_literals: The number of literals specified in codegen
366	// num_offsets: The number of offsets specified in codegen
367	// num_codegens: The number of codegens used in codegen
368	// is_eof: Is it the end-of-file? (end of stream)
369	fn writeDynamicHeader(
370	self: *Self,
371	num_literals: u32,
372	num_offsets: u32,
373	num_codegens: u32,
374	is_eof: bool,
375	) Error!void {
376	if (self.err) {
377	return;
378	}
379	var first_bits: u32 = 4;
380	if (is_eof) {
381	first_bits = 5;
382	}
383	try self.writeBits(first_bits, 3);
384	try self.writeBits(@intCast(u32, num_literals - 257), 5);
385	try self.writeBits(@intCast(u32, num_offsets - 1), 5);
386	try self.writeBits(@intCast(u32, num_codegens - 4), 4);
387
388	var i: u32 = 0;
389	while (i < num_codegens) : (i += 1) {
390	var value = @intCast(u32, self.codegen_encoding.codes[codegen_order[i]].len);
391	try self.writeBits(@intCast(u32, value), 3);
392	}
393
394	i = 0;
395	while (true) {
396	var code_word: u32 = @intCast(u32, self.codegen[i]);
397	i += 1;
398	if (code_word == bad_code) {
399	break;
400	}
401	try self.writeCode(self.codegen_encoding.codes[@intCast(u32, code_word)]);
402
403	switch (code_word) {
404	16 => {
405	try self.writeBits(@intCast(u32, self.codegen[i]), 2);
406	i += 1;
407	},
408	17 => {
409	try self.writeBits(@intCast(u32, self.codegen[i]), 3);
410	i += 1;
411	},
412	18 => {
413	try self.writeBits(@intCast(u32, self.codegen[i]), 7);
414	i += 1;
415	},
416	else => {},
417	}
418	}
419	}
420
421	pub fn writeStoredHeader(self: *Self, length: usize, is_eof: bool) Error!void {
422	if (self.err) {
423	return;
424	}
425	var flag: u32 = 0;
426	if (is_eof) {
427	flag = 1;
428	}
429	try self.writeBits(flag, 3);
430	try self.flush();
431	try self.writeBits(@intCast(u32, length), 16);
432	try self.writeBits(@intCast(u32, ~@intCast(u16, length)), 16);
433	}
434
435	fn writeFixedHeader(self: *Self, is_eof: bool) Error!void {
436	if (self.err) {
437	return;
438	}
439	// Indicate that we are a fixed Huffman block
440	var value: u32 = 2;
441	if (is_eof) {
442	value = 3;
443	}
444	try self.writeBits(value, 3);
445	}
446
447	// Write a block of tokens with the smallest encoding.
448	// The original input can be supplied, and if the huffman encoded data
449	// is larger than the original bytes, the data will be written as a
450	// stored block.
451	// If the input is null, the tokens will always be Huffman encoded.
452	pub fn writeBlock(
453	self: *Self,
454	tokens: []const token.Token,
455	eof: bool,
456	input: ?[]u8,
457	) Error!void {
458	if (self.err) {
459	return;
460	}
461
462	var lit_and_off = self.indexTokens(tokens);
463	var num_literals = lit_and_off.num_literals;
464	var num_offsets = lit_and_off.num_offsets;
465
466	var extra_bits: u32 = 0;
467	var ret = storedSizeFits(input);
468	var stored_size = ret.size;
469	var storable = ret.storable;
470
471	if (storable) {
472	// We only bother calculating the costs of the extra bits required by
473	// the length of offset fields (which will be the same for both fixed
474	// and dynamic encoding), if we need to compare those two encodings
475	// against stored encoding.
476	var length_code: u32 = length_codes_start + 8;
477	while (length_code < num_literals) : (length_code += 1) {
478	// First eight length codes have extra size = 0.
479	extra_bits += @intCast(u32, self.literal_freq[length_code]) *
480	@intCast(u32, length_extra_bits[length_code - length_codes_start]);
481	}
482	var offset_code: u32 = 4;
483	while (offset_code < num_offsets) : (offset_code += 1) {
484	// First four offset codes have extra size = 0.
485	extra_bits += @intCast(u32, self.offset_freq[offset_code]) *
486	@intCast(u32, offset_extra_bits[offset_code]);
487	}
488	}
489
490	// Figure out smallest code.
491	// Fixed Huffman baseline.
492	var literal_encoding = &self.fixed_literal_encoding;
493	var offset_encoding = &self.fixed_offset_encoding;
494	var size = self.fixedSize(extra_bits);
495
496	// Dynamic Huffman?
497	var num_codegens: u32 = 0;
498
499	// Generate codegen and codegenFrequencies, which indicates how to encode
500	// the literal_encoding and the offset_encoding.
501	self.generateCodegen(
502	num_literals,
503	num_offsets,
504	&self.literal_encoding,
505	&self.offset_encoding,
506	);
507	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
508	var dynamic_size = self.dynamicSize(
509	&self.literal_encoding,
510	&self.offset_encoding,
511	extra_bits,
512	);
513	var dyn_size = dynamic_size.size;
514	num_codegens = dynamic_size.num_codegens;
515
516	if (dyn_size < size) {
517	size = dyn_size;
518	literal_encoding = &self.literal_encoding;
519	offset_encoding = &self.offset_encoding;
520	}
521
522	// Stored bytes?
523	if (storable and stored_size < size) {
524	try self.writeStoredHeader(input.?.len, eof);
525	try self.writeBytes(input.?);
526	return;
527	}
528
529	// Huffman.
530	if (@ptrToInt(literal_encoding) == @ptrToInt(&self.fixed_literal_encoding)) {
531	try self.writeFixedHeader(eof);
532	} else {
533	try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
534	}
535
536	// Write the tokens.
537	try self.writeTokens(tokens, literal_encoding.codes, offset_encoding.codes);
538	}
539
540	// writeBlockDynamic encodes a block using a dynamic Huffman table.
541	// This should be used if the symbols used have a disproportionate
542	// histogram distribution.
543	// If input is supplied and the compression savings are below 1/16th of the
544	// input size the block is stored.
545	pub fn writeBlockDynamic(
546	self: *Self,
547	tokens: []const token.Token,
548	eof: bool,
549	input: ?[]u8,
550	) Error!void {
551	if (self.err) {
552	return;
553	}
554
555	var total_tokens = self.indexTokens(tokens);
556	var num_literals = total_tokens.num_literals;
557	var num_offsets = total_tokens.num_offsets;
558
559	// Generate codegen and codegenFrequencies, which indicates how to encode
560	// the literal_encoding and the offset_encoding.
561	self.generateCodegen(
562	num_literals,
563	num_offsets,
564	&self.literal_encoding,
565	&self.offset_encoding,
566	);
567	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
568	var dynamic_size = self.dynamicSize(&self.literal_encoding, &self.offset_encoding, 0);
569	var size = dynamic_size.size;
570	var num_codegens = dynamic_size.num_codegens;
571
572	// Store bytes, if we don't get a reasonable improvement.
573
574	var stored_size = storedSizeFits(input);
575	var ssize = stored_size.size;
576	var storable = stored_size.storable;
577	if (storable and ssize < (size + (size >> 4))) {
578	try self.writeStoredHeader(input.?.len, eof);
579	try self.writeBytes(input.?);
580	return;
581	}
582
583	// Write Huffman table.
584	try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
585
586	// Write the tokens.
587	try self.writeTokens(tokens, self.literal_encoding.codes, self.offset_encoding.codes);
588	}
589
590	const TotalIndexedTokens = struct {
591	num_literals: u32,
592	num_offsets: u32,
593	};
594
595	// Indexes a slice of tokens followed by an end_block_marker, and updates
596	// literal_freq and offset_freq, and generates literal_encoding
597	// and offset_encoding.
598	// The number of literal and offset tokens is returned.
599	fn indexTokens(self: *Self, tokens: []const token.Token) TotalIndexedTokens {
600	var num_literals: u32 = 0;
601	var num_offsets: u32 = 0;
602
603	for (self.literal_freq) \|_, i\| {
604	self.literal_freq[i] = 0;
605	}
606	for (self.offset_freq) \|_, i\| {
607	self.offset_freq[i] = 0;
608	}
609
610	for (tokens) \|t\| {
611	if (t < token.match_type) {
612	self.literal_freq[token.literal(t)] += 1;
613	continue;
614	}
615	var length = token.length(t);
616	var offset = token.offset(t);
617	self.literal_freq[length_codes_start + token.lengthCode(length)] += 1;
618	self.offset_freq[token.offsetCode(offset)] += 1;
619	}
620	// add end_block_marker token at the end
621	self.literal_freq[token.literal(deflate_const.end_block_marker)] += 1;
622
623	// get the number of literals
624	num_literals = @intCast(u32, self.literal_freq.len);
625	while (self.literal_freq[num_literals - 1] == 0) {
626	num_literals -= 1;
627	}
628	// get the number of offsets
629	num_offsets = @intCast(u32, self.offset_freq.len);
630	while (num_offsets > 0 and self.offset_freq[num_offsets - 1] == 0) {
631	num_offsets -= 1;
632	}
633	if (num_offsets == 0) {
634	// We haven't found a single match. If we want to go with the dynamic encoding,
635	// we should count at least one offset to be sure that the offset huffman tree could be encoded.
636	self.offset_freq[0] = 1;
637	num_offsets = 1;
638	}
639	self.literal_encoding.generate(self.literal_freq, 15);
640	self.offset_encoding.generate(self.offset_freq, 15);
641	return TotalIndexedTokens{
642	.num_literals = num_literals,
643	.num_offsets = num_offsets,
644	};
645	}
646
647	// Writes a slice of tokens to the output followed by and end_block_marker.
648	// codes for literal and offset encoding must be supplied.
649	fn writeTokens(
650	self: *Self,
651	tokens: []const token.Token,
652	le_codes: []hm_code.HuffCode,
653	oe_codes: []hm_code.HuffCode,
654	) Error!void {
655	if (self.err) {
656	return;
657	}
658	for (tokens) \|t\| {
659	if (t < token.match_type) {
660	try self.writeCode(le_codes[token.literal(t)]);
661	continue;
662	}
663	// Write the length
664	var length = token.length(t);
665	var length_code = token.lengthCode(length);
666	try self.writeCode(le_codes[length_code + length_codes_start]);
667	var extra_length_bits = @intCast(u32, length_extra_bits[length_code]);
668	if (extra_length_bits > 0) {
669	var extra_length = @intCast(u32, length - length_base[length_code]);
670	try self.writeBits(extra_length, extra_length_bits);
671	}
672	// Write the offset
673	var offset = token.offset(t);
674	var offset_code = token.offsetCode(offset);
675	try self.writeCode(oe_codes[offset_code]);
676	var extra_offset_bits = @intCast(u32, offset_extra_bits[offset_code]);
677	if (extra_offset_bits > 0) {
678	var extra_offset = @intCast(u32, offset - offset_base[offset_code]);
679	try self.writeBits(extra_offset, extra_offset_bits);
680	}
681	}
682	// add end_block_marker at the end
683	try self.writeCode(le_codes[token.literal(deflate_const.end_block_marker)]);
684	}
685
686	// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
687	// if the results only gains very little from compression.
688	pub fn writeBlockHuff(self: *Self, eof: bool, input: []u8) Error!void {
689	if (self.err) {
690	return;
691	}
692
693	// Clear histogram
694	for (self.literal_freq) \|_, i\| {
695	self.literal_freq[i] = 0;
696	}
697
698	// Add everything as literals
699	histogram(input, &self.literal_freq);
700
701	self.literal_freq[deflate_const.end_block_marker] = 1;
702
703	const num_literals = deflate_const.end_block_marker + 1;
704	self.offset_freq[0] = 1;
705	const num_offsets = 1;
706
707	self.literal_encoding.generate(self.literal_freq, 15);
708
709	// Figure out smallest code.
710	// Always use dynamic Huffman or Store
711	var num_codegens: u32 = 0;
712
713	// Generate codegen and codegenFrequencies, which indicates how to encode
714	// the literal_encoding and the offset_encoding.
715	self.generateCodegen(
716	num_literals,
717	num_offsets,
718	&self.literal_encoding,
719	&self.huff_offset,
720	);
721	self.codegen_encoding.generate(self.codegen_freq[0..], 7);
722	var dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_offset, 0);
723	var size = dynamic_size.size;
724	num_codegens = dynamic_size.num_codegens;
725
726	// Store bytes, if we don't get a reasonable improvement.
727
728	var stored_size_ret = storedSizeFits(input);
729	var ssize = stored_size_ret.size;
730	var storable = stored_size_ret.storable;
731
732	if (storable and ssize < (size + (size >> 4))) {
733	try self.writeStoredHeader(input.len, eof);
734	try self.writeBytes(input);
735	return;
736	}
737
738	// Huffman.
739	try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
740	var encoding = self.literal_encoding.codes[0..257];
741	var n = self.nbytes;
742	for (input) \|t\| {
743	// Bitwriting inlined, ~30% speedup
744	var c = encoding[t];
745	self.bits \|= @intCast(u64, c.code) << @intCast(u6, self.nbits);
746	self.nbits += @intCast(u32, c.len);
747	if (self.nbits < 48) {
748	continue;
749	}
750	// Store 6 bytes
751	var bits = self.bits;
752	self.bits >>= 48;
753	self.nbits -= 48;
754	var bytes = self.bytes[n .. n + 6];
755	bytes[0] = @truncate(u8, bits);
756	bytes[1] = @truncate(u8, bits >> 8);
757	bytes[2] = @truncate(u8, bits >> 16);
758	bytes[3] = @truncate(u8, bits >> 24);
759	bytes[4] = @truncate(u8, bits >> 32);
760	bytes[5] = @truncate(u8, bits >> 40);
761	n += 6;
762	if (n < buffer_flush_size) {
763	continue;
764	}
765	try self.write(self.bytes[0..n]);
766	if (self.err) {
767	return; // Return early in the event of write failures
768	}
769	n = 0;
770	}
771	self.nbytes = n;
772	try self.writeCode(encoding[deflate_const.end_block_marker]);
773	}
774
775	pub fn deinit(self: *Self) void {
776	self.allocator.free(self.literal_freq);
777	self.allocator.free(self.offset_freq);
778	self.allocator.free(self.codegen);
779	self.literal_encoding.deinit();
780	self.codegen_encoding.deinit();
781	self.offset_encoding.deinit();
782	self.fixed_literal_encoding.deinit();
783	self.fixed_offset_encoding.deinit();
784	self.huff_offset.deinit();
785	}
786	};
787	}
788
789	const DynamicSize = struct {
790	size: u32,
791	num_codegens: u32,
792	};
793
794	const StoredSize = struct {
795	size: u32,
796	storable: bool,
797	};
798
799	pub fn huffmanBitWriter(allocator: Allocator, writer: anytype) !HuffmanBitWriter(@TypeOf(writer)) {
800	var offset_freq = [1]u16{0} ** deflate_const.offset_code_count;
801	offset_freq[0] = 1;
802	// huff_offset is a static offset encoder used for huffman only encoding.
803	// It can be reused since we will not be encoding offset values.
804	var huff_offset = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count);
805	huff_offset.generate(offset_freq[0..], 15);
806
807	return HuffmanBitWriter(@TypeOf(writer)){
808	.inner_writer = writer,
809	.bytes_written = 0,
810	.bits = 0,
811	.nbits = 0,
812	.nbytes = 0,
813	.bytes = [1]u8{0} ** buffer_size,
814	.codegen_freq = [1]u16{0} ** codegen_code_count,
815	.literal_freq = try allocator.alloc(u16, deflate_const.max_num_lit),
816	.offset_freq = try allocator.alloc(u16, deflate_const.offset_code_count),
817	.codegen = try allocator.alloc(u8, deflate_const.max_num_lit + deflate_const.offset_code_count + 1),
818	.literal_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.max_num_lit),
819	.codegen_encoding = try hm_code.newHuffmanEncoder(allocator, codegen_code_count),
820	.offset_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count),
821	.allocator = allocator,
822	.fixed_literal_encoding = try hm_code.generateFixedLiteralEncoding(allocator),
823	.fixed_offset_encoding = try hm_code.generateFixedOffsetEncoding(allocator),
824	.huff_offset = huff_offset,
825	};
826	}
827
828	// histogram accumulates a histogram of b in h.
829	//
830	// h.len must be >= 256, and h's elements must be all zeroes.
831	fn histogram(b: []u8, h: *[]u16) void {
832	var lh = h.*[0..256];
833	for (b) \|t\| {
834	lh[t] += 1;
835	}
836	}
837
838	// tests
839	const expect = std.testing.expect;
840	const fmt = std.fmt;
841	const math = std.math;
842	const mem = std.mem;
843	const testing = std.testing;
844
845	const ArrayList = std.ArrayList;
846
847	test "writeBlockHuff" {
848	// Tests huffman encoding against reference files to detect possible regressions.
849	// If encoding/bit allocation changes you can regenerate these files
850
851	try testBlockHuff(
852	"huffman-null-max.input",
853	"huffman-null-max.golden",
854	);
855	try testBlockHuff(
856	"huffman-pi.input",
857	"huffman-pi.golden",
858	);
859	try testBlockHuff(
860	"huffman-rand-1k.input",
861	"huffman-rand-1k.golden",
862	);
863	try testBlockHuff(
864	"huffman-rand-limit.input",
865	"huffman-rand-limit.golden",
866	);
867	try testBlockHuff(
868	"huffman-rand-max.input",
869	"huffman-rand-max.golden",
870	);
871	try testBlockHuff(
872	"huffman-shifts.input",
873	"huffman-shifts.golden",
874	);
875	try testBlockHuff(
876	"huffman-text.input",
877	"huffman-text.golden",
878	);
879	try testBlockHuff(
880	"huffman-text-shift.input",
881	"huffman-text-shift.golden",
882	);
883	try testBlockHuff(
884	"huffman-zero.input",
885	"huffman-zero.golden",
886	);
887	}
888
889	fn testBlockHuff(in_name: []const u8, want_name: []const u8) !void {
890	// Skip wasi because it does not support std.fs.openDirAbsolute()
891	if (builtin.os.tag == .wasi) return error.SkipZigTest;
892
893	const current_dir = try std.fs.openDirAbsolute(std.fs.path.dirname(@src().file).?, .{});
894	const testdata_dir = try current_dir.openDir("testdata", .{});
895	const in_file = try testdata_dir.openFile(in_name, .{ .read = true });
896	defer in_file.close();
897	const want_file = try testdata_dir.openFile(want_name, .{ .read = true });
898	defer want_file.close();
899
900	var in = try in_file.reader().readAllAlloc(testing.allocator, math.maxInt(usize));
901	defer testing.allocator.free(in);
902	var want = try want_file.reader().readAllAlloc(testing.allocator, math.maxInt(usize));
903	defer testing.allocator.free(want);
904
905	var buf = ArrayList(u8).init(testing.allocator);
906	defer buf.deinit();
907	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
908	defer bw.deinit();
909	try bw.writeBlockHuff(false, in);
910	try bw.flush();
911
912	try expect(mem.eql(u8, buf.items, want));
913
914	// Test if the writer produces the same output after reset.
915	var buf_after_reset = ArrayList(u8).init(testing.allocator);
916	defer buf_after_reset.deinit();
917
918	bw.reset(buf_after_reset.writer());
919
920	try bw.writeBlockHuff(false, in);
921	try bw.flush();
922
923	try expect(mem.eql(u8, buf_after_reset.items, buf.items));
924	try expect(mem.eql(u8, buf_after_reset.items, want));
925
926	try testWriterEOF(.write_huffman_block, &[0]token.Token{}, in);
927	}
928
929	const HuffTest = struct {
930	tokens: []const token.Token,
931	input: []const u8 = "", // File name of input data matching the tokens.
932	want: []const u8 = "", // File name of data with the expected output with input available.
933	want_no_input: []const u8 = "", // File name of the expected output when no input is available.
934	};
935
936	const ml = 0x7fc00000; // Maximum length token. Used to reduce the size of writeBlockTests
937
938	const writeBlockTests = &[_]HuffTest{
939	HuffTest{
940	.input = "huffman-null-max.input",
941	.want = "huffman-null-max.{s}.expect",
942	.want_no_input = "huffman-null-max.{s}.expect-noinput",
943	.tokens = &[_]token.Token{
944	0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
945	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
946	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
947	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
948	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
949	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
950	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
951	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
952	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
953	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
954	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
955	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
956	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x0, 0x0,
957	},
958	},
959	HuffTest{
960	.input = "huffman-pi.input",
961	.want = "huffman-pi.{s}.expect",
962	.want_no_input = "huffman-pi.{s}.expect-noinput",
963	.tokens = &[_]token.Token{
964	0x33, 0x2e, 0x31, 0x34, 0x31, 0x35, 0x39,
965	0x32, 0x36, 0x35, 0x33, 0x35, 0x38, 0x39,
966	0x37, 0x39, 0x33, 0x32, 0x33, 0x38, 0x34,
967	0x36, 0x32, 0x36, 0x34, 0x33, 0x33, 0x38,
968	0x33, 0x32, 0x37, 0x39, 0x35, 0x30, 0x32,
969	0x38, 0x38, 0x34, 0x31, 0x39, 0x37, 0x31,
970	0x36, 0x39, 0x33, 0x39, 0x39, 0x33, 0x37,
971	0x35, 0x31, 0x30, 0x35, 0x38, 0x32, 0x30,
972	0x39, 0x37, 0x34, 0x39, 0x34, 0x34, 0x35,
973	0x39, 0x32, 0x33, 0x30, 0x37, 0x38, 0x31,
974	0x36, 0x34, 0x30, 0x36, 0x32, 0x38, 0x36,
975	0x32, 0x30, 0x38, 0x39, 0x39, 0x38, 0x36,
976	0x32, 0x38, 0x30, 0x33, 0x34, 0x38, 0x32,
977	0x35, 0x33, 0x34, 0x32, 0x31, 0x31, 0x37,
978	0x30, 0x36, 0x37, 0x39, 0x38, 0x32, 0x31,
979	0x34, 0x38, 0x30, 0x38, 0x36, 0x35, 0x31,
980	0x33, 0x32, 0x38, 0x32, 0x33, 0x30, 0x36,
981	0x36, 0x34, 0x37, 0x30, 0x39, 0x33, 0x38,
982	0x34, 0x34, 0x36, 0x30, 0x39, 0x35, 0x35,
983	0x30, 0x35, 0x38, 0x32, 0x32, 0x33, 0x31,
984	0x37, 0x32, 0x35, 0x33, 0x35, 0x39, 0x34,
985	0x30, 0x38, 0x31, 0x32, 0x38, 0x34, 0x38,
986	0x31, 0x31, 0x31, 0x37, 0x34, 0x4040007e, 0x34,
987	0x31, 0x30, 0x32, 0x37, 0x30, 0x31, 0x39,
988	0x33, 0x38, 0x35, 0x32, 0x31, 0x31, 0x30,
989	0x35, 0x35, 0x35, 0x39, 0x36, 0x34, 0x34,
990	0x36, 0x32, 0x32, 0x39, 0x34, 0x38, 0x39,
991	0x35, 0x34, 0x39, 0x33, 0x30, 0x33, 0x38,
992	0x31, 0x40400012, 0x32, 0x38, 0x38, 0x31, 0x30,
993	0x39, 0x37, 0x35, 0x36, 0x36, 0x35, 0x39,
994	0x33, 0x33, 0x34, 0x34, 0x36, 0x40400047, 0x37,
995	0x35, 0x36, 0x34, 0x38, 0x32, 0x33, 0x33,
996	0x37, 0x38, 0x36, 0x37, 0x38, 0x33, 0x31,
997	0x36, 0x35, 0x32, 0x37, 0x31, 0x32, 0x30,
998	0x31, 0x39, 0x30, 0x39, 0x31, 0x34, 0x4040001a,
999	0x35, 0x36, 0x36, 0x39, 0x32, 0x33, 0x34,
1000	0x36, 0x404000b2, 0x36, 0x31, 0x30, 0x34, 0x35,
1001	0x34, 0x33, 0x32, 0x36, 0x40400032, 0x31, 0x33,
1002	0x33, 0x39, 0x33, 0x36, 0x30, 0x37, 0x32,
1003	0x36, 0x30, 0x32, 0x34, 0x39, 0x31, 0x34,
1004	0x31, 0x32, 0x37, 0x33, 0x37, 0x32, 0x34,
1005	0x35, 0x38, 0x37, 0x30, 0x30, 0x36, 0x36,
1006	0x30, 0x36, 0x33, 0x31, 0x35, 0x35, 0x38,
1007	0x38, 0x31, 0x37, 0x34, 0x38, 0x38, 0x31,
1008	0x35, 0x32, 0x30, 0x39, 0x32, 0x30, 0x39,
1009	0x36, 0x32, 0x38, 0x32, 0x39, 0x32, 0x35,
1010	0x34, 0x30, 0x39, 0x31, 0x37, 0x31, 0x35,
1011	0x33, 0x36, 0x34, 0x33, 0x36, 0x37, 0x38,
1012	0x39, 0x32, 0x35, 0x39, 0x30, 0x33, 0x36,
1013	0x30, 0x30, 0x31, 0x31, 0x33, 0x33, 0x30,
1014	0x35, 0x33, 0x30, 0x35, 0x34, 0x38, 0x38,
1015	0x32, 0x30, 0x34, 0x36, 0x36, 0x35, 0x32,
1016	0x31, 0x33, 0x38, 0x34, 0x31, 0x34, 0x36,
1017	0x39, 0x35, 0x31, 0x39, 0x34, 0x31, 0x35,
1018	0x31, 0x31, 0x36, 0x30, 0x39, 0x34, 0x33,
1019	0x33, 0x30, 0x35, 0x37, 0x32, 0x37, 0x30,
1020	0x33, 0x36, 0x35, 0x37, 0x35, 0x39, 0x35,
1021	0x39, 0x31, 0x39, 0x35, 0x33, 0x30, 0x39,
1022	0x32, 0x31, 0x38, 0x36, 0x31, 0x31, 0x37,
1023	0x404000e9, 0x33, 0x32, 0x40400009, 0x39, 0x33, 0x31,
1024	0x30, 0x35, 0x31, 0x31, 0x38, 0x35, 0x34,
1025	0x38, 0x30, 0x37, 0x4040010e, 0x33, 0x37, 0x39,
1026	0x39, 0x36, 0x32, 0x37, 0x34, 0x39, 0x35,
1027	0x36, 0x37, 0x33, 0x35, 0x31, 0x38, 0x38,
1028	0x35, 0x37, 0x35, 0x32, 0x37, 0x32, 0x34,
1029	0x38, 0x39, 0x31, 0x32, 0x32, 0x37, 0x39,
1030	0x33, 0x38, 0x31, 0x38, 0x33, 0x30, 0x31,
1031	0x31, 0x39, 0x34, 0x39, 0x31, 0x32, 0x39,
1032	0x38, 0x33, 0x33, 0x36, 0x37, 0x33, 0x33,
1033	0x36, 0x32, 0x34, 0x34, 0x30, 0x36, 0x35,
1034	0x36, 0x36, 0x34, 0x33, 0x30, 0x38, 0x36,
1035	0x30, 0x32, 0x31, 0x33, 0x39, 0x34, 0x39,
1036	0x34, 0x36, 0x33, 0x39, 0x35, 0x32, 0x32,
1037	0x34, 0x37, 0x33, 0x37, 0x31, 0x39, 0x30,
1038	0x37, 0x30, 0x32, 0x31, 0x37, 0x39, 0x38,
1039	0x40800099, 0x37, 0x30, 0x32, 0x37, 0x37, 0x30,
1040	0x35, 0x33, 0x39, 0x32, 0x31, 0x37, 0x31,
1041	0x37, 0x36, 0x32, 0x39, 0x33, 0x31, 0x37,
1042	0x36, 0x37, 0x35, 0x40800232, 0x37, 0x34, 0x38,
1043	0x31, 0x40400006, 0x36, 0x36, 0x39, 0x34, 0x30,
1044	0x404001e7, 0x30, 0x30, 0x30, 0x35, 0x36, 0x38,
1045	0x31, 0x32, 0x37, 0x31, 0x34, 0x35, 0x32,
1046	0x36, 0x33, 0x35, 0x36, 0x30, 0x38, 0x32,
1047	0x37, 0x37, 0x38, 0x35, 0x37, 0x37, 0x31,
1048	0x33, 0x34, 0x32, 0x37, 0x35, 0x37, 0x37,
1049	0x38, 0x39, 0x36, 0x40400129, 0x33, 0x36, 0x33,
1050	0x37, 0x31, 0x37, 0x38, 0x37, 0x32, 0x31,
1051	0x34, 0x36, 0x38, 0x34, 0x34, 0x30, 0x39,
1052	0x30, 0x31, 0x32, 0x32, 0x34, 0x39, 0x35,
1053	0x33, 0x34, 0x33, 0x30, 0x31, 0x34, 0x36,
1054	0x35, 0x34, 0x39, 0x35, 0x38, 0x35, 0x33,
1055	0x37, 0x31, 0x30, 0x35, 0x30, 0x37, 0x39,
1056	0x404000ca, 0x36, 0x40400153, 0x38, 0x39, 0x32, 0x33,
1057	0x35, 0x34, 0x404001c9, 0x39, 0x35, 0x36, 0x31,
1058	0x31, 0x32, 0x31, 0x32, 0x39, 0x30, 0x32,
1059	0x31, 0x39, 0x36, 0x30, 0x38, 0x36, 0x34,
1060	0x30, 0x33, 0x34, 0x34, 0x31, 0x38, 0x31,
1061	0x35, 0x39, 0x38, 0x31, 0x33, 0x36, 0x32,
1062	0x39, 0x37, 0x37, 0x34, 0x40400074, 0x30, 0x39,
1063	0x39, 0x36, 0x30, 0x35, 0x31, 0x38, 0x37,
1064	0x30, 0x37, 0x32, 0x31, 0x31, 0x33, 0x34,
1065	0x39, 0x40800000, 0x38, 0x33, 0x37, 0x32, 0x39,
1066	0x37, 0x38, 0x30, 0x34, 0x39, 0x39, 0x404002da,
1067	0x39, 0x37, 0x33, 0x31, 0x37, 0x33, 0x32,
1068	0x38, 0x4040018a, 0x36, 0x33, 0x31, 0x38, 0x35,
1069	0x40400301, 0x404002e8, 0x34, 0x35, 0x35, 0x33, 0x34,
1070	0x36, 0x39, 0x30, 0x38, 0x33, 0x30, 0x32,
1071	0x36, 0x34, 0x32, 0x35, 0x32, 0x32, 0x33,
1072	0x30, 0x404002e3, 0x40400267, 0x38, 0x35, 0x30, 0x33,
1073	0x35, 0x32, 0x36, 0x31, 0x39, 0x33, 0x31,
1074	0x31, 0x40400212, 0x31, 0x30, 0x31, 0x30, 0x30,
1075	0x30, 0x33, 0x31, 0x33, 0x37, 0x38, 0x33,
1076	0x38, 0x37, 0x35, 0x32, 0x38, 0x38, 0x36,
1077	0x35, 0x38, 0x37, 0x35, 0x33, 0x33, 0x32,
1078	0x30, 0x38, 0x33, 0x38, 0x31, 0x34, 0x32,
1079	0x30, 0x36, 0x40400140, 0x4040012b, 0x31, 0x34, 0x37,
1080	0x33, 0x30, 0x33, 0x35, 0x39, 0x4080032e, 0x39,
1081	0x30, 0x34, 0x32, 0x38, 0x37, 0x35, 0x35,
1082	0x34, 0x36, 0x38, 0x37, 0x33, 0x31, 0x31,
1083	0x35, 0x39, 0x35, 0x40400355, 0x33, 0x38, 0x38,
1084	0x32, 0x33, 0x35, 0x33, 0x37, 0x38, 0x37,
1085	0x35, 0x4080037f, 0x39, 0x4040013a, 0x31, 0x40400148, 0x38,
1086	0x30, 0x35, 0x33, 0x4040018a, 0x32, 0x32, 0x36,
1087	0x38, 0x30, 0x36, 0x36, 0x31, 0x33, 0x30,
1088	0x30, 0x31, 0x39, 0x32, 0x37, 0x38, 0x37,
1089	0x36, 0x36, 0x31, 0x31, 0x31, 0x39, 0x35,
1090	0x39, 0x40400237, 0x36, 0x40800124, 0x38, 0x39, 0x33,
1091	0x38, 0x30, 0x39, 0x35, 0x32, 0x35, 0x37,
1092	0x32, 0x30, 0x31, 0x30, 0x36, 0x35, 0x34,
1093	0x38, 0x35, 0x38, 0x36, 0x33, 0x32, 0x37,
1094	0x4040009a, 0x39, 0x33, 0x36, 0x31, 0x35, 0x33,
1095	0x40400220, 0x4080015c, 0x32, 0x33, 0x30, 0x33, 0x30,
1096	0x31, 0x39, 0x35, 0x32, 0x30, 0x33, 0x35,
1097	0x33, 0x30, 0x31, 0x38, 0x35, 0x32, 0x40400171,
1098	0x40400075, 0x33, 0x36, 0x32, 0x32, 0x35, 0x39,
1099	0x39, 0x34, 0x31, 0x33, 0x40400254, 0x34, 0x39,
1100	0x37, 0x32, 0x31, 0x37, 0x404000de, 0x33, 0x34,
1101	0x37, 0x39, 0x31, 0x33, 0x31, 0x35, 0x31,
1102	0x35, 0x35, 0x37, 0x34, 0x38, 0x35, 0x37,
1103	0x32, 0x34, 0x32, 0x34, 0x35, 0x34, 0x31,
1104	0x35, 0x30, 0x36, 0x39, 0x4040013f, 0x38, 0x32,
1105	0x39, 0x35, 0x33, 0x33, 0x31, 0x31, 0x36,
1106	0x38, 0x36, 0x31, 0x37, 0x32, 0x37, 0x38,
1107	0x40400337, 0x39, 0x30, 0x37, 0x35, 0x30, 0x39,
1108	0x4040010d, 0x37, 0x35, 0x34, 0x36, 0x33, 0x37,
1109	0x34, 0x36, 0x34, 0x39, 0x33, 0x39, 0x33,
1110	0x31, 0x39, 0x32, 0x35, 0x35, 0x30, 0x36,
1111	0x30, 0x34, 0x30, 0x30, 0x39, 0x4040026b, 0x31,
1112	0x36, 0x37, 0x31, 0x31, 0x33, 0x39, 0x30,
1113	0x30, 0x39, 0x38, 0x40400335, 0x34, 0x30, 0x31,
1114	0x32, 0x38, 0x35, 0x38, 0x33, 0x36, 0x31,
1115	0x36, 0x30, 0x33, 0x35, 0x36, 0x33, 0x37,
1116	0x30, 0x37, 0x36, 0x36, 0x30, 0x31, 0x30,
1117	0x34, 0x40400172, 0x38, 0x31, 0x39, 0x34, 0x32,
1118	0x39, 0x4080041e, 0x404000ef, 0x4040028b, 0x37, 0x38, 0x33,
1119	0x37, 0x34, 0x404004a8, 0x38, 0x32, 0x35, 0x35,
1120	0x33, 0x37, 0x40800209, 0x32, 0x36, 0x38, 0x4040002e,
1121	0x34, 0x30, 0x34, 0x37, 0x404001d1, 0x34, 0x404004b5,
1122	0x4040038d, 0x38, 0x34, 0x404003a8, 0x36, 0x40c0031f, 0x33,
1123	0x33, 0x31, 0x33, 0x36, 0x37, 0x37, 0x30,
1124	0x32, 0x38, 0x39, 0x38, 0x39, 0x31, 0x35,
1125	0x32, 0x40400062, 0x35, 0x32, 0x31, 0x36, 0x32,
1126	0x30, 0x35, 0x36, 0x39, 0x36, 0x40400411, 0x30,
1127	0x35, 0x38, 0x40400477, 0x35, 0x40400498, 0x35, 0x31,
1128	0x31, 0x40400209, 0x38, 0x32, 0x34, 0x33, 0x30,
1129	0x30, 0x33, 0x35, 0x35, 0x38, 0x37, 0x36,
1130	0x34, 0x30, 0x32, 0x34, 0x37, 0x34, 0x39,
1131	0x36, 0x34, 0x37, 0x33, 0x32, 0x36, 0x33,
1132	0x4040043e, 0x39, 0x39, 0x32, 0x4040044b, 0x34, 0x32,
1133	0x36, 0x39, 0x40c002c5, 0x37, 0x404001d6, 0x34, 0x4040053d,
1134	0x4040041d, 0x39, 0x33, 0x34, 0x31, 0x37, 0x404001ad,
1135	0x31, 0x32, 0x4040002a, 0x34, 0x4040019e, 0x31, 0x35,
1136	0x30, 0x33, 0x30, 0x32, 0x38, 0x36, 0x31,
1137	0x38, 0x32, 0x39, 0x37, 0x34, 0x35, 0x35,
1138	0x35, 0x37, 0x30, 0x36, 0x37, 0x34, 0x40400135,
1139	0x35, 0x30, 0x35, 0x34, 0x39, 0x34, 0x35,
1140	0x38, 0x404001c5, 0x39, 0x40400051, 0x35, 0x36, 0x404001ec,
1141	0x37, 0x32, 0x31, 0x30, 0x37, 0x39, 0x40400159,
1142	0x33, 0x30, 0x4040010a, 0x33, 0x32, 0x31, 0x31,
1143	0x36, 0x35, 0x33, 0x34, 0x34, 0x39, 0x38,
1144	0x37, 0x32, 0x30, 0x32, 0x37, 0x4040011b, 0x30,
1145	0x32, 0x33, 0x36, 0x34, 0x4040022e, 0x35, 0x34,
1146	0x39, 0x39, 0x31, 0x31, 0x39, 0x38, 0x40400418,
1147	0x34, 0x4040011b, 0x35, 0x33, 0x35, 0x36, 0x36,
1148	0x33, 0x36, 0x39, 0x40400450, 0x32, 0x36, 0x35,
1149	0x404002e4, 0x37, 0x38, 0x36, 0x32, 0x35, 0x35,
1150	0x31, 0x404003da, 0x31, 0x37, 0x35, 0x37, 0x34,
1151	0x36, 0x37, 0x32, 0x38, 0x39, 0x30, 0x39,
1152	0x37, 0x37, 0x37, 0x37, 0x40800453, 0x30, 0x30,
1153	0x30, 0x404005fd, 0x37, 0x30, 0x404004df, 0x36, 0x404003e9,
1154	0x34, 0x39, 0x31, 0x4040041e, 0x40400297, 0x32, 0x31,
1155	0x34, 0x37, 0x37, 0x32, 0x33, 0x35, 0x30,
1156	0x31, 0x34, 0x31, 0x34, 0x40400643, 0x33, 0x35,
1157	0x36, 0x404004af, 0x31, 0x36, 0x31, 0x33, 0x36,
1158	0x31, 0x31, 0x35, 0x37, 0x33, 0x35, 0x32,
1159	0x35, 0x40400504, 0x33, 0x34, 0x4040005b, 0x31, 0x38,
1160	0x4040047b, 0x38, 0x34, 0x404005e7, 0x33, 0x33, 0x32,
1161	0x33, 0x39, 0x30, 0x37, 0x33, 0x39, 0x34,
1162	0x31, 0x34, 0x33, 0x33, 0x33, 0x34, 0x35,
1163	0x34, 0x37, 0x37, 0x36, 0x32, 0x34, 0x40400242,
1164	0x32, 0x35, 0x31, 0x38, 0x39, 0x38, 0x33,
1165	0x35, 0x36, 0x39, 0x34, 0x38, 0x35, 0x35,
1166	0x36, 0x32, 0x30, 0x39, 0x39, 0x32, 0x31,
1167	0x39, 0x32, 0x32, 0x32, 0x31, 0x38, 0x34,
1168	0x32, 0x37, 0x4040023e, 0x32, 0x404000ba, 0x36, 0x38,
1169	0x38, 0x37, 0x36, 0x37, 0x31, 0x37, 0x39,
1170	0x30, 0x40400055, 0x30, 0x40800106, 0x36, 0x36, 0x404003e7,
1171	0x38, 0x38, 0x36, 0x32, 0x37, 0x32, 0x404006dc,
1172	0x31, 0x37, 0x38, 0x36, 0x30, 0x38, 0x35,
1173	0x37, 0x40400073, 0x33, 0x408002fc, 0x37, 0x39, 0x37,
1174	0x36, 0x36, 0x38, 0x31, 0x404002bd, 0x30, 0x30,
1175	0x39, 0x35, 0x33, 0x38, 0x38, 0x40400638, 0x33,
1176	0x404006a5, 0x30, 0x36, 0x38, 0x30, 0x30, 0x36,
1177	0x34, 0x32, 0x32, 0x35, 0x31, 0x32, 0x35,
1178	0x32, 0x4040057b, 0x37, 0x33, 0x39, 0x32, 0x40400297,
1179	0x40400474, 0x34, 0x408006b3, 0x38, 0x36, 0x32, 0x36,
1180	0x39, 0x34, 0x35, 0x404001e5, 0x34, 0x31, 0x39,
1181	0x36, 0x35, 0x32, 0x38, 0x35, 0x30, 0x40400099,
1182	0x4040039c, 0x31, 0x38, 0x36, 0x33, 0x404001be, 0x34,
1183	0x40800154, 0x32, 0x30, 0x33, 0x39, 0x4040058b, 0x34,
1184	0x35, 0x404002bc, 0x32, 0x33, 0x37, 0x4040042c, 0x36,
1185	0x40400510, 0x35, 0x36, 0x40400638, 0x37, 0x31, 0x39,
1186	0x31, 0x37, 0x32, 0x38, 0x40400171, 0x37, 0x36,
1187	0x34, 0x36, 0x35, 0x37, 0x35, 0x37, 0x33,
1188	0x39, 0x40400101, 0x33, 0x38, 0x39, 0x40400748, 0x38,
1189	0x33, 0x32, 0x36, 0x34, 0x35, 0x39, 0x39,
1190	0x35, 0x38, 0x404006a7, 0x30, 0x34, 0x37, 0x38,
1191	0x404001de, 0x40400328, 0x39, 0x4040002d, 0x36, 0x34, 0x30,
1192	0x37, 0x38, 0x39, 0x35, 0x31, 0x4040008e, 0x36,
1193	0x38, 0x33, 0x4040012f, 0x32, 0x35, 0x39, 0x35,
1194	0x37, 0x30, 0x40400468, 0x38, 0x32, 0x32, 0x404002c8,
1195	0x32, 0x4040061b, 0x34, 0x30, 0x37, 0x37, 0x32,
1196	0x36, 0x37, 0x31, 0x39, 0x34, 0x37, 0x38,
1197	0x40400319, 0x38, 0x32, 0x36, 0x30, 0x31, 0x34,
1198	0x37, 0x36, 0x39, 0x39, 0x30, 0x39, 0x404004e8,
1199	0x30, 0x31, 0x33, 0x36, 0x33, 0x39, 0x34,
1200	0x34, 0x33, 0x4040027f, 0x33, 0x30, 0x40400105, 0x32,
1201	0x30, 0x33, 0x34, 0x39, 0x36, 0x32, 0x35,
1202	0x32, 0x34, 0x35, 0x31, 0x37, 0x404003b5, 0x39,
1203	0x36, 0x35, 0x31, 0x34, 0x33, 0x31, 0x34,
1204	0x32, 0x39, 0x38, 0x30, 0x39, 0x31, 0x39,
1205	0x30, 0x36, 0x35, 0x39, 0x32, 0x40400282, 0x37,
1206	0x32, 0x32, 0x31, 0x36, 0x39, 0x36, 0x34,
1207	0x36, 0x40400419, 0x4040007a, 0x35, 0x4040050e, 0x34, 0x40800565,
1208	0x38, 0x40400559, 0x39, 0x37, 0x4040057b, 0x35, 0x34,
1209	0x4040049d, 0x4040023e, 0x37, 0x4040065a, 0x38, 0x34, 0x36,
1210	0x38, 0x31, 0x33, 0x4040008c, 0x36, 0x38, 0x33,
1211	0x38, 0x36, 0x38, 0x39, 0x34, 0x32, 0x37,
1212	0x37, 0x34, 0x31, 0x35, 0x35, 0x39, 0x39,
1213	0x31, 0x38, 0x35, 0x4040005a, 0x32, 0x34, 0x35,
1214	0x39, 0x35, 0x33, 0x39, 0x35, 0x39, 0x34,
1215	0x33, 0x31, 0x404005b7, 0x37, 0x40400012, 0x36, 0x38,
1216	0x30, 0x38, 0x34, 0x35, 0x404002e7, 0x37, 0x33,
1217	0x4040081e, 0x39, 0x35, 0x38, 0x34, 0x38, 0x36,
1218	0x35, 0x33, 0x38, 0x404006e8, 0x36, 0x32, 0x404000f2,
1219	0x36, 0x30, 0x39, 0x404004b6, 0x36, 0x30, 0x38,
1220	0x30, 0x35, 0x31, 0x32, 0x34, 0x33, 0x38,
1221	0x38, 0x34, 0x4040013a, 0x4040000b, 0x34, 0x31, 0x33,
1222	0x4040030f, 0x37, 0x36, 0x32, 0x37, 0x38, 0x40400341,
1223	0x37, 0x31, 0x35, 0x4040059b, 0x33, 0x35, 0x39,
1224	0x39, 0x37, 0x37, 0x30, 0x30, 0x31, 0x32,
1225	0x39, 0x40400472, 0x38, 0x39, 0x34, 0x34, 0x31,
1226	0x40400277, 0x36, 0x38, 0x35, 0x35, 0x4040005f, 0x34,
1227	0x30, 0x36, 0x33, 0x404008e6, 0x32, 0x30, 0x37,
1228	0x32, 0x32, 0x40400158, 0x40800203, 0x34, 0x38, 0x31,
1229	0x35, 0x38, 0x40400205, 0x404001fe, 0x4040027a, 0x40400298, 0x33,
1230	0x39, 0x34, 0x35, 0x32, 0x32, 0x36, 0x37,
1231	0x40c00496, 0x38, 0x4040058a, 0x32, 0x31, 0x404002ea, 0x32,
1232	0x40400387, 0x35, 0x34, 0x36, 0x36, 0x36, 0x4040051b,
1233	0x32, 0x33, 0x39, 0x38, 0x36, 0x34, 0x35,
1234	0x36, 0x404004c4, 0x31, 0x36, 0x33, 0x35, 0x40800253,
1235	0x40400811, 0x37, 0x404008ad, 0x39, 0x38, 0x4040045e, 0x39,
1236	0x33, 0x36, 0x33, 0x34, 0x4040075b, 0x37, 0x34,
1237	0x33, 0x32, 0x34, 0x4040047b, 0x31, 0x35, 0x30,
1238	0x37, 0x36, 0x404004bb, 0x37, 0x39, 0x34, 0x35,
1239	0x31, 0x30, 0x39, 0x4040003e, 0x30, 0x39, 0x34,
1240	0x30, 0x404006a6, 0x38, 0x38, 0x37, 0x39, 0x37,
1241	0x31, 0x30, 0x38, 0x39, 0x33, 0x404008f0, 0x36,
1242	0x39, 0x31, 0x33, 0x36, 0x38, 0x36, 0x37,
1243	0x32, 0x4040025b, 0x404001fe, 0x35, 0x4040053f, 0x40400468, 0x40400801,
1244	0x31, 0x37, 0x39, 0x32, 0x38, 0x36, 0x38,
1245	0x404008cc, 0x38, 0x37, 0x34, 0x37, 0x4080079e, 0x38,
1246	0x32, 0x34, 0x4040097a, 0x38, 0x4040025b, 0x37, 0x31,
1247	0x34, 0x39, 0x30, 0x39, 0x36, 0x37, 0x35,
1248	0x39, 0x38, 0x404006ef, 0x33, 0x36, 0x35, 0x40400134,
1249	0x38, 0x31, 0x4040005c, 0x40400745, 0x40400936, 0x36, 0x38,
1250	0x32, 0x39, 0x4040057e, 0x38, 0x37, 0x32, 0x32,
1251	0x36, 0x35, 0x38, 0x38, 0x30, 0x40400611, 0x35,
1252	0x40400249, 0x34, 0x32, 0x37, 0x30, 0x34, 0x37,
1253	0x37, 0x35, 0x35, 0x4040081e, 0x33, 0x37, 0x39,
1254	0x36, 0x34, 0x31, 0x34, 0x35, 0x31, 0x35,
1255	0x32, 0x404005fd, 0x32, 0x33, 0x34, 0x33, 0x36,
1256	0x34, 0x35, 0x34, 0x404005de, 0x34, 0x34, 0x34,
1257	0x37, 0x39, 0x35, 0x4040003c, 0x40400523, 0x408008e6, 0x34,
1258	0x31, 0x4040052a, 0x33, 0x40400304, 0x35, 0x32, 0x33,
1259	0x31, 0x40800841, 0x31, 0x36, 0x36, 0x31, 0x404008b2,
1260	0x35, 0x39, 0x36, 0x39, 0x35, 0x33, 0x36,
1261	0x32, 0x33, 0x31, 0x34, 0x404005ff, 0x32, 0x34,
1262	0x38, 0x34, 0x39, 0x33, 0x37, 0x31, 0x38,
1263	0x37, 0x31, 0x31, 0x30, 0x31, 0x34, 0x35,
1264	0x37, 0x36, 0x35, 0x34, 0x40400761, 0x30, 0x32,
1265	0x37, 0x39, 0x39, 0x33, 0x34, 0x34, 0x30,
1266	0x33, 0x37, 0x34, 0x32, 0x30, 0x30, 0x37,
1267	0x4040093f, 0x37, 0x38, 0x35, 0x33, 0x39, 0x30,
1268	0x36, 0x32, 0x31, 0x39, 0x40800299, 0x40400345, 0x38,
1269	0x34, 0x37, 0x408003d2, 0x38, 0x33, 0x33, 0x32,
1270	0x31, 0x34, 0x34, 0x35, 0x37, 0x31, 0x40400284,
1271	0x40400776, 0x34, 0x33, 0x35, 0x30, 0x40400928, 0x40400468,
1272	0x35, 0x33, 0x31, 0x39, 0x31, 0x30, 0x34,
1273	0x38, 0x34, 0x38, 0x31, 0x30, 0x30, 0x35,
1274	0x33, 0x37, 0x30, 0x36, 0x404008bc, 0x4080059d, 0x40800781,
1275	0x31, 0x40400559, 0x37, 0x4040031b, 0x35, 0x404007ec, 0x4040040c,
1276	0x36, 0x33, 0x408007dc, 0x34, 0x40400971, 0x4080034e, 0x408003f5,
1277	0x38, 0x4080052d, 0x40800887, 0x39, 0x40400187, 0x39, 0x31,
1278	0x404008ce, 0x38, 0x31, 0x34, 0x36, 0x37, 0x35,
1279	0x31, 0x4040062b, 0x31, 0x32, 0x33, 0x39, 0x40c001a9,
1280	0x39, 0x30, 0x37, 0x31, 0x38, 0x36, 0x34,
1281	0x39, 0x34, 0x32, 0x33, 0x31, 0x39, 0x36,
1282	0x31, 0x35, 0x36, 0x404001ec, 0x404006bc, 0x39, 0x35,
1283	0x40400926, 0x40400469, 0x4040011b, 0x36, 0x30, 0x33, 0x38,
1284	0x40400a25, 0x4040016f, 0x40400384, 0x36, 0x32, 0x4040045a, 0x35,
1285	0x4040084c, 0x36, 0x33, 0x38, 0x39, 0x33, 0x37,
1286	0x37, 0x38, 0x37, 0x404008c5, 0x404000f8, 0x39, 0x37,
1287	0x39, 0x32, 0x30, 0x37, 0x37, 0x33, 0x404005d7,
1288	0x32, 0x31, 0x38, 0x32, 0x35, 0x36, 0x404007df,
1289	0x36, 0x36, 0x404006d6, 0x34, 0x32, 0x4080067e, 0x36,
1290	0x404006e6, 0x34, 0x34, 0x40400024, 0x35, 0x34, 0x39,
1291	0x32, 0x30, 0x32, 0x36, 0x30, 0x35, 0x40400ab3,
1292	0x408003e4, 0x32, 0x30, 0x31, 0x34, 0x39, 0x404004d2,
1293	0x38, 0x35, 0x30, 0x37, 0x33, 0x40400599, 0x36,
1294	0x36, 0x36, 0x30, 0x40400194, 0x32, 0x34, 0x33,
1295	0x34, 0x30, 0x40400087, 0x30, 0x4040076b, 0x38, 0x36,
1296	0x33, 0x40400956, 0x404007e4, 0x4040042b, 0x40400174, 0x35, 0x37,
1297	0x39, 0x36, 0x32, 0x36, 0x38, 0x35, 0x36,
1298	0x40400140, 0x35, 0x30, 0x38, 0x40400523, 0x35, 0x38,
1299	0x37, 0x39, 0x36, 0x39, 0x39, 0x40400711, 0x35,
1300	0x37, 0x34, 0x40400a18, 0x38, 0x34, 0x30, 0x404008b3,
1301	0x31, 0x34, 0x35, 0x39, 0x31, 0x4040078c, 0x37,
1302	0x30, 0x40400234, 0x30, 0x31, 0x40400be7, 0x31, 0x32,
1303	0x40400c74, 0x30, 0x404003c3, 0x33, 0x39, 0x40400b2a, 0x40400112,
1304	0x37, 0x31, 0x35, 0x404003b0, 0x34, 0x32, 0x30,
1305	0x40800bf2, 0x39, 0x40400bc2, 0x30, 0x37, 0x40400341, 0x40400795,
1306	0x40400aaf, 0x40400c62, 0x32, 0x31, 0x40400960, 0x32, 0x35,
1307	0x31, 0x4040057b, 0x40400944, 0x39, 0x32, 0x404001b2, 0x38,
1308	0x32, 0x36, 0x40400b66, 0x32, 0x40400278, 0x33, 0x32,
1309	0x31, 0x35, 0x37, 0x39, 0x31, 0x39, 0x38,
1310	0x34, 0x31, 0x34, 0x4080087b, 0x39, 0x31, 0x36,
1311	0x34, 0x408006e8, 0x39, 0x40800b58, 0x404008db, 0x37, 0x32,
1312	0x32, 0x40400321, 0x35, 0x404008a4, 0x40400141, 0x39, 0x31,
1313	0x30, 0x404000bc, 0x40400c5b, 0x35, 0x32, 0x38, 0x30,
1314	0x31, 0x37, 0x40400231, 0x37, 0x31, 0x32, 0x40400914,
1315	0x38, 0x33, 0x32, 0x40400373, 0x31, 0x40400589, 0x30,
1316	0x39, 0x33, 0x35, 0x33, 0x39, 0x36, 0x35,
1317	0x37, 0x4040064b, 0x31, 0x30, 0x38, 0x33, 0x40400069,
1318	0x35, 0x31, 0x4040077a, 0x40400d5a, 0x31, 0x34, 0x34,
1319	0x34, 0x32, 0x31, 0x30, 0x30, 0x40400202, 0x30,
1320	0x33, 0x4040019c, 0x31, 0x31, 0x30, 0x33, 0x40400c81,
1321	0x40400009, 0x40400026, 0x40c00602, 0x35, 0x31, 0x36, 0x404005d9,
1322	0x40800883, 0x4040092a, 0x35, 0x40800c42, 0x38, 0x35, 0x31,
1323	0x37, 0x31, 0x34, 0x33, 0x37, 0x40400605, 0x4040006d,
1324	0x31, 0x35, 0x35, 0x36, 0x35, 0x30, 0x38,
1325	0x38, 0x404003b9, 0x39, 0x38, 0x39, 0x38, 0x35,
1326	0x39, 0x39, 0x38, 0x32, 0x33, 0x38, 0x404001cf,
1327	0x404009ba, 0x33, 0x4040016c, 0x4040043e, 0x404009c3, 0x38, 0x40800e05,
1328	0x33, 0x32, 0x40400107, 0x35, 0x40400305, 0x33, 0x404001ca,
1329	0x39, 0x4040041b, 0x39, 0x38, 0x4040087d, 0x34, 0x40400cb8,
1330	0x37, 0x4040064b, 0x30, 0x37, 0x404000e5, 0x34, 0x38,
1331	0x31, 0x34, 0x31, 0x40400539, 0x38, 0x35, 0x39,
1332	0x34, 0x36, 0x31, 0x40400bc9, 0x38, 0x30,
1333	},
1334	},
1335	HuffTest{
1336	.input = "huffman-rand-1k.input",
1337	.want = "huffman-rand-1k.{s}.expect",
1338	.want_no_input = "huffman-rand-1k.{s}.expect-noinput",
1339	.tokens = &[_]token.Token{
1340	0xf8, 0x8b, 0x96, 0x76, 0x48, 0xd, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d,
1341	0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b,
1342	0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4,
1343	0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde,
1344	0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14,
1345	0x13, 0x6f, 0x23, 0xa9, 0xd, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c,
1346	0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5,
1347	0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20,
1348	0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4,
1349	0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28,
1350	0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0,
1351	0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda,
1352	0xea, 0x9b, 0xa, 0x64, 0xb, 0xe0, 0x23, 0x29, 0xbd, 0xf7, 0xe7, 0x83, 0x3c, 0xfb,
1353	0xdf, 0xb3, 0xae, 0x4f, 0xa4, 0x47, 0x55, 0x99, 0xde, 0x2f, 0x96, 0x6e, 0x1c, 0x43,
1354	0x4c, 0x87, 0xe2, 0x7c, 0xd9, 0x5f, 0x4c, 0x7c, 0xe8, 0x90, 0x3, 0xdb, 0x30, 0x95,
1355	0xd6, 0x22, 0xc, 0x47, 0xb8, 0x4d, 0x6b, 0xbd, 0x24, 0x11, 0xab, 0x2c, 0xd7, 0xbe,
1356	0x6e, 0x7a, 0xd6, 0x8, 0xa3, 0x98, 0xd8, 0xdd, 0x15, 0x6a, 0xfa, 0x93, 0x30, 0x1,
1357	0x25, 0x1d, 0xa2, 0x74, 0x86, 0x4b, 0x6a, 0x95, 0xe8, 0xe1, 0x4e, 0xe, 0x76, 0xb9,
1358	0x49, 0xa9, 0x5f, 0xa0, 0xa6, 0x63, 0x3c, 0x7e, 0x7e, 0x20, 0x13, 0x4f, 0xbb, 0x66,
1359	0x92, 0xb8, 0x2e, 0xa4, 0xfa, 0x48, 0xcb, 0xae, 0xb9, 0x3c, 0xaf, 0xd3, 0x1f, 0xe1,
1360	0xd5, 0x8d, 0x42, 0x6d, 0xf0, 0xfc, 0x8c, 0xc, 0x0, 0xde, 0x40, 0xab, 0x8b, 0x47,
1361	0x97, 0x4e, 0xa8, 0xcf, 0x8e, 0xdb, 0xa6, 0x8b, 0x20, 0x9, 0x84, 0x7a, 0x66, 0xe5,
1362	0x98, 0x29, 0x2, 0x95, 0xe6, 0x38, 0x32, 0x60, 0x3, 0xe3, 0x9a, 0x1e, 0x54, 0xe8,
1363	0x63, 0x80, 0x48, 0x9c, 0xe7, 0x63, 0x33, 0x6e, 0xa0, 0x65, 0x83, 0xfa, 0xc6, 0xba,
1364	0x7a, 0x43, 0x71, 0x5, 0xf5, 0x68, 0x69, 0x85, 0x9c, 0xba, 0x45, 0xcd, 0x6b, 0xb,
1365	0x19, 0xd1, 0xbb, 0x7f, 0x70, 0x85, 0x92, 0xd1, 0xb4, 0x64, 0x82, 0xb1, 0xe4, 0x62,
1366	0xc5, 0x3c, 0x46, 0x1f, 0x92, 0x31, 0x1c, 0x4e, 0x41, 0x77, 0xf7, 0xe7, 0x87, 0xa2,
1367	0xf, 0x6e, 0xe8, 0x92, 0x3, 0x6b, 0xa, 0xe7, 0xa9, 0x3b, 0x11, 0xda, 0x66, 0x8a,
1368	0x29, 0xda, 0x79, 0xe1, 0x64, 0x8d, 0xe3, 0x54, 0xd4, 0xf5, 0xef, 0x64, 0x87, 0x3b,
1369	0xf4, 0xc2, 0xf4, 0x71, 0x13, 0xa9, 0xe9, 0xe0, 0xa2, 0x6, 0x14, 0xab, 0x5d, 0xa7,
1370	0x96, 0x0, 0xd6, 0xc3, 0xcc, 0x57, 0xed, 0x39, 0x6a, 0x25, 0xcd, 0x76, 0xea, 0xba,
1371	0x3a, 0xf2, 0xa1, 0x95, 0x5d, 0xe5, 0x71, 0xcf, 0x9c, 0x62, 0x9e, 0x6a, 0xfa, 0xd5,
1372	0x31, 0xd1, 0xa8, 0x66, 0x30, 0x33, 0xaa, 0x51, 0x17, 0x13, 0x82, 0x99, 0xc8, 0x14,
1373	0x60, 0x9f, 0x4d, 0x32, 0x6d, 0xda, 0x19, 0x26, 0x21, 0xdc, 0x7e, 0x2e, 0x25, 0x67,
1374	0x72, 0xca, 0xf, 0x92, 0xcd, 0xf6, 0xd6, 0xcb, 0x97, 0x8a, 0x33, 0x58, 0x73, 0x70,
1375	0x91, 0x1d, 0xbf, 0x28, 0x23, 0xa3, 0xc, 0xf1, 0x83, 0xc3, 0xc8, 0x56, 0x77, 0x68,
1376	0xe3, 0x82, 0xba, 0xb9, 0x57, 0x56, 0x57, 0x9c, 0xc3, 0xd6, 0x14, 0x5, 0x3c, 0xb1,
1377	0xaf, 0x93, 0xc8, 0x8a, 0x57, 0x7f, 0x53, 0xfa, 0x2f, 0xaa, 0x6e, 0x66, 0x83, 0xfa,
1378	0x33, 0xd1, 0x21, 0xab, 0x1b, 0x71, 0xb4, 0x7c, 0xda, 0xfd, 0xfb, 0x7f, 0x20, 0xab,
1379	0x5e, 0xd5, 0xca, 0xfd, 0xdd, 0xe0, 0xee, 0xda, 0xba, 0xa8, 0x27, 0x99, 0x97, 0x69,
1380	0xc1, 0x3c, 0x82, 0x8c, 0xa, 0x5c, 0x2d, 0x5b, 0x88, 0x3e, 0x34, 0x35, 0x86, 0x37,
1381	0x46, 0x79, 0xe1, 0xaa, 0x19, 0xfb, 0xaa, 0xde, 0x15, 0x9, 0xd, 0x1a, 0x57, 0xff,
1382	0xb5, 0xf, 0xf3, 0x2b, 0x5a, 0x6a, 0x4d, 0x19, 0x77, 0x71, 0x45, 0xdf, 0x4f, 0xb3,
1383	0xec, 0xf1, 0xeb, 0x18, 0x53, 0x3e, 0x3b, 0x47, 0x8, 0x9a, 0x73, 0xa0, 0x5c, 0x8c,
1384	0x5f, 0xeb, 0xf, 0x3a, 0xc2, 0x43, 0x67, 0xb4, 0x66, 0x67, 0x80, 0x58, 0xe, 0xc1,
1385	0xec, 0x40, 0xd4, 0x22, 0x94, 0xca, 0xf9, 0xe8, 0x92, 0xe4, 0x69, 0x38, 0xbe, 0x67,
1386	0x64, 0xca, 0x50, 0xc7, 0x6, 0x67, 0x42, 0x6e, 0xa3, 0xf0, 0xb7, 0x6c, 0xf2, 0xe8,
1387	0x5f, 0xb1, 0xaf, 0xe7, 0xdb, 0xbb, 0x77, 0xb5, 0xf8, 0xcb, 0x8, 0xc4, 0x75, 0x7e,
1388	0xc0, 0xf9, 0x1c, 0x7f, 0x3c, 0x89, 0x2f, 0xd2, 0x58, 0x3a, 0xe2, 0xf8, 0x91, 0xb6,
1389	0x7b, 0x24, 0x27, 0xe9, 0xae, 0x84, 0x8b, 0xde, 0x74, 0xac, 0xfd, 0xd9, 0xb7, 0x69,
1390	0x2a, 0xec, 0x32, 0x6f, 0xf0, 0x92, 0x84, 0xf1, 0x40, 0xc, 0x8a, 0xbc, 0x39, 0x6e,
1391	0x2e, 0x73, 0xd4, 0x6e, 0x8a, 0x74, 0x2a, 0xdc, 0x60, 0x1f, 0xa3, 0x7, 0xde, 0x75,
1392	0x8b, 0x74, 0xc8, 0xfe, 0x63, 0x75, 0xf6, 0x3d, 0x63, 0xac, 0x33, 0x89, 0xc3, 0xf0,
1393	0xf8, 0x2d, 0x6b, 0xb4, 0x9e, 0x74, 0x8b, 0x5c, 0x33, 0xb4, 0xca, 0xa8, 0xe4, 0x99,
1394	0xb6, 0x90, 0xa1, 0xef, 0xf, 0xd3, 0x61, 0xb2, 0xc6, 0x1a, 0x94, 0x7c, 0x44, 0x55,
1395	0xf4, 0x45, 0xff, 0x9e, 0xa5, 0x5a, 0xc6, 0xa0, 0xe8, 0x2a, 0xc1, 0x8d, 0x6f, 0x34,
1396	0x11, 0xb9, 0xbe, 0x4e, 0xd9, 0x87, 0x97, 0x73, 0xcf, 0x3d, 0x23, 0xae, 0xd5, 0x1a,
1397	0x5e, 0xae, 0x5d, 0x6a, 0x3, 0xf9, 0x22, 0xd, 0x10, 0xd9, 0x47, 0x69, 0x15, 0x3f,
1398	0xee, 0x52, 0xa3, 0x8, 0xd2, 0x3c, 0x51, 0xf4, 0xf8, 0x9d, 0xe4, 0x98, 0x89, 0xc8,
1399	0x67, 0x39, 0xd5, 0x5e, 0x35, 0x78, 0x27, 0xe8, 0x3c, 0x80, 0xae, 0x79, 0x71, 0xd2,
1400	0x93, 0xf4, 0xaa, 0x51, 0x12, 0x1c, 0x4b, 0x1b, 0xe5, 0x6e, 0x15, 0x6f, 0xe4, 0xbb,
1401	0x51, 0x9b, 0x45, 0x9f, 0xf9, 0xc4, 0x8c, 0x2a, 0xfb, 0x1a, 0xdf, 0x55, 0xd3, 0x48,
1402	0x93, 0x27, 0x1, 0x26, 0xc2, 0x6b, 0x55, 0x6d, 0xa2, 0xfb, 0x84, 0x8b, 0xc9, 0x9e,
1403	0x28, 0xc2, 0xef, 0x1a, 0x24, 0xec, 0x9b, 0xae, 0xbd, 0x60, 0xe9, 0x15, 0x35, 0xee,
1404	0x42, 0xa4, 0x33, 0x5b, 0xfa, 0xf, 0xb6, 0xf7, 0x1, 0xa6, 0x2, 0x4c, 0xca, 0x90,
1405	0x58, 0x3a, 0x96, 0x41, 0xe7, 0xcb, 0x9, 0x8c, 0xdb, 0x85, 0x4d, 0xa8, 0x89, 0xf3,
1406	0xb5, 0x8e, 0xfd, 0x75, 0x5b, 0x4f, 0xed, 0xde, 0x3f, 0xeb, 0x38, 0xa3, 0xbe, 0xb0,
1407	0x73, 0xfc, 0xb8, 0x54, 0xf7, 0x4c, 0x30, 0x67, 0x2e, 0x38, 0xa2, 0x54, 0x18, 0xba,
1408	0x8, 0xbf, 0xf2, 0x39, 0xd5, 0xfe, 0xa5, 0x41, 0xc6, 0x66, 0x66, 0xba, 0x81, 0xef,
1409	0x67, 0xe4, 0xe6, 0x3c, 0xc, 0xca, 0xa4, 0xa, 0x79, 0xb3, 0x57, 0x8b, 0x8a, 0x75,
1410	0x98, 0x18, 0x42, 0x2f, 0x29, 0xa3, 0x82, 0xef, 0x9f, 0x86, 0x6, 0x23, 0xe1, 0x75,
1411	0xfa, 0x8, 0xb1, 0xde, 0x17, 0x4a,
1412	},
1413	},
1414	HuffTest{
1415	.input = "huffman-rand-limit.input",
1416	.want = "huffman-rand-limit.{s}.expect",
1417	.want_no_input = "huffman-rand-limit.{s}.expect-noinput",
1418	.tokens = &[_]token.Token{
1419	0x61, 0x51c00000, 0xa, 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xa, 0x85, 0x94, 0x25, 0x80,
1420	0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde,
1421	0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73,
1422	0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15,
1423	0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9,
1424	0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xa,
1425	0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70,
1426	0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59,
1427	0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46,
1428	0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b,
1429	0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28,
1430	0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f,
1431	0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4,
1432	0x75, 0xda, 0xea, 0x9b, 0xa,
1433	},
1434	},
1435	HuffTest{
1436	.input = "huffman-shifts.input",
1437	.want = "huffman-shifts.{s}.expect",
1438	.want_no_input = "huffman-shifts.{s}.expect-noinput",
1439	.tokens = &[_]token.Token{
1440	0x31, 0x30, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1441	0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1442	0x7fc00001, 0x7fc00001, 0x7fc00001, 0x52400001, 0xd, 0xa, 0x32,
1443	0x33, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1444	0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7f400001,
1445	},
1446	},
1447	HuffTest{
1448	.input = "huffman-text-shift.input",
1449	.want = "huffman-text-shift.{s}.expect",
1450	.want_no_input = "huffman-text-shift.{s}.expect-noinput",
1451	.tokens = &[_]token.Token{
1452	0x2f, 0x2f, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68,
1453	0x74, 0x32, 0x30, 0x30, 0x39, 0x54, 0x68, 0x47, 0x6f, 0x41,
1454	0x75, 0x74, 0x68, 0x6f, 0x72, 0x2e, 0x41, 0x6c, 0x6c, 0x40800016,
1455	0x72, 0x72, 0x76, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x55,
1456	0x6f, 0x66, 0x74, 0x68, 0x69, 0x6f, 0x75, 0x72, 0x63, 0x63,
1457	0x6f, 0x64, 0x69, 0x67, 0x6f, 0x76, 0x72, 0x6e, 0x64, 0x62,
1458	0x79, 0x42, 0x53, 0x44, 0x2d, 0x74, 0x79, 0x6c, 0x40400020, 0x6c,
1459	0x69, 0x63, 0x6e, 0x74, 0x68, 0x74, 0x63, 0x6e, 0x62, 0x66,
1460	0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x74, 0x68, 0x4c, 0x49,
1461	0x43, 0x45, 0x4e, 0x53, 0x45, 0x66, 0x69, 0x6c, 0x2e, 0xd,
1462	0xa, 0xd, 0xa, 0x70, 0x63, 0x6b, 0x67, 0x6d, 0x69, 0x6e,
1463	0x4040000a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x6f, 0x22,
1464	0x4040000c, 0x66, 0x75, 0x6e, 0x63, 0x6d, 0x69, 0x6e, 0x28, 0x29,
1465	0x7b, 0xd, 0xa, 0x9, 0x76, 0x72, 0x62, 0x3d, 0x6d, 0x6b,
1466	0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x2c, 0x36, 0x35, 0x35,
1467	0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x5f, 0x3a,
1468	0x3d, 0x6f, 0x2e, 0x43, 0x72, 0x74, 0x28, 0x22, 0x68, 0x75,
1469	0x66, 0x66, 0x6d, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d,
1470	0x6d, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x40800021, 0x2e, 0x57, 0x72,
1471	0x69, 0x74, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa,
1472	0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a,
1473	0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54,
1474	0x55, 0x56, 0x58, 0x78, 0x79, 0x7a, 0x21, 0x22, 0x23, 0xc2,
1475	0xa4, 0x25, 0x26, 0x2f, 0x3f, 0x22,
1476	},
1477	},
1478	HuffTest{
1479	.input = "huffman-text.input",
1480	.want = "huffman-text.{s}.expect",
1481	.want_no_input = "huffman-text.{s}.expect-noinput",
1482	.tokens = &[_]token.Token{
1483	0x2f, 0x2f, 0x20, 0x7a, 0x69, 0x67, 0x20, 0x76,
1484	0x30, 0x2e, 0x31, 0x30, 0x2e, 0x30, 0x0a, 0x2f,
1485	0x2f, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65,
1486	0x20, 0x61, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x40400004,
1487	0x6c, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68,
1488	0x20, 0x30, 0x78, 0x30, 0x30, 0x0a, 0x63, 0x6f,
1489	0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, 0x64, 0x20,
1490	0x3d, 0x20, 0x40, 0x69, 0x6d, 0x70, 0x6f, 0x72,
1491	0x74, 0x28, 0x22, 0x73, 0x74, 0x64, 0x22, 0x29,
1492	0x3b, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x20, 0x66,
1493	0x6e, 0x20, 0x6d, 0x61, 0x69, 0x6e, 0x28, 0x29,
1494	0x20, 0x21, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x7b,
1495	0x0a, 0x20, 0x20, 0x20, 0x20, 0x76, 0x61, 0x72,
1496	0x20, 0x62, 0x20, 0x3d, 0x20, 0x5b, 0x31, 0x5d,
1497	0x75, 0x38, 0x7b, 0x30, 0x7d, 0x20, 0x2a, 0x2a,
1498	0x20, 0x36, 0x35, 0x35, 0x33, 0x35, 0x3b, 0x4080001e,
1499	0x40c00055, 0x66, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x79,
1500	0x4040005d, 0x2e, 0x66, 0x73, 0x2e, 0x63, 0x77, 0x64,
1501	0x28, 0x29, 0x2e, 0x40c0008f, 0x46, 0x69, 0x6c, 0x65,
1502	0x28, 0x4080002a, 0x40400000, 0x22, 0x68, 0x75, 0x66, 0x66,
1503	0x6d, 0x61, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c,
1504	0x2d, 0x6d, 0x61, 0x78, 0x2e, 0x69, 0x6e, 0x22,
1505	0x2c, 0x4180001e, 0x2e, 0x7b, 0x20, 0x2e, 0x72, 0x65,
1506	0x61, 0x64, 0x4080004e, 0x75, 0x65, 0x20, 0x7d, 0x40c0001a,
1507	0x29, 0x40c0006b, 0x64, 0x65, 0x66, 0x65, 0x72, 0x20,
1508	0x66, 0x2e, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x28,
1509	0x404000b6, 0x40400015, 0x5f, 0x4100007b, 0x66, 0x2e, 0x77, 0x72,
1510	0x69, 0x74, 0x65, 0x41, 0x6c, 0x6c, 0x28, 0x62,
1511	0x5b, 0x30, 0x2e, 0x2e, 0x5d, 0x29, 0x3b, 0x0a,
1512	0x7d, 0x0a,
1513	},
1514	},
1515	HuffTest{
1516	.input = "huffman-zero.input",
1517	.want = "huffman-zero.{s}.expect",
1518	.want_no_input = "huffman-zero.{s}.expect-noinput",
1519	.tokens = &[_]token.Token{ 0x30, ml, 0x4b800000 },
1520	},
1521	HuffTest{
1522	.input = "",
1523	.want = "",
1524	.want_no_input = "null-long-match.{s}.expect-noinput",
1525	.tokens = &[_]token.Token{
1526	0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1527	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1528	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1529	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1530	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1531	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1532	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1533	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1534	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1535	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1536	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1537	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1538	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1539	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1540	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1541	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1542	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1543	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1544	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1545	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1546	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1547	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1548	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1549	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1550	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1551	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1552	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1553	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1554	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1555	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1556	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1557	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1558	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1559	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1560	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1561	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1562	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1563	ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1564	ml, ml, ml, 0x41400000,
1565	},
1566	},
1567	};
1568
1569	const TestType = enum {
1570	write_block,
1571	write_dyn_block, // write dynamic block
1572	write_huffman_block,
1573
1574	fn to_s(self: TestType) []const u8 {
1575	return switch (self) {
1576	.write_block => "wb",
1577	.write_dyn_block => "dyn",
1578	.write_huffman_block => "huff",
1579	};
1580	}
1581	};
1582
1583	test "writeBlock" {
1584	// tests if the writeBlock encoding has changed.
1585
1586	const ttype: TestType = .write_block;
1587	try testBlock(writeBlockTests[0], ttype);
1588	try testBlock(writeBlockTests[1], ttype);
1589	try testBlock(writeBlockTests[2], ttype);
1590	try testBlock(writeBlockTests[3], ttype);
1591	try testBlock(writeBlockTests[4], ttype);
1592	try testBlock(writeBlockTests[5], ttype);
1593	try testBlock(writeBlockTests[6], ttype);
1594	try testBlock(writeBlockTests[7], ttype);
1595	try testBlock(writeBlockTests[8], ttype);
1596	}
1597
1598	test "writeBlockDynamic" {
1599	// tests if the writeBlockDynamic encoding has changed.
1600
1601	const ttype: TestType = .write_dyn_block;
1602	try testBlock(writeBlockTests[0], ttype);
1603	try testBlock(writeBlockTests[1], ttype);
1604	try testBlock(writeBlockTests[2], ttype);
1605	try testBlock(writeBlockTests[3], ttype);
1606	try testBlock(writeBlockTests[4], ttype);
1607	try testBlock(writeBlockTests[5], ttype);
1608	try testBlock(writeBlockTests[6], ttype);
1609	try testBlock(writeBlockTests[7], ttype);
1610	try testBlock(writeBlockTests[8], ttype);
1611	}
1612
1613	// testBlock tests a block against its references,
1614	// or regenerate the references, if "-update" flag is set.
1615	fn testBlock(comptime ht: HuffTest, ttype: TestType) !void {
1616	// Skip wasi because it does not support std.fs.openDirAbsolute()
1617	if (builtin.os.tag == .wasi) return error.SkipZigTest;
1618
1619	var want_name: []u8 = undefined;
1620	var want_name_no_input: []u8 = undefined;
1621	var input: []u8 = undefined;
1622	var want: []u8 = undefined;
1623	var want_ni: []u8 = undefined; // want no input: what we expect when input is empty
1624
1625	const current_dir = try std.fs.openDirAbsolute(std.fs.path.dirname(@src().file).?, .{});
1626	const testdata_dir = try current_dir.openDir("testdata", .{});
1627
1628	var want_name_type = if (ht.want.len == 0) .{} else .{ttype.to_s()};
1629	want_name = try fmt.allocPrint(testing.allocator, ht.want, want_name_type);
1630	defer testing.allocator.free(want_name);
1631
1632	if (!mem.eql(u8, ht.input, "")) {
1633	const in_file = try testdata_dir.openFile(ht.input, .{ .read = true });
1634	input = try in_file.reader().readAllAlloc(testing.allocator, math.maxInt(usize));
1635	defer testing.allocator.free(input);
1636
1637	const want_file = try testdata_dir.openFile(want_name, .{ .read = true });
1638	want = try want_file.reader().readAllAlloc(testing.allocator, math.maxInt(usize));
1639	defer testing.allocator.free(want);
1640
1641	var buf = ArrayList(u8).init(testing.allocator);
1642	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1643	try writeToType(ttype, &bw, ht.tokens, input);
1644
1645	var got = buf.items;
1646	try expect(mem.eql(u8, got, want)); // expect writeBlock to yield expected result
1647
1648	// Test if the writer produces the same output after reset.
1649	buf.deinit();
1650	buf = ArrayList(u8).init(testing.allocator);
1651	defer buf.deinit();
1652
1653	bw.reset(buf.writer());
1654	defer bw.deinit();
1655
1656	try writeToType(ttype, &bw, ht.tokens, input);
1657	try bw.flush();
1658	got = buf.items;
1659	try expect(mem.eql(u8, got, want)); // expect writeBlock to yield expected result
1660	try testWriterEOF(.write_block, ht.tokens, input);
1661	}
1662
1663	want_name_no_input = try fmt.allocPrint(testing.allocator, ht.want_no_input, .{ttype.to_s()});
1664	defer testing.allocator.free(want_name_no_input);
1665
1666	const want_no_input_file = try testdata_dir.openFile(want_name_no_input, .{ .read = true });
1667	want_ni = try want_no_input_file.reader().readAllAlloc(testing.allocator, math.maxInt(usize));
1668	defer testing.allocator.free(want_ni);
1669
1670	var buf = ArrayList(u8).init(testing.allocator);
1671	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1672
1673	try writeToType(ttype, &bw, ht.tokens, null);
1674
1675	var got = buf.items;
1676	try expect(mem.eql(u8, got, want_ni)); // expect writeBlock to yield expected result
1677	try expect(got[0] & 1 != 1); // expect no EOF
1678
1679	// Test if the writer produces the same output after reset.
1680	buf.deinit();
1681	buf = ArrayList(u8).init(testing.allocator);
1682	defer buf.deinit();
1683
1684	bw.reset(buf.writer());
1685	defer bw.deinit();
1686
1687	try writeToType(ttype, &bw, ht.tokens, null);
1688	try bw.flush();
1689	got = buf.items;
1690
1691	try expect(mem.eql(u8, got, want_ni)); // expect writeBlock to yield expected result
1692	try testWriterEOF(.write_block, ht.tokens, &[0]u8{});
1693	}
1694
1695	fn writeToType(ttype: TestType, bw: anytype, tok: []const token.Token, input: ?[]u8) !void {
1696	switch (ttype) {
1697	.write_block => try bw.writeBlock(tok, false, input),
1698	.write_dyn_block => try bw.writeBlockDynamic(tok, false, input),
1699	else => unreachable,
1700	}
1701	try bw.flush();
1702	}
1703
1704	// Tests if the written block contains an EOF marker.
1705	fn testWriterEOF(ttype: TestType, ht_tokens: []const token.Token, input: []u8) !void {
1706	var buf = ArrayList(u8).init(testing.allocator);
1707	defer buf.deinit();
1708	var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1709	defer bw.deinit();
1710
1711	switch (ttype) {
1712	.write_block => try bw.writeBlock(ht_tokens, true, input),
1713	.write_dyn_block => try bw.writeBlockDynamic(ht_tokens, true, input),
1714	.write_huffman_block => try bw.writeBlockHuff(true, input),
1715	}
1716
1717	try bw.flush();
1718
1719	var b = buf.items;
1720	try expect(b.len > 0);
1721	try expect(b[0] & 1 == 1);
1722	}

lib/std/compress/deflate/huffman_code.zig created+432

...	@@ -0,0 +1,432 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const math = std.math;
4	const mem = std.mem;
5	const sort = std.sort;
6	const testing = std.testing;
7
8	const Allocator = std.mem.Allocator;
9
10	const bu = @import("bits_utils.zig");
11	const deflate_const = @import("deflate_const.zig");
12
13	const max_bits_limit = 16;
14
15	const LiteralNode = struct {
16	literal: u16,
17	freq: u16,
18	};
19
20	// Describes the state of the constructed tree for a given depth.
21	const LevelInfo = struct {
22	// Our level. for better printing
23	level: u32,
24
25	// The frequency of the last node at this level
26	last_freq: u32,
27
28	// The frequency of the next character to add to this level
29	next_char_freq: u32,
30
31	// The frequency of the next pair (from level below) to add to this level.
32	// Only valid if the "needed" value of the next lower level is 0.
33	next_pair_freq: u32,
34
35	// The number of chains remaining to generate for this level before moving
36	// up to the next level
37	needed: u32,
38	};
39
40	// hcode is a huffman code with a bit code and bit length.
41	pub const HuffCode = struct {
42	code: u16 = 0,
43	len: u16 = 0,
44
45	// set sets the code and length of an hcode.
46	fn set(self: *HuffCode, code: u16, length: u16) void {
47	self.len = length;
48	self.code = code;
49	}
50	};
51
52	pub const HuffmanEncoder = struct {
53	codes: []HuffCode,
54	freq_cache: []LiteralNode = undefined,
55	bit_count: [17]u32 = undefined,
56	lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
57	lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
58	allocator: Allocator,
59
60	pub fn deinit(self: *HuffmanEncoder) void {
61	self.allocator.free(self.codes);
62	self.allocator.free(self.freq_cache);
63	}
64
65	// Update this Huffman Code object to be the minimum code for the specified frequency count.
66	//
67	// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
68	// max_bits The maximum number of bits to use for any literal.
69	pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void {
70	var list = self.freq_cache[0 .. freq.len + 1];
71	// Number of non-zero literals
72	var count: u32 = 0;
73	// Set list to be the set of all non-zero literals and their frequencies
74	for (freq) \|f, i\| {
75	if (f != 0) {
76	list[count] = LiteralNode{ .literal = @intCast(u16, i), .freq = f };
77	count += 1;
78	} else {
79	list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
80	self.codes[i].len = 0;
81	}
82	}
83	list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
84
85	list = list[0..count];
86	if (count <= 2) {
87	// Handle the small cases here, because they are awkward for the general case code. With
88	// two or fewer literals, everything has bit length 1.
89	for (list) \|node, i\| {
90	// "list" is in order of increasing literal value.
91	self.codes[node.literal].set(@intCast(u16, i), 1);
92	}
93	return;
94	}
95	self.lfs = list;
96	sort.sort(LiteralNode, self.lfs, {}, byFreq);
97
98	// Get the number of literals for each bit count
99	var bit_count = self.bitCounts(list, max_bits);
100	// And do the assignment
101	self.assignEncodingAndSize(bit_count, list);
102	}
103
104	pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 {
105	var total: u32 = 0;
106	for (freq) \|f, i\| {
107	if (f != 0) {
108	total += @intCast(u32, f) * @intCast(u32, self.codes[i].len);
109	}
110	}
111	return total;
112	}
113
114	// Return the number of literals assigned to each bit size in the Huffman encoding
115	//
116	// This method is only called when list.len >= 3
117	// The cases of 0, 1, and 2 literals are handled by special case code.
118	//
119	// list: An array of the literals with non-zero frequencies
120	// and their associated frequencies. The array is in order of increasing
121	// frequency, and has as its last element a special element with frequency
122	// std.math.maxInt(i32)
123	//
124	// max_bits: The maximum number of bits that should be used to encode any literal.
125	// Must be less than 16.
126	//
127	// Returns an integer array in which array[i] indicates the number of literals
128	// that should be encoded in i bits.
129	fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 {
130	var max_bits = max_bits_to_use;
131	var n = list.len;
132
133	assert(max_bits < max_bits_limit);
134
135	// The tree can't have greater depth than n - 1, no matter what. This
136	// saves a little bit of work in some small cases
137	max_bits = @minimum(max_bits, n - 1);
138
139	// Create information about each of the levels.
140	// A bogus "Level 0" whose sole purpose is so that
141	// level1.prev.needed == 0. This makes level1.next_pair_freq
142	// be a legitimate value that never gets chosen.
143	var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
144	// leaf_counts[i] counts the number of literals at the left
145	// of ancestors of the rightmost node at level i.
146	// leaf_counts[i][j] is the number of literals at the left
147	// of the level j ancestor.
148	var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
149
150	{
151	var level = @as(u32, 1);
152	while (level <= max_bits) : (level += 1) {
153	// For every level, the first two items are the first two characters.
154	// We initialize the levels as if we had already figured this out.
155	levels[level] = LevelInfo{
156	.level = level,
157	.last_freq = list[1].freq,
158	.next_char_freq = list[2].freq,
159	.next_pair_freq = list[0].freq + list[1].freq,
160	.needed = 0,
161	};
162	leaf_counts[level][level] = 2;
163	if (level == 1) {
164	levels[level].next_pair_freq = math.maxInt(i32);
165	}
166	}
167	}
168
169	// We need a total of 2*n - 2 items at top level and have already generated 2.
170	levels[max_bits].needed = 2 * @intCast(u32, n) - 4;
171
172	{
173	var level = max_bits;
174	while (true) {
175	var l = &levels[level];
176	if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
177	// We've run out of both leafs and pairs.
178	// End all calculations for this level.
179	// To make sure we never come back to this level or any lower level,
180	// set next_pair_freq impossibly large.
181	l.needed = 0;
182	levels[level + 1].next_pair_freq = math.maxInt(i32);
183	level += 1;
184	continue;
185	}
186
187	var prev_freq = l.last_freq;
188	if (l.next_char_freq < l.next_pair_freq) {
189	// The next item on this row is a leaf node.
190	var next = leaf_counts[level][level] + 1;
191	l.last_freq = l.next_char_freq;
192	// Lower leaf_counts are the same of the previous node.
193	leaf_counts[level][level] = next;
194	if (next >= list.len) {
195	l.next_char_freq = maxNode().freq;
196	} else {
197	l.next_char_freq = list[next].freq;
198	}
199	} else {
200	// The next item on this row is a pair from the previous row.
201	// next_pair_freq isn't valid until we generate two
202	// more values in the level below
203	l.last_freq = l.next_pair_freq;
204	// Take leaf counts from the lower level, except counts[level] remains the same.
205	mem.copy(u32, leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
206	levels[l.level - 1].needed = 2;
207	}
208
209	l.needed -= 1;
210	if (l.needed == 0) {
211	// We've done everything we need to do for this level.
212	// Continue calculating one level up. Fill in next_pair_freq
213	// of that level with the sum of the two nodes we've just calculated on
214	// this level.
215	if (l.level == max_bits) {
216	// All done!
217	break;
218	}
219	levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
220	level += 1;
221	} else {
222	// If we stole from below, move down temporarily to replenish it.
223	while (levels[level - 1].needed > 0) {
224	level -= 1;
225	if (level == 0) {
226	break;
227	}
228	}
229	}
230	}
231	}
232
233	// Somethings is wrong if at the end, the top level is null or hasn't used
234	// all of the leaves.
235	assert(leaf_counts[max_bits][max_bits] == n);
236
237	var bit_count = self.bit_count[0 .. max_bits + 1];
238	var bits: u32 = 1;
239	var counts = &leaf_counts[max_bits];
240	{
241	var level = max_bits;
242	while (level > 0) : (level -= 1) {
243	// counts[level] gives the number of literals requiring at least "bits"
244	// bits to encode.
245	bit_count[bits] = counts[level] - counts[level - 1];
246	bits += 1;
247	if (level == 0) {
248	break;
249	}
250	}
251	}
252	return bit_count;
253	}
254
255	// Look at the leaves and assign them a bit count and an encoding as specified
256	// in RFC 1951 3.2.2
257	fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void {
258	var code = @as(u16, 0);
259	var list = list_arg;
260
261	for (bit_count) \|bits, n\| {
262	code <<= 1;
263	if (n == 0 or bits == 0) {
264	continue;
265	}
266	// The literals list[list.len-bits] .. list[list.len-bits]
267	// are encoded using "bits" bits, and get the values
268	// code, code + 1, .... The code values are
269	// assigned in literal order (not frequency order).
270	var chunk = list[list.len - @intCast(u32, bits) ..];
271
272	self.lns = chunk;
273	sort.sort(LiteralNode, self.lns, {}, byLiteral);
274
275	for (chunk) \|node\| {
276	self.codes[node.literal] = HuffCode{
277	.code = bu.bitReverse(u16, code, @intCast(u5, n)),
278	.len = @intCast(u16, n),
279	};
280	code += 1;
281	}
282	list = list[0 .. list.len - @intCast(u32, bits)];
283	}
284	}
285	};
286
287	fn maxNode() LiteralNode {
288	return LiteralNode{
289	.literal = math.maxInt(u16),
290	.freq = math.maxInt(u16),
291	};
292	}
293
294	pub fn newHuffmanEncoder(allocator: Allocator, size: u32) !HuffmanEncoder {
295	return HuffmanEncoder{
296	.codes = try allocator.alloc(HuffCode, size),
297	// Allocate a reusable buffer with the longest possible frequency table.
298	// (deflate_const.max_num_frequencies).
299	.freq_cache = try allocator.alloc(LiteralNode, deflate_const.max_num_frequencies + 1),
300	.allocator = allocator,
301	};
302	}
303
304	// Generates a HuffmanCode corresponding to the fixed literal table
305	pub fn generateFixedLiteralEncoding(allocator: Allocator) !HuffmanEncoder {
306	var h = try newHuffmanEncoder(allocator, deflate_const.max_num_frequencies);
307	var codes = h.codes;
308	var ch: u16 = 0;
309
310	while (ch < deflate_const.max_num_frequencies) : (ch += 1) {
311	var bits: u16 = undefined;
312	var size: u16 = undefined;
313	switch (ch) {
314	0...143 => {
315	// size 8, 000110000 .. 10111111
316	bits = ch + 48;
317	size = 8;
318	},
319	144...255 => {
320	// size 9, 110010000 .. 111111111
321	bits = ch + 400 - 144;
322	size = 9;
323	},
324	256...279 => {
325	// size 7, 0000000 .. 0010111
326	bits = ch - 256;
327	size = 7;
328	},
329	else => {
330	// size 8, 11000000 .. 11000111
331	bits = ch + 192 - 280;
332	size = 8;
333	},
334	}
335	codes[ch] = HuffCode{ .code = bu.bitReverse(u16, bits, @intCast(u5, size)), .len = size };
336	}
337	return h;
338	}
339
340	pub fn generateFixedOffsetEncoding(allocator: Allocator) !HuffmanEncoder {
341	var h = try newHuffmanEncoder(allocator, 30);
342	var codes = h.codes;
343	for (codes) \|_, ch\| {
344	codes[ch] = HuffCode{ .code = bu.bitReverse(u16, @intCast(u16, ch), 5), .len = 5 };
345	}
346	return h;
347	}
348
349	fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
350	_ = context;
351	return a.literal < b.literal;
352	}
353
354	fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
355	_ = context;
356	if (a.freq == b.freq) {
357	return a.literal < b.literal;
358	}
359	return a.freq < b.freq;
360	}
361
362	test "generate a Huffman code from an array of frequencies" {
363	var freqs: [19]u16 = [_]u16{
364	8, // 0
365	1, // 1
366	1, // 2
367	2, // 3
368	5, // 4
369	10, // 5
370	9, // 6
371	1, // 7
372	0, // 8
373	0, // 9
374	0, // 10
375	0, // 11
376	0, // 12
377	0, // 13
378	0, // 14
379	0, // 15
380	1, // 16
381	3, // 17
382	5, // 18
383	};
384
385	var enc = try newHuffmanEncoder(testing.allocator, freqs.len);
386	defer enc.deinit();
387	enc.generate(freqs[0..], 7);
388
389	try testing.expect(enc.bitLength(freqs[0..]) == 141);
390
391	try testing.expect(enc.codes[0].len == 3);
392	try testing.expect(enc.codes[1].len == 6);
393	try testing.expect(enc.codes[2].len == 6);
394	try testing.expect(enc.codes[3].len == 5);
395	try testing.expect(enc.codes[4].len == 3);
396	try testing.expect(enc.codes[5].len == 2);
397	try testing.expect(enc.codes[6].len == 2);
398	try testing.expect(enc.codes[7].len == 6);
399	try testing.expect(enc.codes[8].len == 0);
400	try testing.expect(enc.codes[9].len == 0);
401	try testing.expect(enc.codes[10].len == 0);
402	try testing.expect(enc.codes[11].len == 0);
403	try testing.expect(enc.codes[12].len == 0);
404	try testing.expect(enc.codes[13].len == 0);
405	try testing.expect(enc.codes[14].len == 0);
406	try testing.expect(enc.codes[15].len == 0);
407	try testing.expect(enc.codes[16].len == 6);
408	try testing.expect(enc.codes[17].len == 5);
409	try testing.expect(enc.codes[18].len == 3);
410
411	try testing.expect(enc.codes[5].code == 0x0);
412	try testing.expect(enc.codes[6].code == 0x2);
413	try testing.expect(enc.codes[0].code == 0x1);
414	try testing.expect(enc.codes[4].code == 0x5);
415	try testing.expect(enc.codes[18].code == 0x3);
416	try testing.expect(enc.codes[3].code == 0x7);
417	try testing.expect(enc.codes[17].code == 0x17);
418	try testing.expect(enc.codes[1].code == 0x0f);
419	try testing.expect(enc.codes[2].code == 0x2f);
420	try testing.expect(enc.codes[7].code == 0x1f);
421	try testing.expect(enc.codes[16].code == 0x3f);
422	}
423
424	test "generate a Huffman code for the fixed litteral table specific to Deflate" {
425	var enc = try generateFixedLiteralEncoding(testing.allocator);
426	defer enc.deinit();
427	}
428
429	test "generate a Huffman code for the 30 possible relative offsets (LZ77 distances) of Deflate" {
430	var enc = try generateFixedOffsetEncoding(testing.allocator);
431	defer enc.deinit();
432	}

lib/std/compress/deflate/mem_utils.zig created+15

...	@@ -0,0 +1,15 @@
1	const std = @import("std");
2	const math = std.math;
3	const mem = std.mem;
4
5	// Copies elements from a source `src` slice into a destination `dst` slice.
6	// The copy never returns an error but might not be complete if the destination is too small.
7	// Returns the number of elements copied, which will be the minimum of `src.len` and `dst.len`.
8	pub fn copy(dst: []u8, src: []const u8) usize {
9	if (dst.len <= src.len) {
10	mem.copy(u8, dst[0..], src[0..dst.len]);
11	} else {
12	mem.copy(u8, dst[0..src.len], src[0..]);
13	}
14	return math.min(dst.len, src.len);
15	}

lib/std/compress/deflate/testdata/compress-e.txt created+1

...	@@ -0,0 +1 @@
1	2.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788567430285974600143785483237068701190078499404930918919181649327259774030074879681484882342932023012128032327460392219687528340516906974194257614673978110715464186273369091584973185011183960482533518748438923177292613543024932562896371361977285456622924461644497284597867711574125670307871885109336344480149675240618536569532074170533486782754827815415561966911055101472799040386897220465550833170782394808785990501947563108984124144672821865459971596639015641941751820935932616316888380132758752601460507676098392625726411120135288591317848299475682472564885533357279772205543568126302535748216585414000805314820697137262149755576051890481622376790414926742600071045922695314835188137463887104273544767623577933993970632396604969145303273887874557905934937772320142954803345000695256980935282887783710670585567749481373858630385762823040694005665340584887527005308832459182183494318049834199639981458773435863115940570443683515285383609442955964360676090221741896883548131643997437764158365242234642619597390455450680695232850751868719449064767791886720306418630751053512149851051207313846648717547518382979990189317751550639981016466414592102406838294603208535554058147159273220677567669213664081505900806952540610628536408293276621931939933861623836069111767785448236129326858199965239275488427435414402884536455595124735546139403154952097397051896240157976832639450633230452192645049651735466775699295718989690470902730288544945416699791992948038254980285946029052763145580316514066229171223429375806143993484914362107993576737317948964252488813720435579287511385856973381976083524423240466778020948399639946684833774706725483618848273000648319163826022110555221246733323184463005504481849916996622087746140216157021029603318588727333298779352570182393861244026868339555870607758169954398469568540671174444932479519572159419645863736126915526457574786985964242176592896862383506370433939811671397544736228625506803682664135541448048997721373174119199970017293907303350869020922519124447393278376156321810842898207706974138707053266117683698647741787180202729412982310888796831880854367327806879771659111654224453806625861711729498038248879986504061563975629936962809358189761491017145343556659542757064194408833816841111166200759787244137082333917886114708228657531078536674695018462140736493917366254937783014074302668422150335117736471853872324040421037907750266020114814935482228916663640782450166815341213505278578539332606110249802273093636740213515386431693015267460536064351732154701091440650878823636764236831187390937464232609021646365627553976834019482932795750624399645272578624400375983422050808935129023122475970644105678361870877172333555465482598906861201410107222465904008553798235253885171623518256518482203125214950700378300411216212126052726059944320443056274522916128891766814160639131235975350390320077529587392412476451850809163911459296071156344204347133544720981178461451077872399140606290228276664309264900592249810291068759434533858330391178747575977065953570979640012224092199031158229259667913153991561438070129260780197022589662923368154312499412259460023399472228171056603931877226800493833148980338548909468685130789292064242819174795866199944411196208730498064385006852620258432842085582338566936649849720817046135376163584015342840674118587581546514598270228676671855309311923340191286170613364873183197560812569460089402953094429119590295968563923037689976327462283900735457144596414108229285922239332836210192822937243590283003884445701383771632056518351970100115722010956997890484964453434612129224964732356126321951155701565824427661599326463155806672053127596948538057364208384918887095176052287817339462747644656858900936266123311152910816041524100214195937349786431661556732702792109593543055579732660554677963552005378304619540636971842916168582734122217145885870814274090248185446421774876925093328785670674677381226752831653559245204578070541352576903253522738963847495646255940378924925007624386893776475310102323746733771474581625530698032499033676455430305274561512961214585944432150749051491453950981001388737926379964873728396416897555132275962011838248650746985492038097691932606437608743209385602815642849756549307909733854185583515789409814007691892389063090542534883896831762904120212949167195811935791203162514344096503132835216728021372415947344095498316138322505486708172221475138425166790445416617303200820330902895488808516797258495813407132180533988828139346049850532340472595097214331492586604248511405819579711564191458842833000525684776874305916390494306871343118796189637475503362820939949343690321031976898112055595369465424704173323895394046035325396758354395350516720261647961347790912327995264929045151148307923369382166010702872651938143844844532639517394110131152502750465749343063766541866128915264446926222884366299462732467958736383501937142786471398054038215513463223702071533134887083174146591492406359493020921122052610312390682941345696785958518393491382340884274312419099152870804332809132993078936867127413922890033069995875921815297612482409116951587789964090352577345938248232053055567238095022266790439614231852991989181065554412477204508510210071522352342792531266930108270633942321762570076323139159349709946933241013908779161651226804414809765618979735043151396066913258379033748620836695475083280318786707751177525663963479259219733577949555498655214193398170268639987388347010255262052312317215254062571636771270010760912281528326508984359568975961038372157726831170734552250194121701541318793651818502020877326906133592182000762327269503283827391243828198170871168108951187896746707073377869592565542713340052326706040004348843432902760360498027862160749469654989210474443927871934536701798673920803845633723311983855862638008516345597194441994344624761123844617615736242015935078520825600604101556889899501732554337298073561699861101908472096600708320280569917042590103876928658336557728758684250492690370934262028022399861803400211320742198642917383679176232826444645756330336556777374808644109969141827774253417010988435853189339175934511574023847292909015468559163792696196841000676598399744972047287881831200233383298030567865480871476464512824264478216644266616732096012564794514827125671326697067367144617795643752391742928503987022583734069852309190464967260243411270345611114149835783901793499713790913696706497637127248466613279908254305449295528594932793818341607827091326680865655921102733746700132583428715240835661522165574998431236278287106649401564670141943713823863454729606978693335973109537126499416282656463708490580151538205338326511289504938566468752921135932220265681856418260827538790002407915892646028490894922299966167437731347776134150965262448332709343898412056926145108857812249139616912534202918139898683901335795857624435194008943955180554746554000051766240202825944828833811886381749594284892013520090951007864941868256009273977667585642598378587497776669563350170748579027248701370264203283965756348010818356182372177082236423186591595883669487322411726504487268392328453010991677518376831599821263237123854357312681202445175401852132663740538802901249728180895021553100673598184430429105288459323064725590442355960551978839325930339572934663055160430923785677229293537208416693134575284011873746854691620648991164726909428982971065606801805807843600461866223562874591385185904416250663222249561448724413813849763797102676020845531824111963927941069619465426480006761727618115630063644321116224837379105623611358836334550102286170517890440570419577859833348463317921904494652923021469259756566389965893747728751393377105569802455757436190501772466214587592374418657530064998056688376964229825501195065837843125232135309371235243969149662310110328243570065781487677299160941153954063362752423712935549926713485031578238899567545287915578420483105749330060197958207739558522807307048950936235550769837881926357141779338750216344391014187576711938914416277109602859415809719913429313295145924373636456473035037374538503489286113141638094752301745088784885645741275003353303416138096560043105860548355773946625033230034341587814634602169235079216111013148948281895391028916816328709309713184139815427678818067628650978085718262117003140003377301581536334149093237034703637513354537634521050370995452942055232078817449370937677056009306353645510913481627378204985657055608784211964039972344556458607689515569686899384896439195225232309703301037277227710870564912966121061494072782442033414057441446459968236966118878411656290355117839944070961772567164919790168195234523807446299877664824873753313018142763910519234685081979001796519907050490865237442841652776611425351538665162781316090964802801234493372427866930894827913465443931965254154829494577875758599482099181824522449312077768250830768282335001597040419199560509705364696473142448453825888112602753909548852639708652339052941829691802357120545328231809270356491743371932080628731303589640570873779967845174740515317401384878082881006046388936711640477755985481263907504747295012609419990373721246201677030517790352952793168766305099837441859803498821239340919805055103821539827677291373138006715339240126954586376422065097810852907639079727841301764553247527073788764069366420012194745702358295481365781809867944020220280822637957006755393575808086318932075864444206644691649334467698180811716568665213389686173592450920801465312529777966137198695916451869432324246404401672381978020728394418264502183131483366019384891972317817154372192103946638473715630226701801343515930442853848941825678870721238520597263859224934763623122188113706307506918260109689069251417142514218153491532129077723748506635489170892850760234351768218355008829647410655814882049239533702270536705630750317499788187009989251020178015601042277836283644323729779929935160925884515772055232896978333126427671291093993103773425910592303277652667641874842441076564447767097790392324958416348527735171981064673837142742974468992320406932506062834468937543016787815320616009057693404906146176607094380110915443261929000745209895959201159412324102274845482605404361871836330268992858623582145643879695210235266673372434423091577183277565800211928270391042391966426911155333594569685782817020325495552528875464466074620294766116004435551604735044292127916358748473501590215522120388281168021413865865168464569964810015633741255098479730138656275460161279246359783661480163871602794405482710196290774543628092612567507181773641749763254436773503632580004042919906963117397787875081560227368824967077635559869284901628768699628053790181848148810833946900016380791075960745504688912686792812391148880036720729730801354431325347713094186717178607522981373539126772812593958220524289991371690685650421575056729991274177149279608831502358697816190894908487717722503860872618384947939757440664912760518878124233683125467278331513186758915668300679210215947336858591201395360301678110413444411030903388761520488296909104689167671555373346622545575975202624771242796225983278405833585897671474205724047439720232895903726148688388003174146490203843590358527993123871042845981608996101945691646983837718267264685264869172948414153004604004299585035164101899027529366867431834955447458124140190754681607770977920579383895378192128847409929537040546962226547278807248685508046571043123854873351653070570784584243335550958221912862797205455466267099131902370311779690892786623112661337671178512943059323281605826535623848164192144732543731002062738466812351691016359252588256806438946389880872735284406462208149513862275239938938734905082625472417781702582044129853760499827899020083498387362992498125742354568439023012261733665820546785671147973065077035475620567428300187473019197310881157516777005071432012726354601912460800451608108641835539669946936947322271670748972850464195392966434725254724357659192969949061670189061433616907056148280980363243454128229968275980226694045642181328624517549652147221620839824594576613342710564957193564431561774500828376935700995419541839029151033187933907614207467028867968594985439789457300768939890070073924697461812855764662265412913204052279071212820653775058280040897163467163709024906774736309136904002615646432159560910851092445162454420141442641660181385990017417408244245378610158433361777292580611159192008414091888191208858207627011483671760749046980914443057262211104583300789331698191603917150622792986282709446275915009683226345073725451366858172483498470080840163868209726371345205439802277866337293290829914010645589761697455978409211409167684020269370229231743334499986901841510888993165125090001163719114994852024821586396216294981753094623047604832399379391002142532996476235163569009445086058091202459904612118623318278614464727795523218635916551883057930657703331498510068357135624341881884405780028844018129031378653794869614630467726914552953690154167025838032477842272417994513653582260971652588356712133519546838335349801503269359798167463231847628306340588324731228951257944267639877946713121042763380872695738609314631539148548792514028885025189788076023838995615684850391995855029256054176767663145354058496296796781349420116003325874431438746248313850214980401681940795687219268462617287403480967931949965604299190281810597603263251746405016454606266765529010639868703668263299050577706266397868453584384057673298268163448646707439990917504018892319267557518354054956017732907127219134577524905771512773358423314008356080926962298894163047287780054743798498545562870729968407382937218623831766524716090967192007237658894226186550487552614557855898773008703234726418384831040394818743616224455286163287628541175946460497027724490799275146445792982549802258601001772437840167723166802004162547244179415547810554178036773553354467030326469619447560812831933095679685582771932031205941616693902049665352189672822671972640029493307384717544753761937017882976382487233361813499414541694736549254840633793674361541081593464960431603544354737728802361047743115330785159902977771499610274627769759612488879448609863349422852847651310277926279743981957617505591300993377368240510902583759345170015340522266144077237050890044496613295859536020556034009492820943862994618834790932894161098856594954213114335608810239423706087108026465913203560121875933791639666437282836752328391688865373751335794859860107569374889645657187292540448508624449947816273842517229343960137212406286783636675845331904743954740664015260871940915743955282773904303868772728262065663129387459875317749973799293043294371763801856280061141619563942414312254397099163565102848315765427037906837175764870230052388197498746636856292655058222887713221781440489538099681072143012394693530931524054081215705402274414521876541901428386744260011889041724570537470755550581632831687247110220353727166112304857340460879272501694701067831178927095527253222125224361673343366384756590949728221809418684074238351567868893421148203905824224324264643630201441787982022116248471657468291146315407563770222740135841109076078464780070182766336227978104546331131294044833570134869585165267459515187680033395522410548181767867772152798270250117195816577603549732923724732067853690257536233971216884390878879262188202305529937132397194333083536231248870386416194361506529551267334207198502259771408638122015980894363561808597010080081622557455039101321981979045520049618583777721048046635533806616517023595097133203631578945644487800945620369784973459902004606886572701865867757842758530645706617127194967371083950603267501532435909029491516973738110897934782297684100117657987098185725131372267749706609250481876835516003714638685918913011736805218743265426063700710595364425062760458252336880552521181566417553430681181548267844169315284408461087588214317641649835663127518728182948655658524206852221830755306118393326934164459415342651778653397980580828158806300749952897558204686612590853678738603318442905510689778698417735603118111677563872589911516803236547002987989628986181014596471307916144369564690909518788574398821730583884980809523077569358851616027719521488998358632323127308909861560777386006984035267826785387215920936255817889813416247486456433211043194821421299793188104636399541496539441501383868748384870224681829391860319598667962363489309283087840712400431022706137591368056518861313458307990705003607588327248867879324093380071864152853317943535073401891193638546730000660453783784472469288830546979000131248952100446949032058838294923613919284305249167833012980192255157050378521810552961623637523647962685751660066539364142273063001648652613891842243501797455993616794063303522111829071597538821839777552812981538570168702202620274678647916644030729018445497956399844836807851997088201407769199261674991148329821854382718946282165387064858588646221611410343570342878862979083418871606214430014533275029715104673156021000043869510583773779766003460887624861640938645252177935289947578496255243925598620521409052346250847830487046492688313289470553891357290706967599556298586669559721686506052072801342104355762779184021797626656484580261591407173477009039475168017709900129391137881248534255949312866653465033728846390649968460644741907524313323903404908195233044389559060547854954620263256676813262435925020249516275607080900436460421497025691488555265022810327762115842282433269528629137662675481993546118143913367579700141255870143319434764035725376914388899683088262844616425575034001428982557620386364384137906519612917777354183694676232982904981261717676191554292570438432239918482261744350470199171258214687683172646078959690569981353264435973965173473319484798758064137926885413552523275720457329477215706850016950046959758389373527538622664943456437071610511521617176237598050900553232154896062817794302268640579555845730600598376482703339859420098582351400179507104569019191359062304102336798080907240196312675268916362136351032648077232914950859151265812143823371072949148088472355286394195993455684156344577951727033374238129903260198160571971183950662758220321837136059718025940870615534713104482272716848395524105913605919812444978458110854511231668173534838253724825347636777581712867205865148285317273569069839935110763432091319780314031658897379628301178409806410175016511072932907832177487566289310650383806093372841399226733384778203302020700517188941706465146238366720632742644336612174011766914919235570905644803016342294301837655263108450172510307540942604409687066288066265900569082451407632599158164499361455172452057020443093722305550217222299706209749268609762787409626448772056043078634808885709143464793241536214303199965695610753570417207285334250171325558818113295504095217830139465216436594262960768570585698507157151317262928960072587601564840556088613165411835958628710665496282599535127193244635791046554389165150954187306071015034430609582302257455974944275067630926322529966338219395202927917973247094559691016402983683080426309910481567503623509654924302589575273521412445149542462972258510120707802110188106722347972579330653187713438466713807546383471635428854957610942841898601794658721444495198801550804042506452191484989920400007310672369944655246020908767882300064337725657385010969899058191290957079866699453765080407917852438222041070599278889267745752084287526377986730360561230710723922581504781379172731261234878334034473833573601973235946604273704635201327182592410906040097638585857716958419563109577748529579836844756803121874818202833941887076311731615289811756429711334181497218078040465077657204457082859417475114926179367379999220181789399433337731146911970737861041963986422166045588965683206701337505745038872111332436739840284188639147633491695114032583475841514170325690161784931455706904169858050217798497637014758914810543205854914100662201721719726878930012101267481270235940855162601689425111458499658315589660460091525797881670384625905383256920520425791378948827579603278877535466861441826827797651258953563761485994485049706638406266121957141911063246061774180577212381659872472432252969098533628440799030007594546281549235506086481557928961969617060715201589825299772803520002610888814176506636216905928021516429198484077446143617891415191517976537848282687018750030264867608433204658525470555882410254654806040437372771834769014720664234434374255514129178503032471263418076525187802925534774001104853996960549926508093910691337614841834884596365621526610332239417467064368340504749943339802285610313083038484571294767389856293937641914407036507544622061186499127249643799875806537850203753189972618014404667793050140301580709266213229273649718653952866567538572115133606114457222800851183757899219543063413692302293139751143702404830227357629039911794499248480915071002444078482866598579406525539141041497342780203520135419925977628178182825372022920108186449448349255421793982723279357095828748597126780783134286180750497175747373730296280477376908932558914598141724852658299510882230055223242218586191394795184220131553319634363922684259164168669438122537135960710031743651959027712571604588486044820674410935215327906816032054215967959066411120187618531256710150212239401285668608469435937408158536481912528004920724042172170913983123118054043277015835629513656274610248827706488865037765175678806872498861657094846665770674577000207144332525555736557083150320019082992096545498737419756608619533492312940263904930982014700371161829485939931199955070455381196711289367735249958182011774799788636393286405807810818657337668157893827656450642917396685579555053188715314552353070355994740186225988149854660737787698781542360397080977412361518245964026869979609564523828584235953564615185448165799966460648261396618720304839119560250381111550938420209894591555760083897989949964566262540514195610780090298667014635238532066032574466820259430618801773091109212741138269148784355679352572808875543164693077235363768226036080174040660997151176880434927489197133087822951123746632635635328517394189466510943745768270782209928468034684157443127739811044186762032954475468077511126663685479944460934809992951875666499902261686019672053749149951226823637895865245462813439289338365156536992413109638102559114643923805213907862893561660998836479175633176725856523591069520326895990054884753424160586689820067483163174286329119633399132709086065074595260357157323069712106423424081597068328707624437165532750228797802598690981111226558888151520837482450034463046505984569690276166958278982913613535306291331427881888249342136442417833519319786543940201465328083410341785272489879050919932369270996567133507711905899945951923990615156165480300145359212550696405345263823452155999210578191371030188979206408883974767667144727314254467923500524618849237455307575734902707342496298879996942094595961008702501329453325358045689285707241207965919809225550560061971283541270202072583994171175520920820151096509526685113897577150810849443508285458749912943857563115668324566827992991861539009255871716840495663991959154034218364537212023678608655364745175654879318925644085274489190918193411667583563439758886046349413111875241038425467937999203546910411935443113219136068129657568583611774564654674861061988591414805799318725367531243470335482637527081353105570818049642498584646147973467599315946514787025065271083508782350656532331797738656666181652390017664988485456054961300215776115255813396184027067814900350252876823607822107397102339146870159735868589015297010347780503292154014359595298683404657471756232196640515401477953167461726208727304820634652469109953327375561090578378455945469160223687689641425960164689647106348074109928546482353083540132332924864037318003195202317476206537726163717445360549726690601711176761047774971666890152163838974311714180622222345718567941507299526201086205084783127474791909996889937275229053674785020500038630036526218800670926674104806027341997756660029427941090400064654281074454007616429525362460261476180471744322889953285828397762184600967669267581270302806519535452053173536808954589902180783145775891280203970053633193821100095443241244197949192916205234421346395653840781209416214835001155883618421164283992454027590719621537570187067083731012246141362048926555668109467076386536083015847614512581588569610030337081197058344452874666198891534664244887911940711423940115986970795745946337170243268484864632018986352827092313047089215684758207753034387689978702323438584381125011714013265769320554911860153519551654627941175593967947958810333935413289702528893533748106257875620364294270257512121137330213811951395756419122685155962476203282038726342066227347868223036522019655729325905068134849292299647248229359787842720945578267329975853818536442370617353517653060396801087899490506654491544577952166038552398013798104340564182403396162494910454712104839439200945914647542424785991096900046541371091630096785951563947332190934511838669964622788855817353221326876634958059123761251203010983867841195725887799206041260049865895027247133146763722204388398558347770112599424691208308595666787531942465131444389971195968105937957532155524204659410081418351120174196853432672343271868099625045432475688702055341969199545300952644398446384346598830418262932239295612610045884644244285011551557765935780379565026806130721758672048541797157896401554276881090475899564605488362989140226580026134158039480357971019004151547655018391755772677897148793477372747525743898158705040701968215101218826088040084551332795162841280679678965570163917067779841529149397403158167896865448841319046368332179115059107813898261026271979696826411179918656038993895418928488851750122504754778999508544083983800725431468842988412616042682248823097788556495765424017114510393927980290997604904428832198976751320535115230545666467143795931915272680278210241540629795828828466355623580986725638200565215519951793551069127710538552661926903526081367717666435071213453983711357500975854405939558661737828297120544693182260401670308530911657973113259516101749193468250063285777004686987177255226525708428745733039859744230639751837209975339055095883623642814493247460522424051972825153787541962759327436278819283740253185668545040893929401040561666867664402868211607294830305236465560955351079987185041352121321534713770667681396211443891632403235741573773787908838267618458756361026435182951815392455211729022985278518025598478407179607904114472041476091765804302984501746867981277584971731733287305281134969591668387877072315968334322509070204019030503595891994666652037530271923764252552910347950343816357721698115464329245608951158732012675424975710520894362639501382962152214033621065422821876739580121286442788547491928976959315766891987305176388698461503354594898541849550251690616888419122873385522699976822609645007504500096116866129171093180282355042553653997166054753907348915189650027442328981181709248273610863801576007240601649547082331349361582435128299050405405333992577071321011503713898695076713447940748097845416328110406350804863393555238405735580863718763530261867971725608155328716436111474875107033512913923595452951407437943144900950809932872153235195999616750297532475931909938012968640379783553559071355708369947311923538531051736669154087312467233440702525006918026747725078958903448856673081487299464807786497709361969389290891718228134002845552513917355978456150353144603409441211512001738697261466786933733154341007587514908295822756919350542184106448264951943804240543255345965248373785310657979037977505031436474651422484768831323479762673689855474944277949916560108528257618964374464656819789319422077536824661110427671936481836360534108748971066866318805026555929568123959680449295166615409802610781691689418764353363449482900125929366840591370059526914934421861891742142561071896846626335874414976973921566392767687720145153302241853125308442727245771161505550519076276250016522166274796257424425420546785767478190959486500575711016264847833741198041625940813327229905891486422127968042984725356237202887830051788539737909455265135144073130049869453403245984236934627060242579432563660640597549471239092372458126154582526667304702319359866523378856244229188278436440434628094888288712101968642736370461639297485616780079779959696843367730352483047478240669928277140069031660709951473154191919911453182543906294573298686613524886500574780251977607442660798300291573030523199052185718628543687577860915726925232573171665625274275808460620177046433101212443409281314659760221360416223031167750085960128475289259463348312408766740128170543067985261868949895004918275008304998926472034986965363326210919830621495095877228260815566702155693484634079776879525038204442326697479264829899016938511552124688935873289878336267819361764023681714606495185508780596635354698788205094762016350757090024201498400967867845405354130050482404996646978558002628931826518708714613909521454987992300431779500489569529280112698632533646737179519363094399609176354568799002814515169743717518330632232942199132137614506411391269837128970829395360832883050256072727563548374205497856659895469089938558918441085605111510354367477810778500572718180809661542709143010161515013086522842238721618109043183163796046431523184434669799904865336375319295967726080853457652274714047941973192220960296582500937408249714373040087376988068797038047223488825819819025644086847749767508999164153502160223967816357097637814023962825054332801828798160046910336602415904504637333597488119998663995617171089911809851197616486499233594328274275983382931099806461605360243604040848379619072542165869409486682092396143083817303621520642297839982533698027039931804024928814430649614747600087654305571672697259114631990688823893005380061568007730984416061355843701277573463708822073792921409548717956947854414951731561828176343929570234710460088230637509877521391223419548471196982303169544468045517922669260631327498272520906329003279972932906827204647650366969765227673645419031639887433042226322021325368176044169612053532174352764937901877252263626883107879345194133825996368795020985033021472307603375442346871647223795507794130304865403488955400210765171630884759704098331306109510294140865574071074640401937347718815339902047036749084359309086354777210564861918603858715882024476138160390378532660185842568914109194464566162667753712365992832481865739251429498555141512136758288423285957759412684479036912662015308418041737698963759002546999454131659341985624780714434977201991702665380714107259910648709897259362243300706760476097690456341576573395549588448948093604077155688747288451838106069038026528318275560395905381507241627615047252487759578650784894547389096573312763852962664517004459626327934637721151028545472312880039058405918498833810711366073657536918428084655898982349219315205257478363855266205400703561310260405145079325925798227406012199249391735122145336707913500607486561657301854049217477162051678486507913573336334257685988361252720250944019430674728667983441293018131344299088234006652915385763779110955708000600143579956351811596764725075668367726052352939773016348235753572874236648294604770429166438403558846422370760111774821079625901180265548868995181239470625954254584491340203400196442965370643088660925268811549596291166168612036195319253262662271108142149856132646467211954801142455133946382385908540917878668826947602781853283155445565265933912487885639504644196022475186011405239187543742526581685003052301877096152411653980646785444273124462179491306502631062903402737260479940181929954454297256377507172705659271779285537195547433852182309492703218343678206382655341157162788603990157495208065443409462446634653253581574814022471260618973060860559065082163068709634119751925774318683671722139063093061019303182326666420628155129647685313861018672921889347039342072245556791239578260248978371473556820782675452142687314252252601795889759116238720807580527221031327444754083319215135934526961397220564699247718289310588394769170851420631557192703636345039529604362885088555160008371973526383838996789184600327073682083234847108471706160879195227388252347506380811606090840124222431476103563328940609282430125462013806032608121942876847907192546246309055749298781661271916548229644317263587524548607563020667656942355342774617635549231817456159185668061686428714964129290560130053913469569829490891003991259088290348791943368696942620662946948514931472688923571615032405542263391673583102728579723061998175868700492227418629077079508809336215346303842967525604369606110193842723883107587771653594778681499030978765900869583480043137176832954871752604714113064847270887246697164585218774442100900090916189819413456305028950484575822161887397443918833085509908566008543102796375247476265353031558684515120283396640547496946343986288291957510384781539068343717740714095628337554413567955424664601335663617305811711646062717854078898495334329100315985673932305693426085376230981047171826940937686754301837015557540822371538037838383342702379535934403549452173960327095407712107332936507766465603712364707109272580867897181182493799540477008369348889220963814281561595610931815183701135104790176383595168144627670903450457460997444500166918675661035889313483800512736411157304599205955471122443903196476642761038164285918037488354360663299436899730090925177601162043761411616688128178292382311221745850238080733727204908880095181889576314103157447684338100457385008523652069340710078955916549813037292944462306371284357984809871964143085146878525033128989319500645722582281175483887671061073178169281242483613796475692482076321356427357261609825142445262515952514875273805633150964052552659776922077806644338105562443538136258941809788015677378951310313157361136026047890761945591820289365770116416881703644242694283057457471567494391573593353763114830246668754727566653059819746822346578699972291792416156043557665183382167059157867799311835820189855730344883681934418305987021880502259192818047775223884407167894780414701414651073580452021499197980812095692195622632313741870979731320870864552236740416185590793816745658234353037283309503729022429802768451559528656923189798000383061378732434546500582722712325031420712488100290697226311129067629080951145758060270806092801504406139446350643069742785469477459876821004441453438033759717384777232052065301037861326418823586036569054773343070911759152582503029410738914441818378779490613137536794654893375260322906277631983337976816641721083140551864133302224787118511817036598365960493964571491686005656771360533192423185262166760222073368844844409234470948568027905894191829969467724456269443308241243846160408284006424867072583661011433404214473683453638496544701067827313169538435919120440283949541956874453676459875488726170687163109591315801609722382049772577307454562979127906177531663252857205858766376754282917933549923678212008601904369428956102301731743150352204665675088491593025926618816581008701658499456495586855628208747248318351516339189292646558880593601275151838235485893426165223086697314511412035659916934103076974774451947043836739600076578628245472064617380804602903639144493859012422380173377038154675297645596518492676039300171943042511794045679862114630138402371099347243455794730048929825402680821621522346560274258486595687074510352794291633405915025075992398611224340312056999780516223878772230396359709132856830486160362127579561601328561866388146004722200580017580282279272167842720649966956840905752590774886105493806116954293569077377792821084159737469613143291808510446953973485067590503662391722108732333169909603363771705474725026941732982890400239372879549386540463828596742216318201530139629734398479588628632934746650690284066719018081265539973675916799759010867483920062877888531102781695087545740384607594616919584610655963327283485609570305572502494416337066573150237126843581984154103154401008430380631442183776750349813408169325201240813452285974626715177152223063741359255747513535160669108359443999692315898156732033027129284241219651936303734407981204656795322986357374589031654007016472204989445629050395873788912680565516464274460174738175296313458739390484560414203426465560422112239134631023161290836446988901247285192778589195228773637440432659264672239982186452797664826673070168802722052338600372842903155828454593854349099449420750911108532138744823216151007808922516285123275724355101999038195993350032641446053470357293073912578481757987468353429629749652545426864234949270336399427519354240001973125098882419600095766257217621860474573769577649582201796258392376391717855799468922496750179251915218219624653575570564228220399546682648329822996167217080156801080799777126517156274295763666959661983507435667132218383358509536665806605597148376773866922551603463644386269977295750658468929599809168949981898588529537874489519527097766262684177088590284321676352132630838812766335363319004134332844347630067982023716933653652880580156390360562722752187272454764258840995216482554453662083811789117725225682611478014242896970967121967502094421226279437073328703410646312100557376727450271638975234111426287828736758358819056742163061523416789476056879277154789714326222041069587947186435439940738639948986836168919377836648327137363654676901173760246643082285362494712605173293777247276797635865806019396287718060679122426813922872134061694882029506831654589707623668302556167559477498715183426989208952182644710514911419441192277010977616645850068963849426165593473112961064282379048216056210094265076173838082479030510998790719611852832556787472942907151041468948104916751035295897242381802288151276582257190705537652455285511598636421244284176256230139538669970308943645907600684938040875210854159851278070333207779865635907968462191534944587677170063778573171211036517486371634098385626541555573292664616402279791195975248525300376741774056125700303625811704838385391207273191845064713669122576415213769896260940351804147432053600369234179035440735703058314741623452840188940808983125191307741823338981880316339159565954543405777784331681162551898060409183018907512170192983622897099598983405484962284289398469847938668614293324543983592637036699355184231661615244505980576745765335552338715678211466689996845227042954589710922163652573965950289645637766038988037941517917867910675199009966139206238732318786758420544279396366759104126821843375015743069045967947046685602358283919759975285865384338189120042853787549302768972168199113340697282255535300044743958830079799736518459131437946494086272149669719100359399974735262764126125995350902609540048669398955899487421379590802893196914845826873123710180229775301190684280440780938156598081694611679374425663244656799606363751546304833112722231812338371779800439731087402647536582575657351059978314264831879619843765495877803685261751835391844920488198629786329743136948511780579298636452193232481339393090754566368038513630619718033957979522539508697432546502659123585049283028832934489284591373621624852528877442891851104093746333590660233239711922814450735588373324057814862662207486215513375036775585494138678352928273109003823116855374520901095101174796663003330352534143230024288248051396631446632656081582045216883922312025671065388459503224002320453633895521539919011035217362720909565500846486605368975498478995875596103167696587161281951919668893326641203784750417081752273735270989343717167642329956935697166213782736138899530515711822960896394055380431939398453970864418654291655853168697537052760701061488025700785387150835779480952313152747735711713643356413242974208137266896149109564214803567792270566625834289773407718710649866150447478726164249976671481383053947984958938064202886667951943482750168192023591633247099185942520392818083953020434979919361853380201407072481627304313418985942503858404365993281651941497377286729589582881907490040331593436076189609669494800067194371424058105327517721952474344983414191979918179909864631583246021516575531754156198940698289315745851842783390581029411600498699307751428513021286202539508732388779357409781288187000829944831476678183644656510024467827445695591845768068704978044824105799710771577579093525803824227377612436908709875189149049904225568041463131309240101049368241449253427992201346380538342369643767428862595140146178201810734100565466708236854312816339049676558789901487477972479202502227218169405159042170892104287552188658308608452708423928652597536146290037780167001654671681605343292907573031466562485809639550080023347676187068086526878722783177420214068980703410506200235273632267291964034093571225623659496432076928058165514428643204955256838543079254299909353199329432966018220787933122323225928276556048763399988478426451731890365879756498207607478270258861409976050788036706732268192473513646356758611212953074644777149423343867876705824452296605797007134458987594126654609414211447540007211790607458330686866231309155780005966522736183536340439991445294960728379007338249976020630448806064574892740547730693971337007962746135534442514745423654662752252624869916077111131569725392943756732215758704952417232428206555322808868670153681482911738542735797154157943689491063759749151524510096986573825654899585216747260540468342338610760823605782941948009334370046866568258579827323875158302566720152604684361412652956519894291184887986819088277339147282063794512260294515707367105637720023427811802621502691790400488001808901847311751199425460594416773315777951735444490965752131026306836047140331442314298077895617051256930051804287472368435536402764392777908638966566390166776625678575354239947427919442544664643315554138265543388487778859972063679660692327601733858843763144148113561693030468420017434061395220072403658812798249143261731617813894970955038369479594617979829257740992171922783223006387384996138434398468502234780438733784470928703890536420557474836284616809363650973790900204118525835525201575239280826462555785658190226958376345342663420946214426672453987171047721482128157607275305173330963455909323664528978019175132987747952929099598069790148515839540444283988381797511245355548426126784217797728268989735007954505834273726937288386902125284843370917479603207479554080911491866208687184899550445210616155437083299502854903659617362726552868081324793106686855857401668022408227992433394360936223390321499357262507480617409173636062365464458476384647869520547719533384203403990244761056010612777546471464177412625548519830144627405538601855708359981544891286863480720710061787059669365218674805943569985859699554089329219507269337550235821561424994538234781138316591662683103065194730233419384164076823699357668723462219641322516076261161976034708844046473083172682611277723613381938490606534404043904909864126903479263503943531836741051762565704797064478004684323069430241749029731181951132935746854550484711078742905499870600373983113761544808189067620753424526993443755719446665453524088287267537759197074526286322840219629557247932987132852479994638938924943286917770190128914220188747760484939855471168524810559991574441551507431214406120333762869533792439547155394213121021954430556748370425907553004950664994802614794524739012802842646689229455664958621308118913500279654910344806150170407268010067948926855360944990373928383520627992820181576427054962997401900837493444950600754365525758905546552402103412862124809003162941975876195941956592556732874237856112669741771367104424821916671499611728903944393665340294226514575682907490402153401026923964977275904729573320027982816062130523130658731513076913832317193626664465502290735017347656293033318520949298475227462534564256702254695786484819977513326393221579478212493307051107367474918016345667888810782101151826314878755138027101379868751299375133303843885631415175908928986956197561123025310875057188962535763225834275763348421016668109884514141469311719314272028007223449941999003964948245457520704922091620614222912795322688239046498239081592961111003756999529251250673688233852648213896986384052437049402152187547825163347082430303521036927849762517317825860862215614519165573478940019558704784741658847364803865995119651409542615026615147651220820245816010801218275982577477652393859159165067449846149161165153821266726927461290533753163055654440793427876550267301214578324885948736899073512166118397877342715872870912311383472485146035661382188014840560716074652441118841800734067898587159273982452147328317214621907330492060817440914125388918087968538960627860118193099489240811702350413554126823863744341209267781729790694714759018264824761112414556423937732224538665992861551475342773370683344173073150805440138894084087253197595538897613986400165639906934600670780501058567196636796167140097031535132386972899001749862948883362389858632127176571330142071330179992326381982094042993377790345261665892577931395405145369730429462079488033141099249907113241694504241391265397274078984953073730364134893688060340009640631540701820289244667315059736321311926231179142794944897281477264038321021720718017561601025111179022163703476297572233435788863537030535008357679180120653016668316780269873860755423748298548246360981608957670421903145684942967286646362305101773132268579232832164818921732941553151386988781837232271364011755881332524294135348699384658137175857614330952147617551708342432434174779579226338663454959438736807839569911987059388085500837507984051126658973018149321061950769007587519836861526164087252594820126991923916722273718430385263107266000047367872474915828601694439920041571102706081507270147619679971490141639274282889578424398001497985658130305740620028554097382687819891158955487586486645709231721825870342960508203415938806006561845735081804032347750084214100574577342802985404049555529215986404933246481040773076611691605586804857302606467764258503301836174306413323887707999698641372275526317649662882467901094531117120243890323410259937511584651917675138077575448307953064925086002835629697045016137935696266759775923436166369375035368699454550392874449940328328128905560530091416446608691247256021455381248285307613556149618444364923014290938289373215312818797541139219415606631622784836152140668972661027123715779503062132916001988806369127647416567067485490795342762338253943990022498972883660263920518704790601584084302914787302246651371144395418253441269003331181914268070735159284180415100555199146564934872796969351992963117195821262627236458009708099166752820365818699111948365866102758375863322993225541477479210421324166848264953111826527351008031659958888814809945737293785681411438021523876706455063233067233939551964260397443829874822322662036352861302543796600943104500158604854027036789711934695579989189112302233381602302236277726084846296189550730850698061500281436425336666311433321645213882557346329366870956708432252564333895997812402164189946978348320376011613913855499933990786652305860332060641949298931012423081105800169745975038516887112037747631577311831360002742502722451570906304496369230938382329175076469684003556425503797106891999812319602533733677437970687713814747552190142928586781724044248049323750330957002929126630316970587409214456472022710796484778657310660832173093768033821742156446602190335203981531618935787083561603302255162155107179460621892674335641960083663483835896703409115513087820138723494714321400450513941428998350576038799343355677628023346565854351219361896876831439866735726040869511136649881229957801618882834124004126142251475184552502502640896823664946401177803776799157180146386554733265278569418005501363433953502870836220605121839418516239153709790768084909674194289061134979961034672077354959593868862427986411437928435620575955500144308051267664432183688321434583708549082240014585748228606859593502657405750939203135881722442164955416889785558265198046245527898343289578416968890756237467281044803018524217706136533236073856228166664597654076844715963930782091017090763377917711485205493367936868430832404126789220929930411890501756484917499452393770674524578019171841679541825554377930299249277892416277257788147974770446005423669346157135208417428211847353652367573702352791459837645712257646122605628127852169580892808988394594406165340521932514843306105322700231133680378433377389724881307874325614952744243584753011150345103737688223837573804282007358586938044331529253129961025096113761670187568525921208929131354473196308440066835155160913925692912175784379179004808848023029304392630921342768601226558630456913133560978156776098711809238440656353136182676923761613389237802972720736243967239854144480757286813436768000573823963610796223140429490728058551444771338682314499547929338131259971996894072233847404542592316639781608209399269744676323921370773991899853301483814622364299493902073285072098040905300059160091641710175605409814301906444379905831277826625762288108104414704097708248077905168225857235732665234414956169007985520848841886027352780861218049418060017941147110410688703738674378147161236141950474056521041002268987858525470689031657094677131822113205505046579701869337769278257145248837213394613987859786320048011792814546859096532616616068403160077901584946840224344163938313618742275417712170336151163782359059685168880561304838542087505126933144171705880517278127917564053282929427357971823360842784676292324980318169828654166132873909074116734612367109059236155113860447246378721244612580406931724769152219217409096880209008801535633471775664392125733993165330324425899852598966724744126503608416484160724482125980550754851232313331300621490042708542735985913041306918279258584509440150719217604794274047740253314305451367710311947544521321732225875550489799267468541529538871443696399406391099267018219539890685186755868574434469213792094590683677929528246795437302263472495359466300235998990248299853826140395410812427393530207575128774273992824866921285637240069184859771126480352376025469714309316636539718514623865421671429236191647402172547787238964043145364190541101514371773797752463632741619269990461595895793940622986041489302535678633503526382069821487003578061101552210224486633247184367035502326672749787730470216165019711937442505629639916559369593557640005236360445141148916155147776301876302136068825296274460238077523189646894043033182148655637014692476427395401909403584437251915352134557610698046469739424511797999048754951422010043090235713636892619493763602673645872492900162675597083797995647487354531686531900176427222751039446099641439322672532108666047912598938351926694497553568096931962642014042788365702610390456105151611792018698900673027082384103280213487456720062839744828713298223957579105420819286308176631987048287388639069922461848323992902685392499812367091421613488781501234093387999776097433615750910992585468475923085725368613605356762146929424264323906626708602846163376051573599050869800314239735368928435294958099434465414316189806451480849292695749412903363373410480943579407321266012450796613789442208485840536446021616517885568969302685188950832476793300404851688934411125834396590422211152736276278672366665845757559585409486248261694480201791748223085835007862255216359325125768382924978090431102048708975715033330963651576804501966025215527080352103848176167004443740572131294252820989545456276344353575741673638980108310579931697917916718271145837435222026387771805250290791645414791173616253155840768495583288190293564201219633684854080865928095131505012602919562576032932512847250469881908146475324342363863860247943921015193235101390117789997483527186469346024554247028375300033725403910085997650987642832802908445662021678362267272292737780213652404028817217012490974899454430826861772239385250883760749742195942655217301733355851389407457348144161511380845358039740277795072051893487170722955427683655826706766313911972211811528466502223383490906676554168336907959409404576472940901354356409277969379842065738891481990225399022315913388145851487225126560927576795873759207013915029216513720851137197522734365458411622066281660256333632074449918511469174455062297146086578736313585389023662557285424516018080487167823688885575325066254262367702604215835160174851981885460860036597606743233346410471991027562358645341748631726556391320606407754779439671383653877377610828300019937359760370467245737880967939894493795829602910746901609451288456550071458091887879542641820145369659962842686882363495879277007025298960996798975941955735253914237782443302746708282008722602053415292735847582937522487377937899136764642153727843553986244015856488692101644781661602962113570056638347990334049623875941092886778920270077504951511405782565295015024484968204744379710872943108541684540513016310902267112951959140520827546866418137305837933236150599142045255880213558474751516267815309465541240524091663857551298894834797423322854504140527354235070335984964593699534959698554244978249586929179182415068053002553370412778703476446244329205906832901886692400222391918714603175399666877477960121790688623311002908668305431787009355066944389131913333586368037447530664502418437136030852288582121720231274167009740351431532131803978033680228154223490183737494117973254478594157962104378787072154814091725163615415163381388912588517924237727229603497305533840942889918919161186249580560073570527227874940321250645426206304469470804277945973817146810395192821550688079136701210109944220737024613687196031491162370967939354636396448139025711768057799751751298979667073292674886430097398814873780767363792886767781170520534367705731566895899181530825761606591843760505051704242093231358724816618683821026679970982966436224723644898648976857100173643547336955619347638598187756855912376232580849341570570863450733443976604780386678461711520325115528237161469200634713570383377229877321365028868868859434051205798386937002783312365427450532283462669786446920780944052138528653384627970748017872477988461146015077617116261800781557915472305214759943058006652042710117125674185860274188801377931279938153727692612114066810156521441903567333926116697140453812010040811760123270513163743154487571768761575554916236601762880220601068655524141619314312671535587154866747899398685510873576261006923021359580838145290642217792987748784161516349497309700794368305080955621264592795333690631936594413261117944256602433064619312002953123619348034504503004315096798588111896950537335671086336886944665564112662287921812114121425167348136472449021275252555647623248505638391391630760976364990288930588053406631352470996993362568102360392264043588787550723319888417590521211390376609272658409023873553418516426444865247805763826160023858280693148922231457758783791564902227590699346481624734399733206013058796068136378152964615963260698744961105368384203105364183675373594176373955988088591188920114871545460924735613515979992999722298041707112256996310945945097765566409972722824015293663094891067963296735505830412258608050740410916678539569261234499102819759563955711753011823480304181029089719655278245770283085321733741593938595853203645590564229716679900322284081259569032886928291260139267587858284765599075828016611120063145411315144108875767081854894287737618991537664505164279985451077400771946398046265077776614053524831090497899859510873112620613018757108643735744708366215377470972660188656210681516328000908086198554303597948479869789466434027029290899143432223920333487108261968698934611177160561910681226015874410833093070377506876977485840324132474643763087889666151972556180371472590029550718424245405129246729039791532535999005557334600111693557020225722442772950263840538309433999383388018839553821540371447394465152512354603526742382254148328248990134023054550811390236768038649723899924257800315803725555410178461863478690646045865826036072306952576113184134225274786464852363324759102670562466350802553058142201552282050989197818420425028259521880098846231828512448393059455162005455907776121981297954040150653985341579053629101777939776957892084510979265382905626736402636703151957650493344879513766262192237185642999150828898080904189181015450813145034385734032579549707819385285699926238835221520814478940626889936085239827537174490903769904145555260249190126341431327373827075950390882531223536876389814182564965563294518709637484074360669912550026080424160562533591856230955376566866124027875883101021495284600804805028045254063691285010599912421270508133194975917146762267305044225075915290251742774636494555052325186322411388406191257012917881384181566918237215400893603475101448554254698937834239606460813666829750019379115061709452680984785152862123171377897417492087541064556959508967969794980679770961683057941674310519254486327358885118436597143583348756027405400165571178309126113117314169066606067613797690123141099672013123730329707678988740099317309687380126740538923612230370779727025191340850390101739924877352408881040807749924412635346413181858792480760553268122881584307471326768283097203149049868884456187976015468233715478415429742230166504759393312132256510189175368566338139736836336126010908419590215582111816677413843969205870515074254852744810154541079359513596653630049188769523677579147319184225806802539818418929888943038224766186405856591859943091324575886587044653095332668532261321209825839180538360814144791320319699276037194760191286674308615217243049852806380129834255379486287824758850820609389214668693729881191560115633701248675404205911464930888219050248857645752083363921499441937170268576222251074166230901665867067714568862793343153513505688216165112807318529333124070912343832502302341169501745502360505475824093175657701604884577017762183184615567978427541088499501610912720817913532406784267161792013428902861583277304794830971705537485109380418091491750245433432217445924133037928381694330975012918544596923388733288616144238100112755828623259628572648121538348900698511503485369544461542161283241700533583180520082915722904696365553178152398468725451306350506984981006205514844020769539324155096762680887603572463913955278222246439122592651921288446961107463586148252820017348957533954255019475442643148903233373926763409115527189768429887783617346613535388507656327107814312435018965109238453660236940276060642119384227665755210663671879603217527184404651560427289869560206997012906367847161654793068868305846508082886614111979138822898112498261434559408961813509226857611474609406147937240008842153535862052780125014270055274468359151840373309373580494342483940467505708347927948338133276237937844629209323999417593374917899786484958148818865149169302451512835579818112344900827168644548306546633975256079615935830821400021951611342337058359111545217293721664061708131602078213341260356852013161345136871600980378712556766143923146458085652084039744217352744813741215277475202259244561520365608268890193913957991844109971588312780020898275935898106482117936157951837937026741451400902833064466209280549839169261068975151083963132117128513257434964510681479694782619701483204392206140109523453209269311762298139422044308117317394338867965739135764377642819353621467837436136161591167926578700137748127848510041447845416464568496606699139509524527949914769441031612575776863713634644477006787131066832417871556281779122339077841275184193161188155887229676749605752053192594847679397486414128879475647133049543555044790277128690095643357913405127375570391806822344718167939329121448449553897728696601037841520390662890781218240141299368590465146519209198605347788576842696538459445700169758422531241268031418456268722581132040056433413524302102739213788415250475704533878002467378571470021087314693254557923134757243640544448132093266582986850659125571745568328831440322798049274104403921761438405750750288608423536966715191668510428001748971774811216784160854454400190449242294333666338347684438072624307319019363571067447363413698467328522605570126450123348367412135721830146848071241856625742852208909104583727386227300781566668914250733456373259567253354316171586533339843321723688126003809020585719930855573100508771533737446465211874481748868710652311198691114058503492239156755462142467550498676710264926176510110766876596258810039163948397811986615585196216487695936398904500383258041054420595482859955239065758108017936807080830518996468540836412752905182813744878769639548306385089756146421874889271294890398025623046812175145502330254086076115859321603465240763923593699949180470780496764486889980902123735780457040380820770357387588525976042434608851075199334470112741787878845674656640471901619633546770714090590826954225196409446319547658653032104723804625249971910690110456227579220926904132753699634145768795242244563973018311291451151322757841320376225862458224784696669785947914981610522628786944136373683125108310682898766123782697506343047263278453719024447970975017396831214493357290791648779915089163278018852504558488782722376705263811803792477835540018117452957747339714012352011459901984753358434861297092928529424139865507522507808919352104173963493428604871342370429572757862549365917805401652536330410692033704691093097588782938291296447890613200063096560747882082122140978472301680600835812336957051454650181292694364578357815608503303392466039553797630836137289498678842851139853615593352782103740733076818433040893624460576706096188294529171362940967592507631348636606011346115980434147450705511490716640635688739020690279453438236930531133440901381392849163507484449076828386687476663619303412376248380175840467851210698290605196112357188811150723607303158506622574566366740720668999061320627793994112805759798332878792144188725498543014546662945079670707688135022230580562225942983096887732856788971494623888272184647618153045844390967248232348259587963698908456664795754200195991919240707615823002328977439748112690476546256873684352229063217889227643289360535947903046811114130586348244566489159211382258867880972564351646404364328416076247766114349880319792230537889671148058968061594279189647401954989466232962162567264739015818692956765601444248501821713300527995551312539849919933907083138030214072556753022600033565715934283182650908979350869698950542635843046765145668997627989606295925119763672907762567862769469947280606094290314917493590511523235698715397127866718077578671910380368991445381484562682604003456798248689847811138328054940490519768008320299631757043011485087384048591850157264392187414592464617404735275250506783992273121600117160338604710710015235631159734711153198198710616109850375758965576728904060387168114313084172893710817412764581206119054145955378853200366615264923610030157044627231777788649806700723598889528747481372190175074700005571108178930354895017924552067329003818814068686247959272205591627902292600592107710510448103392878991286820705448979977319695574374529708195463942431669050083984398993036790655541596099324867822475424361758944371791403787168166189093900243862038610001362193667280872414291108080291896093127526202667881902085595708111853836166128848729527875143202956393295910508349687029060692838441522579419764824996318479414814660898281725690484184326061946254276693688953540732363428302189694947766126078346328490315128061501009539164530614554234923393806214007779256337619373052025699319099789404390847443596972052065999017828537676265683558625452697455260991024576619614037537859594506363227095122489241931813728141668427013096050734578659047904243852086508154491350136491698639048125666610843702294730266721499164849610746803261583352580352858275799038584091667618877199539888680431991650866887781701439663176815592262016991396613153738021294160006906947533431677802632207226265881842757216055461439677336258462997385077307751473833315101468395296411397329672457933540390136107395245686243008096720460995545708974893048753897955544443791303790422346037768729236001386569593952300768091377768847789746299699489949016141866131552200856673695770822720338936659590666350594330040363762591189195691561626122704788696510356062748423100605472091437069471661080277379848576543481249822444235828329813543645124092220896643987201997945619030397327254617823136363375927622656301565813545578319730419339269008282952718252138855126583037630477490625995514925943105307478901043009876580816508144862607975129633326675259272351611791836777128931053144471668835182920514343609292493191180249366051791485330421043899773019267686085347768149502299280938065840007311767895491286098112311307002535600347898600653805084532572431553654422067661352337408211307834360326940015926958459588297845649462271300855594293344520727007718206398887404742186697709349647758173683580193168322111365547392288184271373843690526638607662451284299368435082612881367358536293873792369928837047900484722240370919885912556341130849457067599032002751632513926694249485692320904596897775676762684224768120033279577059394613185252356456291805905295974791266162882381429824622654141067246487216174351317397697122228010100668178786776119825961537643641828573481088089988571570279722274734750248439022607880448075724807701621064670166965100202654371260046641935546165838945950143502160890185703558173661823437491622669077311800121188299737319891006060966841193266075165452741829459541189277264192546108246351931647783837078295218389645376236304858042774417907169146356546201215125418664885396161542055152375000426794253417764590821513675258479774465114750438460596325820468809667795709044645884673847481638045635188183210386594798204376334738389017759714236223057776395541011294523488098341476645559342209402059733452337956309441446698222457026367119493286653989491344225517746402732596722993581333110831711807234044326813737231209669052411856734897392234152750707954137453460386506786693396236535556479102508529284294227710593056660625152290924148057080971159783458351173168204129645967070633303569271821496292272073250126955216172649821895790908865085382490848904421755530946832055636316431893917626269931034289485184392539670922412565933079102365485294162132200251193795272480340133135247014182195618419055761030190199521647459734401211601239235679307823190770288415814605647291481745105388060109787505925537152356112290181284710137917215124667428500061818271276125025241876177485994084521492727902567005925854431027704636911098800554312457229683836980470864041706010966962231877065395275783874454229129966623016408054769705821417128636329650130416501278156397799631957412627634011130135082721772287129164002237230234809031485343677016544959380750634285293053131127965945266651960426350406454862543383772209428482543536823186182982713182489884498260285705690699045790998144649193654563259496570044689011049923939218088155626191834404362264965506449848521612498442375928443642612004256628602157801140467879662339228190804577624109076487087406157070486658398144845855803277997327929143195789110373530019873110486895656281917362036703039179710646309906285483702836118486672219457621775034511770110458001291255925462680537427727378863726783016568351092332280649908459179620305691566806180826586923920561895421631986004793961133953226395999749526798801074576466538377400437463695133685671362553184054638475191646737948743270916620098057717103475575333102702706317395612448413745782734376330101853438497450236265733191742446567787499665000938706441886733491099877926005340862442833450486907338279348425305698737469497333364267191968992849534561045719338665222471536681145666596959735075972188416698767321649331898967182978657974612216573922404856900225324160367805329990925438960169901664189038843548375648056012628830409421321300206164540821986138099462721214327234457806819925823202851398237118926541234460723597174777907172041523181575194793527456442984630888846385381068621715274531612303165705848974316209831401326306699896632888532682145204083110738032052784669279984003137878996525635126885368435559620598057278951754498694219326972133205286374577983487319388899574634252048213337552584571056619586932031563299451502519194559691231437579991138301656117185508816658756751184338145761060365142858427872190232598107834593970738225147111878311540875777560020664124562293239116606733386480367086953749244898068000217666674827426925968686433731916548717750106343608307376281613984107392410037196754833838054369880310983922140260514297591221159148505938770679068701351029862207502287721123345624421024715163941251258954337788492834236361124473822814504596821452253550035968325337489186278678359443979041598043992124889848660795045011701169092519383155609441705397900600291315024253848282782826223304151370929502192196508374714697845805550615914539506437316401173317807741497557116733034632008408954066541694665746735785483133770133628948904397670025863002540635264006601631712883920305576358989492412827022489373848906764385339931878608019223108328847459816417701264089078551777830131616162049792779670521847212730327970738223860581986744668610994383049960437407323195784473254857416239738852016202384784256163512597161783106850156299135559874758848151014815490937380933394074455700842090155903853444962128368313687375166780513082594599771257467939781491953642874321122421579851584491669362551569370916855252644720786527971466476760328471332985501945689772758983450586004316822658631176606237201721007922216410188299330808409384014213759697185976897042759041500946595252763487628135867117352364964121058854934496645898651826545634382851159137631569519895230262881794959971545221250667461174394884433312659432286710965281109501693028351496524082850120190831078678067061851145740970787563117610746428835593915985421673115153096948758378955979586132649569817205284291038172721213138681565524428109871168862743968021885581515367531218374119972919471325465199144188500672036481975944167950887487934416759598361960010994838744709079104099785974656112459851972157558134628546189728615020774374529539536929655449012953097288963767713353842429715394179547179095580120134210175150931491664699052366350233024087218654727629639065723341455005903913890253699317155917179823065162679744711857951506573868504088229934804445549850597823297898617029498418376255258757455303112991914341109413088238114443068843062655305601658801408561023324210300218460588586954418502977463085858496130037238190325162225570729975710727306066072916922978033647048840958711228045188511908718588299514331534128549297173849768523136276076868494780364948299904475715771141080958058141208956059471668626290036145602625334863284986816039463372436667112964460292915746181117789169695839947080954788863503281129626899231110099889317815313946681882028368363373822281414974006917942192888817139116283910295684918233358930813360131488748366464224381776081007739183393749346933644748150564933649323157235306109385796839902153381449126925350768211098738352197507736653475499431740580563099143218212547336281359488317681489194306530426029773885492974570569448783077945878865062970895499843760181694031056909587141386804846359853684034105948341788438963179956468815791937174656705047441528027712541569401365862097760735632832966564135817028088013546326104892768731829917950379944446328158595181380144716817284996793061814177131912099236282922612543236071226270324572637946863533391758737446552006008819975294017572421299723542069630427857950608911113416534893431149175314953530067419744979017235181671568754163484949491289001739377451431928382431183263265079530371177806185851153508809998200482761808307209649636476943066172549186143700971387567940218696710148540307471561091358933165600167252126542502898612259306484105898847129649230941215144563947889999327145875969555737090855150648002321476443037232466147111552578583071024936898814562568786834745518893385181791667579054210421036349316257870476543126790661216644142285017446278477132740595579600648343288827864837043456066966456899746910373987712891593313271266247505582258634928427718355831641593667712218537642376222104779338956378722902509543014182257180331300148113377736941508488867501893156994849838936052666818012783912005801431596441910546663236810148207799356523056490420711364192200177189107935243234322761787712568251126481332974354926568682748715986654943041648468220593921673359485057849622807932422649812705271398407720995707236227009245067665680069149966555737866411877079767754867028786431817941521796178310655030287157272282250812017060713380339641841211253856248920130010782462165136989511064611133562443838185366273563783436921279354709230119655914915800561707258518503167289370411936374780625824298250726464801821523430268081486978164824349353456855843696378384153838051184406043696871666416514036129729992912630842812149152469877429332305214999981829046119471676727503742221367186614654042534463141660649871499001000660041544868437352208483059495953182872280520828676300361091734508632133033647289584176588755345227938480297724485711815574893561311524926772006362198369980664159549388683836411891430443767715498026544959061738265591178545999378510861446014967645550103653971251138583505085112442517772923814396233043724036032603181442991365750246012787514117944901305803452199992701148071712847770301254994886841867572975189214295652512486943983729047410363121899124217339550688778643130750024823361832738729697376598820053895902935486054979802320400472236873557411858132734337978931582039412878989728973298812553514507641535360519462112217000676321611195841029252568536561813138784086477147099724553013170761712163186600291464501378587854802096244703771373587720086738054108140042311418525803293267396324596914044834665722042880679280616029884043400536534009706581694636096660911110968789751801325224478246957913251892122653056085866541115373584912790254654369020869419871125588453729063224423222287139122012248769976837147645598526739225904997885514250047585260297929306159913444898341973583316070107516452301310796620382579278533125161760789984630103493496981494261055367836366022561213767081421091373531780682420175737470287189310207606953355721704357535177461573524838432101571399813798596607129664438314791296359275429627129436142685922138993054980645399144588692472767598544271527788443836760149912897358259961869729756588978741082189422337344547375227693199222635973520722998387368484349176841191020246627479579564349615012657433845758638834735832242535328142047826934473129971189346354502994681747128179298167439644524956655532311649920677163664580318205849626132234652606175413532444702007661807418914040158148560001030119994109595492321434406067634769713089513389171050503856336503545166431774489640061738861761193622676890576955693918707703942304940038440622614449572516631017080642923345170422426679607075404028551182398361531383751432493056398381877995594942545196756559181968690885283434886050828529642437578712929439366177362830136595872723080969468398938676366226456791132977469812675226595621009318322081754694778878755356188335083870248295346078597023609865656376722755704495258739871812593441903785275571333409842450127258596692434317689018966145404453679047136294238156127656824247864736176671770647002431119711090007474065945650315375044177982192306323700872039212085499569681061379189029961178936752146022386905665481382858280449537530160921422195940638787074787991194920898374091788534417523064715030278397979864517336625329511775105559014160459873338186887977858817291976604516353353556047648420520888811722831990044504284486852338334530105533929637308039738230604714104525470094899407601215247602819963846343554852932377161410869591950786873276075400085220065031871239272857835807010762542769655355964789450166013816295177908531139811092831583216931563867459747449584385282701658246192092219529134323496779345585613140207765996142546463288677356891785576835169608392864188830094883324700447958316931533832382377876344426323456301679513671047510469669001217777128065522453689371871451567394733440447280450959433090683667110655953338602938000999949010642769859623260401863733572846679531229683156358145420890540651226419162015504500430562136991850941034609601030543816694795964585804425194905110733387679946734471718615647723811737035654917628707589456035519195603962301157866323750234725054461073979402475184415558178087962822231972692984516683306919505079993357259165675557294585962182052650473353712351623662770479333289322136141858785972771685682725303734836891911847197133753088446777943274857148827821608844765700041403499921376794209627560883081509438030705666022764678117533361028187800710219794428777313146387857817205661409023041499923248268982477222109852189758140879763486146763606368674611966620347304608917277240045953051376938375381543486981101990651706961774052218247422657652138152740612699012706880875386408669901461740890540981877671880076124151967064152117653084325544261017536348281196837493395825742541244634247233586360777980960199745187758845459645895956779558869098404768259253477849930457883128541747079059795909431627722327844578918694214929451540174214623240300841907975296782445969183509474202123617940309048634960534054931299919496087957952586977170236680033862505764938088740994009589948109397983231108838769236490221499111120870639202892490698435333152727991330986335454324971441378059132240814960156485679843966464780280409057580889190254236606774500413415794312112501275232250148067232979652230488493751166084976116412777395311302041566848265531411348993243747890268935173904043294851610659785832253168204202834993641595980197343889883020994152152288611175126686173051956249367180053845637855129171848417841594797435580617856680758491080185805695567990185198397660693358224779136504562705766735170961550493338390452612404395517449136885115987454340932040102218982707539212403241042424451570052968378815749468441508011138612561164102477190903050040240662278945607061512108266146098662040425010583978098192019726759010749924884966139441184159734610382401178556739080566483321039073867083298691078093495828888707110651559651222542929154212923108071159723275797510859911398076844732639426419452063138217862260999160086752446265457028969067192282283045169111363652774517975842147102219099906257373383472726498678244401048998507631630668050267115944636293525120269424810854530602810627264236538250773340575475701704367039596467715959261029438313074897245505729085688496091346323165819468660587092144653716755655531962091865952628448253731353698162517351930115341581171353292035873164168839107994000677266031617527582917398395852606454113318985505747847121053505795649095931672167565624818782002769963734155880000867852567422461511406015760115910256449002264980039498403358091309140197877843650167960167465370287466062584346329708303725980494653589318912163976013193079476972058034710553111117215859219066231028099212084069283091906017370764654655683413207556315315006453462321007133584907633048328153458698497332599801187479664273140279381289961720524540674695271948079930396730194274036466594154400092799908634806622334906695224044652158992864203435098858422692019340575496840904812955522654754650713532842543496616084954788090727649930252702815067862810825243222979985391759845188868387004477101866772159439708514664612871148749531862180941719676843144666435175837688436786081446319641912566574047718699160915550910878919431253671945651261878486910876729910565595155159739659034383628124629118117760949411880105946336671039049777312004243578115790429823045072038322781246413671297959415082918378213212876890545963586369344879749784841123274921331663162812456388238288715648447883142417650147980187858215768793063001153788998014623690135803753306246148576074932567807682651045738059018831237617271889933790487113395588485234240255002352200613574914318259142479829367775490496399350755839668967578364316618369307625603528602940662803255416535431518013714821941772672244005268401996533334184004345525296592918502940131600651124395297874364222806977720437363717873457948420238745151249157913139411148608416429347958793681868609689684640858334131017858142710955416293375915178392341303110543328703526599993904966822112768158316511246866451167351378214345336650598328347443536290312393672084593164394941881138607974670134709640378534907149089842317891739783650654751982883367395714360000003439863363212091718954899055748693397700245632475954504411422582410783866837655467400137324322809113692670682805397549111166171102397437749479335174036135005397581475520834285772800986189401984375446435081498218360112577632447389452051636938585136484259964518361856989088721789764694721246807900330925083496645841656554261294195108847197209106605105540933731954888406444080280579549008076040034154662137669606444293774985897353625591959618552448187940317374508256072895120945456562159540405425814886929842786582357673195799285293120866275922366115137445767916063621675267440451221051052090834707443986137829082352772895849625656881972792768694795806100573787084121444815034797422312103295359297822377134077549545477791813823542607184617108389097825964406170543546968567030745411634244134486308676327949177682923093183221341455482591367202823284396549001805653203960795517074496039006696990334199278212696767771835209083959545341866777944872740383733381985235884202840150981579594685874537989503257362809837592216229258598599123843993575573285028613155970362934249814178056461615863415338635077223269996508860870999964899373049307170967888740149746147542880387421250689212155876692242387434701120990859082164073576380817386959755176083877600277517253037133445654852635661720197563001580049790223419586738061442401502436288957503206533690825756785507020555105572381878574650371086308158185862815883054564662297694803970618265491385181326737485227188267917919091354407852685476254126683398240534022469989966652573155637645862251862823092085424412805997628505488913098331761884983352975136073772030571342739638126588567405013841074788943393996603591853934198416322617654857376671943132840050626295140357877264680649549355746326408186979718630218760025813995719923601345374229758918285167511358171472625828596940798518571870075823122317068134867930884899275181661399609753105295773584618525865211893339375771859916335112163441037910451845019023066893064178977808158101360449495409665363660370075881004450265734935127707426742578608784898185628869980851665713320835842613381142623855420315774246613108873106318111989880289722849790551075148403702290580483052731884959994156606537314021296702220821915862905952604040620011815269664910068587592655660567562963361434230232810747488395040380984981860056164646099819257616235478710913832967563761506732550860683433720438748186791668975746563456020002562889601191100980453350423842063824039434163502977688802779835087481178298349417211674919425601608685332435385951152061809031241698182079314615062073826097180458265687043623935757495737332781578904386011378078508110273049446611821957450170106059384336519458628360682108585130499820420578458577175933849015564447305834515291412561679970569657426139901681932056241927977282026714297258700193234337873153939403115411184101414292741703537542003698760608765500109345299007034032401334806388514095769557147190364152027721127070187421548123931953220997506553022646844227700020589045922742423904937051507367764629844971682121994198274794049092601715727439368569721862936007387077810797440975556627807371228030350048829843919546433753355787895064018998685060281902452191177018634505171087023903398550540704454189088472042376499749035038518949505897971286631644699407490959473411581934618336692169573605081585080837952036335619947691937965065016808710250735070825260046821242820434367245824478859256555487861614478717581068572356895150707602217433511627331709472765932413249132702425519391509083601346239612335001086614623850633127072987745618984384288764099836164964775714638573247333226653894523588365972955159905187411779288608760239306160016168434070611663449248395156319152882728822831375458678269830696691220130954815935450754923554167766876455212545681242936427474153815692219503331560151614492247512488957534835926226263545406704767033866410025277276800886383266629488582740369655329362236090572479794734434077704284318507901973469071141230364111729224929307731939309795452877412451183953480382210373644697046967493042810911797232448615413264031578430955396671061468083815548947146733652483679138566431084747848676243012018489329109615281108087617422779131629345494425395422727309645057976122885347393189600810965202090151104579377602529543130188938184010247010134929317443562883578609861545691161669857388024973756940558138630581099823372565164920155443216861690537054630176154809626620800633059320775897175589925862195462096455464624399535391743228225433267174308492508396461328929584567927365409119947616225155964704061297047759818551878441419948614013153859322060745185909608884280218943358691959604936409651570327527570641500776261323783648149005245481413195989296398441371781402764122087644989688629798910870164270169014007825748311598976330612951195680427485317886333041169767175063822135213839779138443325644288490872919067009802496281560626258636942322658490628628035057282983101266919109637258378149363774960594515216932644945188292639525772348420077356021656909077097264985642831778694777804964343991762549216500608626285329471055602670413384500507827390640287529864161287496473708235188892189612641279553536442286955430551308700009878557534223100547153412810957024870812654319123261956462149376527526356402127388765103883255007364899937167183280028398832319373301564123277185395654932422977953016534830128490677845037490891749347389015649588574802194996722621185874361039774946338633057887487405540005440439344888192044102134790034598411927024921557026873700970995205391930979319495883265922171508324621942300185974396706491149559411733728199869021311629886680267446443489233020607003821262841723679627307191405008084085703978151998148822390059948911946474438682533745889962375133378280532928272016815977970066488394482446332210928320504045983008943565954267256879714918703447338237767914829203283196838105907715727191903042365315650957464549643425328069510396558733549803850995143463506175361480050195045201350200180281506933241918267855737764414097080945745624854867704904368368717590918057269794010465019484853146726642978667687697789291431128505043098192949736165944259471754765135205245072597538577958372797702972231435199958499522344049394502115428867244188717409524554771867484911475031801773304689909317974472957035192387686405544278134169807249382219749124257510162187439772902147704638010731470653154201300583810458905006764557332998149945854655105526374914354195867992595981412218735238407957416123372264063860431988936249867649693592569592128495906254446474331759999685163660305216426770428154681777589339252115538590526823311608302751194384823861552852465010329467297198112105314125898165100120742688143577590825227466863206188376830450921784582526239594189673003640808624233657620979111641766331328852352062487922978959456450333733139422384778582717195412347860434376165241568717943562570215636666680088531006728947033079540804583324192188488870712275670333173939262509073556164513677064199539111948881240659821685787131385056850623094155206877987539740658484250135205615103489821873770245063583314243624807432542464195984647411575625441010389671576677263196442524931941806472423789334668561083789808830313571333157729435664956078125304917594015895146954965223118559669048559467607968190167266634650186182955669893965019614544401768162810604465068448139561667220729261210164692339016793399632833013163850830967942792934551268435760356901970523138364640961311774904600772840862214747547653221505518116489887879087780918009050706040061220010051271575991225725282523378026809030528461581739558198122397010092017202251606352922464781615533532275453264543087093320924631855976580561717446840450048285353396546862678852330044967795580761661801833668792312510460809773895565488962815089519622093675058841609752282328250433712970186608193748968699961301486924694482420723632912367052542145464162968910442981633373266871675946715392611950649224725627254543274193495995569590243279097174392258098103601486364409101491734183079646345064833303404765711827040276868271418084574998493392039317445402616663674646668754385093967129918067471909885312710726724428584870694307099756567949198418996425748884764622030325637751112534060087936904565779272035205921345924272965206683338510673615276261016026647772485083344719891986802656197236420847504962661607797092906844757798251795569758235084371746103310387911789239441630112634077535773520558040066982523191225570519133631407211349723226549151062961739050617857127509403623146700931176133132018631158730886798239298009805089491510788371194099750375473674305745187265414016446924576792185753680363289139664155342066705623272936001177781498886100830877849571709880858667023104043242526785955562077310543072298032125941107957349146684680220501816192150766649106862033378713826058987655210423668198670177861672671972374156917880001690656659046965316154923604061891820982414006103779407166342002735828911994182647812782659666207030384795881442790246669264032799404016800137293477301530941805070587421153284642203006550763966756168318897005152026656649929417382840327305940740147117478464839241225676523593418554066440983706083636457657081801664285044258224551650808864421212113914352453935225522162483791737330329812349528984098613273709957407786789349311975204237925022851375880436791854547836416773151821457226504640800104202100410766027807729152555503218182387221708112766208665317651926458452495269685376314437998340336947124447247796973890514941120010934140073794061859447165516612674930799374705772930521750426383798367668159183589049652163726492960837147204067428996276720315410211504333742057182854090136325721437592054640471894328548696883599785122262130812989581571391597464534806099601555877223193450760315411663112963843719400333736013305526352571490454327925190794007111504785378036370897340146753465517470747096935814912797188187854376797751675927822300312945518595042883902735494672667647506072643698761394806879080593531793001711000214417701504495496412454361656210150919997862972495905809191825255486358703529320142005857057855419217730505342687533799076038746689684283402648733290888881745453047194740939258407362058242849349024756883352446212456101562729065130618520732925434179252299417447855189995098959999877410951464170076989305620163502192692653166599093238118295411937545448509428621839424186218067457128099385258842631930670182098008050900019819621758458932516877698594110522845465835679362969619219080897536813210484518784516230623911878024604050824909336069998094776253792973597037759066145994638578378211017122446355845171941670344732162722443265914858595797823752976323442911242311368603724514438765801271594060878788638511089680883165505046309006148832545452819908256238805872042843941834687865142541377686054291079721004271658

lib/std/compress/deflate/testdata/compress-gettysburg.txt created+29

...	@@ -0,0 +1,29 @@
1	Four score and seven years ago our fathers brought forth on
2	this continent, a new nation, conceived in Liberty, and dedicated
3	to the proposition that all men are created equal.
4	Now we are engaged in a great Civil War, testing whether that
5	nation, or any nation so conceived and so dedicated, can long
6	endure.
7	We are met on a great battle-field of that war.
8	We have come to dedicate a portion of that field, as a final
9	resting place for those who here gave their lives that that
10	nation might live. It is altogether fitting and proper that
11	we should do this.
12	But, in a larger sense, we can not dedicate - we can not
13	consecrate - we can not hallow - this ground.
14	The brave men, living and dead, who struggled here, have
15	consecrated it, far above our poor power to add or detract.
16	The world will little note, nor long remember what we say here,
17	but it can never forget what they did here.
18	It is for us the living, rather, to be dedicated here to the
19	unfinished work which they who fought here have thus far so
20	nobly advanced. It is rather for us to be here dedicated to
21	the great task remaining before us - that from these honored
22	dead we take increased devotion to that cause for which they
23	gave the last full measure of devotion -
24	that we here highly resolve that these dead shall not have
25	died in vain - that this nation, under God, shall have a new
26	birth of freedom - and that government of the people, by the
27	people, for the people, shall not perish from this earth.
28
29	Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania

lib/std/compress/deflate/testdata/compress-pi.txt created+1

...	@@ -0,0 +1 @@
1	3.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678566722796619885782794848855834397518744545512965634434803966420557982936804352202770984294232533022576341807039476994159791594530069752148293366555661567873640053666564165473217043903521329543529169414599041608753201868379370234888689479151071637852902345292440773659495630510074210871426134974595615138498713757047101787957310422969066670214498637464595280824369445789772330048764765241339075920434019634039114732023380715095222010682563427471646024335440051521266932493419673977041595683753555166730273900749729736354964533288869844061196496162773449518273695588220757355176651589855190986665393549481068873206859907540792342402300925900701731960362254756478940647548346647760411463233905651343306844953979070903023460461470961696886885014083470405460742958699138296682468185710318879065287036650832431974404771855678934823089431068287027228097362480939962706074726455399253994428081137369433887294063079261595995462624629707062594845569034711972996409089418059534393251236235508134949004364278527138315912568989295196427287573946914272534366941532361004537304881985517065941217352462589548730167600298865925786628561249665523533829428785425340483083307016537228563559152534784459818313411290019992059813522051173365856407826484942764411376393866924803118364453698589175442647399882284621844900877769776312795722672655562596282542765318300134070922334365779160128093179401718598599933849235495640057099558561134980252499066984233017350358044081168552653117099570899427328709258487894436460050410892266917835258707859512983441729535195378855345737426085902908176515578039059464087350612322611200937310804854852635722825768203416050484662775045003126200800799804925485346941469775164932709504934639382432227188515974054702148289711177792376122578873477188196825462981268685817050740272550263329044976277894423621674119186269439650671515779586756482399391760426017633870454990176143641204692182370764887834196896861181558158736062938603810171215855272668300823834046564758804051380801633638874216371406435495561868964112282140753302655100424104896783528588290243670904887118190909494533144218287661810310073547705498159680772009474696134360928614849417850171807793068108546900094458995279424398139213505586422196483491512639012803832001097738680662877923971801461343244572640097374257007359210031541508936793008169980536520276007277496745840028362405346037263416554259027601834840306811381855105979705664007509426087885735796037324514146786703688098806097164258497595138069309449401515422221943291302173912538355915031003330325111749156969174502714943315155885403922164097229101129035521815762823283182342548326111912800928252561902052630163911477247331485739107775874425387611746578671169414776421441111263583553871361011023267987756410246824032264834641766369806637857681349204530224081972785647198396308781543221166912246415911776732253264335686146186545222681268872684459684424161078540167681420808850280054143613146230821025941737562389942075713627516745731891894562835257044133543758575342698699472547031656613991999682628247270641336222178923903176085428943733935618891651250424404008952719837873864805847268954624388234375178852014395600571048119498842390606136957342315590796703461491434478863604103182350736502778590897578272731305048893989009923913503373250855982655867089242612429473670193907727130706869170926462548423240748550366080136046689511840093668609546325002145852930950000907151058236267293264537382104938724996699339424685516483261134146110680267446637334375340764294026682973865220935701626384648528514903629320199199688285171839536691345222444708045923966028171565515656661113598231122506289058549145097157553900243931535190902107119457300243880176615035270862602537881797519478061013715004489917210022201335013106016391541589578037117792775225978742891917915522417189585361680594741234193398420218745649256443462392531953135103311476394911995072858430658361935369329699289837914941939406085724863968836903265564364216644257607914710869984315733749648835292769328220762947282381537409961545598798259891093717126218283025848112389011968221429457667580718653806506487026133892822994972574530332838963818439447707794022843598834100358385423897354243956475556840952248445541392394100016207693636846776413017819659379971557468541946334893748439129742391433659360410035234377706588867781139498616478747140793263858738624732889645643598774667638479466504074111825658378878454858148962961273998413442726086061872455452360643153710112746809778704464094758280348769758948328241239292960582948619196670918958089833201210318430340128495116203534280144127617285830243559830032042024512072872535581195840149180969253395075778400067465526031446167050827682772223534191102634163157147406123850425845988419907611287258059113935689601431668283176323567325417073420817332230462987992804908514094790368878687894930546955703072619009502076433493359106024545086453628935456862958531315337183868265617862273637169757741830239860065914816164049449650117321313895747062088474802365371031150898427992754426853277974311395143574172219759799359685252285745263796289612691572357986620573408375766873884266405990993505000813375432454635967504844235284874701443545419576258473564216198134073468541117668831186544893776979566517279662326714810338643913751865946730024434500544995399742372328712494834706044063471606325830649829795510109541836235030309453097335834462839476304775645015008507578949548931393944899216125525597701436858943585877526379625597081677643800125436502371412783467926101995585224717220177723700417808419423948725406801556035998390548985723546745642390585850216719031395262944554391316631345308939062046784387785054239390524731362012947691874975191011472315289326772533918146607300089027768963114810902209724520759167297007850580717186381054967973100167870850694207092232908070383263453452038027860990556900134137182368370991949516489600755049341267876436746384902063964019766685592335654639138363185745698147196210841080961884605456039038455343729141446513474940784884423772175154334260306698831768331001133108690421939031080143784334151370924353013677631084913516156422698475074303297167469640666531527035325467112667522460551199581831963763707617991919203579582007595605302346267757943936307463056901080114942714100939136913810725813781357894005599500183542511841721360557275221035268037357265279224173736057511278872181908449006178013889710770822931002797665935838758909395688148560263224393726562472776037890814458837855019702843779362407825052704875816470324581290878395232453237896029841669225489649715606981192186584926770403956481278102179913217416305810554598801300484562997651121241536374515005635070127815926714241342103301566165356024733807843028655257222753049998837015348793008062601809623815161366903341111386538510919367393835229345888322550887064507539473952043968079067086806445096986548801682874343786126453815834280753061845485903798217994599681154419742536344399602902510015888272164745006820704193761584547123183460072629339550548239557137256840232268213012476794522644820910235647752723082081063518899152692889108455571126603965034397896278250016110153235160519655904211844949907789992007329476905868577878720982901352956613978884860509786085957017731298155314951681467176959760994210036183559138777817698458758104466283998806006162298486169353373865787735983361613384133853684211978938900185295691967804554482858483701170967212535338758621582310133103877668272115726949518179589754693992642197915523385766231676275475703546994148929041301863861194391962838870543677743224276809132365449485366768000001065262485473055861598999140170769838548318875014293890899506854530765116803337322265175662207526951791442252808165171667766727930354851542040238174608923283917032754257508676551178593950027933895920576682789677644531840404185540104351348389531201326378369283580827193783126549617459970567450718332065034556644034490453627560011250184335607361222765949278393706478426456763388188075656121689605041611390390639601620221536849410926053876887148379895599991120991646464411918568277004574243434021672276445589330127781586869525069499364610175685060167145354315814801054588605645501332037586454858403240298717093480910556211671546848477803944756979804263180991756422809873998766973237695737015808068229045992123661689025962730430679316531149401764737693873514093361833216142802149763399189835484875625298752423873077559555955465196394401821840998412489826236737714672260616336432964063357281070788758164043814850188411431885988276944901193212968271588841338694346828590066640806314077757725705630729400492940302420498416565479736705485580445865720227637840466823379852827105784319753541795011347273625774080213476826045022851579795797647467022840999561601569108903845824502679265942055503958792298185264800706837650418365620945554346135134152570065974881916341359556719649654032187271602648593049039787489589066127250794828276938953521753621850796297785146188432719223223810158744450528665238022532843891375273845892384422535472653098171578447834215822327020690287232330053862163479885094695472004795231120150432932266282727632177908840087861480221475376578105819702226309717495072127248479478169572961423658595782090830733233560348465318730293026659645013718375428897557971449924654038681799213893469244741985097334626793321072686870768062639919361965044099542167627840914669856925715074315740793805323925239477557441591845821562518192155233709607483329234921034514626437449805596103307994145347784574699992128599999399612281615219314888769388022281083001986016549416542616968586788372609587745676182507275992950893180521872924610867639958916145855058397274209809097817293239301067663868240401113040247007350857828724627134946368531815469690466968693925472519413992914652423857762550047485295476814795467007050347999588867695016124972282040303995463278830695976249361510102436555352230690612949388599015734661023712235478911292547696176005047974928060721268039226911027772261025441492215765045081206771735712027180242968106203776578837166909109418074487814049075517820385653909910477594141321543284406250301802757169650820964273484146957263978842560084531214065935809041271135920041975985136254796160632288736181367373244506079244117639975974619383584574915988097667447093006546342423460634237474666080431701260052055928493695941434081468529815053947178900451835755154125223590590687264878635752541911288877371766374860276606349603536794702692322971868327717393236192007774522126247518698334951510198642698878471719396649769070825217423365662725928440620430214113719922785269984698847702323823840055655517889087661360130477098438611687052310553149162517283732728676007248172987637569816335415074608838663640693470437206688651275688266149730788657015685016918647488541679154596507234287730699853713904300266530783987763850323818215535597323530686043010675760838908627049841888595138091030423595782495143988590113185835840667472370297149785084145853085781339156270760356390763947311455495832266945702494139831634332378975955680856836297253867913275055542524491943589128405045226953812179131914513500993846311774017971512283785460116035955402864405902496466930707769055481028850208085800878115773817191741776017330738554758006056014337743299012728677253043182519757916792969965041460706645712588834697979642931622965520168797300035646304579308840327480771811555330909887025505207680463034608658165394876951960044084820659673794731680864156456505300498816164905788311543454850526600698230931577765003780704661264706021457505793270962047825615247145918965223608396645624105195510522357239739512881816405978591427914816542632892004281609136937773722299983327082082969955737727375667615527113922588055201898876201141680054687365580633471603734291703907986396522961312801782679717289822936070288069087768660593252746378405397691848082041021944719713869256084162451123980620113184541244782050110798760717155683154078865439041210873032402010685341947230476666721749869868547076781205124736792479193150856444775379853799732234456122785843296846647513336573692387201464723679427870042503255589926884349592876124007558756946413705625140011797133166207153715436006876477318675587148783989081074295309410605969443158477539700943988394914432353668539209946879645066533985738887866147629443414010498889931600512076781035886116602029611936396821349607501116498327856353161451684576956871090029997698412632665023477167286573785790857466460772283415403114415294188047825438761770790430001566986776795760909966936075594965152736349811896413043311662774712338817406037317439705406703109676765748695358789670031925866259410510533584384656023391796749267844763708474978333655579007384191473198862713525954625181604342253729962863267496824058060296421146386436864224724887283434170441573482481833301640566959668866769563491416328426414974533349999480002669987588815935073578151958899005395120853510357261373640343675347141048360175464883004078464167452167371904831096767113443494819262681110739948250607394950735031690197318521195526356325843390998224986240670310768318446607291248747540316179699411397387765899868554170318847788675929026070043212666179192235209382278788809886335991160819235355570464634911320859189796132791319756490976000139962344455350143464268604644958624769094347048293294140411146540923988344435159133201077394411184074107684981066347241048239358274019449356651610884631256785297769734684303061462418035852933159734583038455410337010916767763742762102137013548544509263071901147318485749233181672072137279355679528443925481560913728128406333039373562420016045664557414588166052166608738748047243391212955877763906969037078828527753894052460758496231574369171131761347838827194168606625721036851321566478001476752310393578606896111259960281839309548709059073861351914591819510297327875571049729011487171897180046961697770017913919613791417162707018958469214343696762927459109940060084983568425201915593703701011049747339493877885989417433031785348707603221982970579751191440510994235883034546353492349826883624043327267415540301619505680654180939409982020609994140216890900708213307230896621197755306659188141191577836272927461561857103721724710095214236964830864102592887457999322374955191221951903424452307535133806856807354464995127203174487195403976107308060269906258076020292731455252078079914184290638844373499681458273372072663917670201183004648190002413083508846584152148991276106513741539435657211390328574918769094413702090517031487773461652879848235338297260136110984514841823808120540996125274580881099486972216128524897425555516076371675054896173016809613803811914361143992106380050832140987604599309324851025168294467260666138151745712559754953580239983146982203613380828499356705575524712902745397762140493182014658008021566536067765508783804304134310591804606800834591136640834887408005741272586704792258319127415739080914383138456424150940849133918096840251163991936853225557338966953749026620923261318855891580832455571948453875628786128859004106006073746501402627824027346962528217174941582331749239683530136178653673760642166778137739951006589528877427662636841830680190804609849809469763667335662282915132352788806157768278159588669180238940333076441912403412022316368577860357276941541778826435238131905028087018575047046312933353757285386605888904583111450773942935201994321971171642235005644042979892081594307167019857469273848653833436145794634175922573898588001698014757420542995801242958105456510831046297282937584161162532562516572498078492099897990620035936509934721582965174135798491047111660791587436986541222348341887722929446335178653856731962559852026072947674072616767145573649812105677716893484917660771705277187601199908144113058645577910525684304811440261938402322470939249802933550731845890355397133088446174107959162511714864874468611247605428673436709046678468670274091881014249711149657817724279347070216688295610877794405048437528443375108828264771978540006509704033021862556147332117771174413350281608840351781452541964320309576018694649088681545285621346988355444560249556668436602922195124830910605377201980218310103270417838665447181260397190688462370857518080035327047185659499476124248110999288679158969049563947624608424065930948621507690314987020673533848349550836366017848771060809804269247132410009464014373603265645184566792456669551001502298330798496079949882497061723674493612262229617908143114146609412341593593095854079139087208322733549572080757165171876599449856937956238755516175754380917805280294642004472153962807463602113294255916002570735628126387331060058910652457080244749375431841494014821199962764531068006631183823761639663180931444671298615527598201451410275600689297502463040173514891945763607893528555053173314164570504996443890936308438744847839616840518452732884032345202470568516465716477139323775517294795126132398229602394548579754586517458787713318138752959809412174227300352296508089177705068259248822322154938048371454781647213976820963320508305647920482085920475499857320388876391601995240918938945576768749730856955958010659526503036266159750662225084067428898265907510637563569968211510949669744580547288693631020367823250182323708459790111548472087618212477813266330412076216587312970811230758159821248639807212407868878114501655825136178903070860870198975889807456643955157415363193191981070575336633738038272152798849350397480015890519420879711308051233933221903466249917169150948541401871060354603794643379005890957721180804465743962806186717861017156740967662080295766577051291209907944304632892947306159510430902221439371849560634056189342513057268291465783293340524635028929175470872564842600349629611654138230077313327298305001602567240141851520418907011542885799208121984493156999059182011819733500126187728036812481995877070207532406361259313438595542547781961142935163561223496661522614735399674051584998603552953329245752388810136202347624669055816438967863097627365504724348643071218494373485300606387644566272186661701238127715621379746149861328744117714552444708997144522885662942440230184791205478498574521634696448973892062401943518310088283480249249085403077863875165911302873958787098100772718271874529013972836614842142871705531796543076504534324600536361472618180969976933486264077435199928686323835088756683595097265574815431940195576850437248001020413749831872259677387154958399718444907279141965845930083942637020875635398216962055324803212267498911402678528599673405242031091797899905718821949391320753431707980023736590985375520238911643467185582906853711897952626234492483392496342449714656846591248918556629589329909035239233333647435203707701010843880032907598342170185542283861617210417603011645918780539367447472059985023582891833692922337323999480437108419659473162654825748099482509991833006976569367159689364493348864744213500840700660883597235039532340179582557036016936990988671132109798897070517280755855191269930673099250704070245568507786790694766126298082251633136399521170984528092630375922426742575599892892783704744452189363203489415521044597261883800300677617931381399162058062701651024458869247649246891924612125310275731390840470007143561362316992371694848132554200914530410371354532966206392105479824392125172540132314902740585892063217589494345489068463993137570910346332714153162232805522972979538018801628590735729554162788676498274186164218789885741071649069191851162815285486794173638906653885764229158342500673612453849160674137340173572779956341043326883569507814931378007362354180070619180267328551191942676091221035987469241172837493126163395001239599240508454375698507957046222664619000103500490183034153545842833764378111988556318777792537201166718539541835984438305203762819440761594106820716970302285152250573126093046898423433152732131361216582808075212631547730604423774753505952287174402666389148817173086436111389069420279088143119448799417154042103412190847094080254023932942945493878640230512927119097513536000921971105412096683111516328705423028470073120658032626417116165957613272351566662536672718998534199895236884830999302757419916463841427077988708874229277053891227172486322028898425125287217826030500994510824783572905691988555467886079462805371227042466543192145281760741482403827835829719301017888345674167811398954750448339314689630763396657226727043393216745421824557062524797219978668542798977992339579057581890622525473582205236424850783407110144980478726691990186438822932305382318559732869780922253529591017341407334884761005564018242392192695062083183814546983923664613639891012102177095976704908305081854704194664371312299692358895384930136356576186106062228705599423371631021278457446463989738188566746260879482018647487672727222062676465338099801966883680994159075776852639865146253336312450536402610569605513183813174261184420189088853196356986962795036738424313011331753305329802016688817481342988681585577810343231753064784983210629718425184385534427620128234570716988530518326179641178579608888150329602290705614476220915094739035946646916235396809201394578175891088931992112260073928149169481615273842736264298098234063200244024495894456129167049508235812487391799648641133480324757775219708932772262349486015046652681439877051615317026696929704928316285504212898146706195331970269507214378230476875280287354126166391708245925170010714180854800636923259462019002278087409859771921805158532147392653251559035410209284665925299914353791825314545290598415817637058927906909896911164381187809435371521332261443625314490127454772695739393481546916311624928873574718824071503995009446731954316193855485207665738825139639163576723151005556037263394867208207808653734942440115799667507360711159351331959197120948964717553024531364770942094635696982226673775209945168450643623824211853534887989395673187806606107885440005508276570305587448541805778891719207881423351138662929667179643468760077047999537883387870348718021842437342112273940255717690819603092018240188427057046092622564178375265263358324240661253311529423457965569502506810018310900411245379015332966156970522379210325706937051090830789479999004999395322153622748476603613677697978567386584670936679588583788795625946464891376652199588286933801836011932368578558558195556042156250883650203322024513762158204618106705195330653060606501054887167245377942831338871631395596905832083416898476065607118347136218123246227258841990286142087284956879639325464285343075301105285713829643709990356948885285190402956047346131138263878897551788560424998748316382804046848618938189590542039889872650697620201995548412650005394428203930127481638158530396439925470201672759328574366661644110962566337305409219519675148328734808957477775278344221091073111351828046036347198185655572957144747682552857863349342858423118749440003229690697758315903858039353521358860079600342097547392296733310649395601812237812854584317605561733861126734780745850676063048229409653041118306671081893031108871728167519579675347188537229309616143204006381322465841111157758358581135018569047815368938137718472814751998350504781297718599084707621974605887423256995828892535041937958260616211842368768511418316068315867994601652057740529423053601780313357263267054790338401257305912339601880137825421927094767337191987287385248057421248921183470876629667207272325650565129333126059505777727542471241648312832982072361750574673870128209575544305968395555686861188397135522084452852640081252027665557677495969626612604565245684086139238265768583384698499778726706555191854468698469478495734622606294219624557085371272776523098955450193037732166649182578154677292005212667143463209637891852323215018976126034373684067194193037746880999296877582441047878123266253181845960453853543839114496775312864260925211537673258866722604042523491087026958099647595805794663973419064010036361904042033113579336542426303561457009011244800890020801478056603710154122328891465722393145076071670643556827437743965789067972687438473076346451677562103098604092717090951280863090297385044527182892749689212106670081648583395537735919136950153162018908887484210798706899114804669270650940762046502772528650728905328548561433160812693005693785417861096969202538865034577183176686885923681488475276498468821949739729707737187188400414323127636504814531122850990020742409255859252926103021067368154347015252348786351643976235860419194129697690405264832347009911154242601273438022089331096686367898694977994001260164227609260823493041180643829138347354679725399262338791582998486459271734059225620749105308531537182911681637219395188700957788181586850464507699343940987433514431626330317247747486897918209239480833143970840673084079589358108966564775859905563769525232653614424780230826811831037735887089240613031336477371011628214614661679404090518615260360092521947218890918107335871964142144478654899528582343947050079830388538860831035719306002771194558021911942899922722353458707566246926177663178855144350218287026685610665003531050216318206017609217984684936863161293727951873078972637353717150256378733579771808184878458866504335824377004147710414934927438457587107159731559439426412570270965125108115548247939403597681188117282472158250109496096625393395380922195591918188552678062149923172763163218339896938075616855911752998450132067129392404144593862398809381240452191484831646210147389182510109096773869066404158973610476436500068077105656718486281496371118832192445663945814491486165500495676982690308911185687986929470513524816091743243015383684707292898982846022237301452655679898627767968091469798378268764311598832109043715611299766521539635464420869197567370005738764978437686287681792497469438427465256316323005551304174227341646455127812784577772457520386543754282825671412885834544435132562054464241011037955464190581168623059644769587054072141985212106734332410756767575818456990693046047522770167005684543969234041711089888993416350585157887353430815520811772071880379104046983069578685473937656433631979786803671873079693924236321448450354776315670255390065423117920153464977929066241508328858395290542637687668968805033317227800185885069736232403894700471897619347344308437443759925034178807972235859134245813144049847701732361694719765715353197754997162785663119046912609182591249890367654176979903623755286526375733763526969344354400473067198868901968147428767790866979688522501636949856730217523132529265375896415171479559538784278499866456302878831962099830494519874396369070682762657485810439112232618794059941554063270131989895703761105323606298674803779153767511583043208498720920280929752649812569163425000522908872646925284666104665392171482080130502298052637836426959733707053922789153510568883938113249757071331029504430346715989448786847116438328050692507766274500122003526203709466023414648998390252588830148678162196775194583167718762757200505439794412459900771152051546199305098386982542846407255540927403132571632640792934183342147090412542533523248021932277075355546795871638358750181593387174236061551171013123525633485820365146141870049205704372018261733194715700867578539336078622739558185797587258744102542077105475361294047460100094095444959662881486915903899071865980563617137692227290764197755177720104276496949611056220592502420217704269622154958726453989227697660310524980855759471631075870133208861463266412591148633881220284440694169488261529577625325019870359870674380469821942056381255833436421949232275937221289056420943082352544084110864545369404969271494003319782861318186188811118408257865928757426384450059944229568586460481033015388911499486935436030221810943466764000022362550573631294626296096198760564259963946138692330837196265954739234624134597795748524647837980795693198650815977675350553918991151335252298736112779182748542008689539658359421963331502869561192012298889887006079992795411188269023078913107603617634779489432032102773359416908650071932804017163840644987871753756781185321328408216571107549528294974936214608215583205687232185574065161096274874375098092230211609982633033915469494644491004515280925089745074896760324090768983652940657920198315265410658136823791984090645712468948470209357761193139980246813405200394781949866202624008902150166163813538381515037735022966074627952910384068685569070157516624192987244482719429331004854824454580718897633003232525821581280327467962002814762431828622171054352898348208273451680186131719593324711074662228508710666117703465352839577625997744672185715816126411143271794347885990892808486694914139097716736900277758502686646540565950394867841110790116104008572744562938425494167594605487117235946429105850909950214958793112196135908315882620682332156153086833730838173279328196983875087083483880463884784418840031847126974543709373298362402875197920802321878744882872843727378017827008058782410749357514889978911739746129320351081432703251409030487462262942344327571260086642508333187688650756429271605525289544921537651751492196367181049435317858383453865255656640657251363575064353236508936790431702597878177190314867963840828810209461490079715137717099061954969640070867667102330048672631475510537231757114322317411411680622864206388906210192355223546711662137499693269321737043105987225039456574924616978260970253359475020913836673772894438696400028110344026084712899000746807764844088711341352503367877316797709372778682166117865344231732264637847697875144332095340001650692130546476890985050203015044880834261845208730530973189492916425322933612431514306578264070283898409841602950309241897120971601649265613413433422298827909921786042679812457285345801338260995877178113102167340256562744007296834066198480676615805021691833723680399027931606420436812079900316264449146190219458229690992122788553948783538305646864881655562294315673128274390826450611628942803501661336697824051770155219626522725455850738640585299830379180350432876703809252167907571204061237596327685674845079151147313440001832570344920909712435809447900462494313455028900680648704293534037436032625820535790118395649089354345101342969617545249573960621490288728932792520696535386396443225388327522499605986974759882329916263545973324445163755334377492928990581175786355555626937426910947117002165411718219750519831787137106051063795558588905568852887989084750915764639074693619881507814685262133252473837651192990156109189777922008705793396463827490680698769168197492365624226087154176100430608904377976678519661891404144925270480881971498801542057787006521594009289777601330756847966992955433656139847738060394368895887646054983871478968482805384701730871117761159663505039979343869339119789887109156541709133082607647406305711411098839388095481437828474528838368079418884342666222070438722887413947801017721392281911992365405516395893474263953824829609036900288359327745855060801317988407162446563997948275783650195514221551339281978226984278638391679715091262410548725700924070045488485692950448110738087996547481568913935380943474556972128919827177020766613602489581468119133614121258783895577357194986317210844398901423948496659251731388171602663261931065366535041473070804414939169363262373767777095850313255990095762731957308648042467701212327020533742667053142448208168130306397378736642483672539837487690980602182785786216512738563513290148903509883270617258932575363993979055729175160097615459044771692265806315111028038436017374742152476085152099016158582312571590733421736576267142390478279587281505095633092802668458937649649770232973641319060982740633531089792464242134583740901169391964250459128813403498810635400887596820054408364386516617880557608956896727531538081942077332597917278437625661184319891025007491829086475149794003160703845549465385946027452447466812314687943441610993338908992638411847425257044572517459325738989565185716575961481266020310797628254165590506042479114016957900338356574869252800743025623419498286467914476322774005529460903940177536335655471931000175430047504719144899841040015867946179241610016454716551337074073950260442769538553834397550548871099785205401175169747581344926079433689543783221172450687344231989878844128542064742809735625807066983106979935260693392135685881391214807354728463227784908087002467776303605551232386656295178853719673034634701222939581606792509153217489030840886516061119011498443412350124646928028805996134283511884715449771278473361766285062169778717743824362565711779450064477718370221999106695021656757644044997940765037999954845002710665987813603802314126836905783190460792765297277694043613023051787080546511542469395265127101052927070306673024447125973939950514628404767431363739978259184541176413327906460636584152927019030276017339474866960348694976541752429306040727005059039503148522921392575594845078867977925253931765156416197168443524369794447355964260633391055126826061595726217036698506473281266724521989060549880280782881429796336696744124805982192146339565745722102298677599746738126069367069134081559412016115960190237753525556300606247983261249881288192937343476862689219239777833910733106588256813777172328315329082525092733047850724977139448333892552081175608452966590553940965568541706001179857293813998258319293679100391844099286575605993598910002969864460974714718470101531283762631146774209145574041815908800064943237855839308530828305476076799524357391631221886057549673832243195650655460852881201902363644712703748634421727257879503428486312944916318475347531435041392096108796057730987201352484075057637199253650470908582513936863463863368042891767107602111159828875539940120076013947033661793715396306139863655492213741597905119083588290097656647300733879314678913181465109316761575821351424860442292445304113160652700974330088499034675405518640677342603583409608605533747362760935658853109760994238347382222087292464497684560579562516765574088410321731345627735856052358236389532038534024842273371639123973215995440828421666636023296545694703577184873442034227706653837387506169212768015766181095420097708363604361110592409117889540338021426523948929686439808926114635414571535194342850721353453018315875628275733898268898523557799295727645229391567477566676051087887648453493636068278050564622813598885879259940946446041705204470046315137975431737187756039815962647501410906658866162180038266989961965580587208639721176995219466789857011798332440601811575658074284182910615193917630059194314434605154047710570054339000182453117733718955857603607182860506356479979004139761808955363669603162193113250223851791672055180659263518036251214575926238369348222665895576994660491938112486609099798128571823494006615552196112207203092277646200999315244273589488710576623894693889446495093960330454340842102462401048723328750081749179875543879387381439894238011762700837196053094383940063756116458560943129517597713935396074322792489221267045808183313764165818269562105872892447740035947009268662659651422050630078592002488291860839743732353849083964326147000532423540647042089499210250404726781059083644007466380020870126664209457181702946752278540074508552377720890581683918446592829417018288233014971554235235911774818628592967605048203864343108779562892925405638946621948268711042828163893975711757786915430165058602965217459581988878680408110328432739867198621306205559855266036405046282152306154594474489908839081999738747452969810776201487134000122535522246695409315213115337915798026979555710508507473874750758068765376445782524432638046143042889235934852961058269382103498000405248407084403561167817170512813378805705643450616119330424440798260377951198548694559152051960093041271007277849301555038895360338261929343797081874320949914159593396368110627557295278004254863060054523839151068998913578820019411786535682149118528207852130125518518493711503422159542244511900207393539627400208110465530207932867254740543652717595893500716336076321614725815407642053020045340183572338292661915308354095120226329165054426123619197051613839357326693760156914429944943744856809775696303129588719161129294681884936338647392747601226964158848900965717086160598147204467428664208765334799858222090619802173211614230419477754990738738567941189824660913091691772274207233367635032678340586301930193242996397204445179288122854478211953530898910125342975524727635730226281382091807439748671453590778633530160821559911314144205091447293535022230817193663509346865858656314855575862447818620108711889760652969899269328178705576435143382060141077329261063431525337182243385263520217735440715281898137698755157574546939727150488469793619500477720970561793913828989845327426227288647108883270173723258818244658436249580592560338105215606206155713299156084892064340303395262263451454283678698288074251422567451806184149564686111635404971897682154227722479474033571527436819409892050113653400123846714296551867344153741615042563256713430247655125219218035780169240326699541746087592409207004669340396510178134857835694440760470232540755557764728450751826890418293966113310160131119077398632462778219023650660374041606724962490137433217246454097412995570529142438208076098364823465973886691349919784013108015581343979194852830436739012482082444814128095443773898320059864909159505322857914576884962578665885999179867520554558099004556461178755249370124553217170194282884617402736649978475508294228020232901221630102309772151569446427909802190826689868834263071609207914085197695235553488657743425277531197247430873043619511396119080030255878387644206085044730631299277888942729189727169890575925244679660189707482960949190648764693702750773866432391919042254290235318923377293166736086996228032557185308919284403805071030064776847863243191000223929785255372375566213644740096760539439838235764606992465260089090624105904215453927904411529580345334500256244101006359530039598864466169595626351878060688513723462707997327233134693971456285542615467650632465676620279245208581347717608521691340946520307673391841147504140168924121319826881568664561485380287539331160232292555618941042995335640095786495340935115266454024418775949316930560448686420862757201172319526405023099774567647838488973464317215980626787671838005247696884084989185086149003432403476742686245952395890358582135006450998178244636087317754378859677672919526111213859194725451400301180503437875277664402762618941017576872680428176623860680477885242887430259145247073950546525135339459598789619778911041890292943818567205070964606263541732944649576612651953495701860015412623962286413897796733329070567376962156498184506842263690367849555970026079867996261019039331263768556968767029295371162528005543100786408728939225714512481135778627664902425161990277471090335933309304948380597856628844787441469841499067123764789582263294904679812089984857163571087831191848630254501620929805829208334813638405421720056121989353669371336733392464416125223196943471206417375491216357008573694397305979709719726666642267431117762176403068681310351899112271339724036887000996862922546465006385288620393800504778276912835603372548255793912985251506829969107754257647488325341412132800626717094009098223529657957997803018282428490221470748111124018607613415150387569830918652780658896682362523937845272634530420418802508442363190383318384550522367992357752929106925043261446950109861088899914658551881873582528164302520939285258077969737620845637482114433988162710031703151334402309526351929588680690821355853680161000213740851154484912685841268695899174149133820578492800698255195740201818105641297250836070356851055331787840829000041552511865779453963317538532092149720526607831260281961164858098684587525129997404092797683176639914655386108937587952214971731728131517932904431121815871023518740757222100123768721944747209349312324107065080618562372526732540733324875754482967573450019321902199119960797989373383673242576103938985349278777473980508080015544764061053522202325409443567718794565430406735896491017610775948364540823486130254718476485189575836674399791508512858020607820554462991723202028222914886959399729974297471155371858924238493855858595407438104882624648788053304271463011941589896328792678327322456103852197011130466587100500083285177311776489735230926661234588873102883515626446023671996644554727608310118788389151149340939344750073025855814756190881398752357812331342279866503522725367171230756861045004548970360079569827626392344107146584895780241408158405229536937499710665594894459246286619963556350652623405339439142111271810691052290024657423604130093691889255865784668461215679554256605416005071276641766056874274200329577160643448606201239821698271723197826816628249938714995449137302051843669076723577400053932662622760323659751718925901801104290384274185507894887438832703063283279963007200698012244365116394086922220745320244624121155804354542064215121585056896157356414313068883443185280853975927734433655384188340303517822946253702015782157373265523185763554098954033236382319219892171177449469403678296185920803403867575834111518824177439145077366384071880489358256868542011645031357633355509440319236720348651010561049872726472131986543435450409131859513145181276437310438972507004981987052176272494065214619959232142314439776546708351714749367986186552791715824080651063799500184295938799158350171580759883784962257398512129810326379376218322456594236685376799113140108043139732335449090824910499143325843298821033984698141715756010829706583065211347076803680695322971990599904451209087275776225351040902392888779424630483280319132710495478599180196967835321464441189260631526618167443193550817081875477050802654025294109218264858213857526688155584113198560022135158887210365696087515063187533002942118682221893775546027227291290504292259787710667873840000616772154638441292371193521828499824350920891801685572798156421858191197490985730570332667646460728757430565372602768982373259745084479649545648030771598153955827779139373601717422996027353102768719449444917939785144631597314435351850491413941557329382048542123508173912549749819308714396615132942045919380106231421774199184060180347949887691051557905554806953878540066453375981862846419905220452803306263695626490910827627115903856995051246529996062855443838330327638599800792922846659503551211245284087516229060262011857775313747949362055496401073001348853150735487353905602908933526400713274732621960311773433943673385759124508149335736911664541281788171454023054750667136518258284898099512139193995633241336556777098003081910272040997148687418134667006094051021462690280449159646545330107754695413088714165312544813061192407821188690056027781824235022696189344352547633573536485619363254417756613981703930632872166905722259745209192917262199844409646158269456380239502837121686446561785235565164127712826918688615572716201474934052276946595712198314943381622114006936307430444173284786101777743837977037231795255434107223445512555589998646183876764903972461167959018100035098928641204195163551108763204267612979826529425882951141275841262732790798807559751851576841264742209479721843309352972665210015662514552994745127631550917636730259462132930190402837954246323258550301096706922720227074863419005438302650681214142135057154175057508639907673946335146209082888934938376439399256900604067311422093312195936202982972351163259386772241477911629572780752395056251581603133359382311500518626890530658368129988108663263271980611271548858798093487912913707498230575929091862939195014721197586067270092547718025750337730799397134539532646195269996596385654917590458333585799102012713204583903200853878881633637685182083727885131175227769609787962142372162545214591281831798216044111311671406914827170981015457781939202311563871950805024679725792497605772625913328559726371211201905720771409148645074094926718035815157571514050397610963846755569298970383547314100223802583468767350129775413279532060971154506484212185936490997917766874774481882870632315515865032898164228288232746866106592732197907162384642153489852476216789050260998045266483929542357287343977680495774091449538391575565485459058976495198513801007958010783759945775299196700547602252552034453988712538780171960718164078124847847257912407824544361682345239570689514272269750431873633263011103053423335821609333191218806608268341428910415173247216053355849993224548730778822905252324234861531520976938461042582849714963475341837562003014915703279685301868631572488401526639835689563634657435321783493199825542117308467745297085839507616458229630324424328237737450517028560698067889521768198156710781633405266759539424926280756968326107495323390536223090807081455919837355377748742029039018142937311529334644468151212945097596534306284215319445727118614900017650558177095302468875263250119705209476159416768727784472000192789137251841622857783792284439084301181121496366424659033634194540657183544771912446621259392656620306888520055599121235363718226922531781458792593750441448933981608657900876165024635197045828895481793756681046474614105142498870252139936870509372305447734112641354892806841059107716677821238332810262185587751312721179344448201440425745083063944738363793906283008973306241380614589414227694747931665717623182472168350678076487573420491557628217583972975134478990696589532548940335615613167403276472469212505759116251529654568544633498114317670257295661844775487469378464233737238981920662048511894378868224807279352022501796545343757274163910791972952950812942922205347717304184477915673991738418311710362524395716152714669005814700002633010452643547865903290733205468338872078735444762647925297690170912007874183736735087713376977683496344252419949951388315074877537433849458259765560996555954318040920178497184685497370696212088524377013853757681416632722412634423982152941645378000492507262765150789085071265997036708726692764308377229685985169122305037462744310852934305273078865283977335246017463527703205938179125396915621063637625882937571373840754406468964783100704580613446731271591194608435935825987782835266531151065041623295329047772174083559349723758552138048305090009646676088301540612824308740645594431853413755220166305812111033453120745086824339432159043594430312431227471385842030390106070940315235556172767994160020393975099897629335325855575624808996691829864222677502360193257974726742578211119734709402357457222271212526852384295874273501563660093188045493338989741571490544182559738080871565281430102670460284316819230392535297795765862414392701549740879273131051636119137577008929564823323648298263024607975875767745377160102490804624301856524161756655600160859121534556267602192689982855377872583145144082654583484409478463178777374794653580169960779405568701192328608041130904629350871827125934668712766694873899824598527786499569165464029458935064964335809824765965165142090986755203808309203230487342703468288751604071546653834619611223013759451579252696743642531927390036038608236450762698827497618723575476762889950752114804852527950845033958570838130476937881321123674281319487950228066320170022460331989671970649163741175854851878484012054844672588851401562725019821719066960812627785485964818369621410721714214986361918774754509650308957099470934337856981674465828267911940611956037845397855839240761276344105766751024307559814552786167815949657062559755074306521085301597908073343736079432866757890533483669555486803913433720156498834220893399971641479746938696905480089193067138057171505857307148815649920714086758259602876056459782423770242469805328056632787041926768467116266879463486950464507420219373945259262668613552940624781361206202636498199999498405143868285258956342264328707663299304891723400725471764188685351372332667877921738347541480022803392997357936152412755829569276837231234798989446274330454566790062032420516396282588443085438307201495672106460533238537203143242112607424485845094580494081820927639140008540422023556260218564348994145439950410980591817948882628052066441086319001688568155169229486203010738897181007709290590480749092427141018933542818429995988169660993836961644381528877214085268088757488293258735809905670755817017949161906114001908553744882726200936685604475596557476485674008177381703307380305476973609786543859382187220583902344443508867499866506040645874346005331827436296177862518081893144363251205107094690813586440519229512932450078833398788429339342435126343365204385812912834345297308652909783300671261798130316794385535726296998740359570458452230856390098913179475948752126397078375944861139451960286751210561638976008880092746115860800207803341591451797073036835196977766076373785333012024120112046988609209339085365773222392412449051532780950955866459477634482269986074813297302630975028812103517723124465095349653693090018637764094094349837313251321862080214809922685502948454661814715557444709669530177690434272031892770604717784527939160472281534379803539679861424370956683221491465438014593829277393396032754048009552231816667380357183932757077142046723838624617803976292377131209580789363841447929802588065522129262093623930637313496640186619510811583471173312025805866727639992763579078063818813069156366274125431259589936119647626101405563503399523140323113819656236327198961837254845333702062563464223952766943568376761368711962921818754576081617053031590728828700712313666308722754918661395773730546065997437810987649802414011242142773668082751390959313404155826266789510846776118665957660165998178089414985754976284387856100263796543178313634025135814161151902096499133548733131115022700681930135929595971640197196053625033558479980963488718039111612813595968565478868325856437896173159762002419621552896297904819822199462269487137462444729093456470028537694958859591606789282491054412515996300781368367490209374915732896270028656829344431342347351239298259166739503425995868970697267332582735903121288746660451461487850346142827765991608090398652575717263081833494441820193533385071292345774375579344062178711330063106003324053991693682603746176638565758877580201229366353270267100681261825172914608202541892885935244491070138206211553827793565296914576502048643282865557934707209634807372692141186895467322767751335690190153723669036865389161291688887876407525493494249733427181178892759931596719354758988097924525262363659036320070854440784544797348291802082044926670634420437555325050527522833778887040804033531923407685630109347772125639088640413101073817853338316038135280828119040832564401842053746792992622037698718018061122624490909242641985820861751177113789051609140381575003366424156095216328197122335023167422600567941281406217219641842705784328959802882335059828208196666249035857789940333152274817776952843681630088531769694783690580671064828083598046698841098135158654906933319522394363287923990534810987830274500172065433699066117784554364687723631844464768069142828004551074686645392805399409108754939166095731619715033166968309929466349142798780842257220697148875580637480308862995118473187124777291910070227588893486939456289515802965372150409603107761289831263589964893410247036036645058687287589051406841238124247386385427908282733827973326885504935874303160274749063129572349742611221517417153133618622410913869500688835898962349276317316478340077460886655598733382113829928776911495492184192087771606068472874673681886167507221017261103830671787856694812948785048943063086169948798703160515884108282351274153538513365895332948629494495061868514779105804696039069372662670386512905201137810858616188886947957607413585534585151768051973334433495230120395770739623771316030242887200537320998253008977618973129817881944671731160647231476248457551928732782825127182446807824215216469567819294098238926284943760248852279003620219386696482215628093605373178040863727268426696421929946819214908701707533361094791381804063287387593848269535583077395761447997270003472880182785281389503217986345216111066608839314053226944905455527867894417579202440021450780192099804461382547805858048442416404775031536054906591430078158372430123137511562284015838644270890718284816757527123846782459534334449622010096071051370608461801187543120725491334994247617115633321408934609156561550600317384218701570226103101916603887064661438897736318780940711527528174689576401581047016965247557740891644568677717158500583269943401677202156767724068128366565264122982439465133197359199709403275938502669557470231813203243716420586141033606524536939160050644953060161267822648942437397166717661231048975031885732165554988342121802846912529086101485527815277625623750456375769497734336846015607727035509629049392487088406281067943622418704747008368842671022558302403599841645951122485272633632645114017395248086194635840783753556885622317115520947223065437092606797351000565549381224575483728545711797393615756167641692895805257297522338558611388322171107362265816218842443178857488798109026653793426664216990914056536432249301334867988154886628665052346997235574738424830590423677143278792316422403877764330192600192284778313837632536121025336935812624086866699738275977365682227907215832478888642369346396164363308730139814211430306008730666164803678984091335926293402304324974926887831643602681011309570716141912830686577323532639653677390317661361315965553584999398600565155921936759977717933019744688148371103206503693192894521402650915465184309936553493337183425298433679915939417466223900389527673813330617747629574943868716978453767219493506590875711917720875477107189937960894774512654757501871194870738736785890200617373321075693302216320628432065671192096950585761173961632326217708945426214609858410237813215817727602222738133495410481003073275107799948991977963883530734443457532975914263768405442264784216063122769646967156473999043715903323906560726644116438605404838847161912109008701019130726071044114143241976796828547885524779476481802959736049439700479596040292746299203572099761950140348315380947714601056333446998820822120587281510729182971211917876424880354672316916541852256729234429187128163232596965413548589577133208339911288775917226115273379010341362085614577992398778325083550730199818459025958355989260553299673770491722454935329683300002230181517226575787524058832249085821280089747909326100762578770428656006996176212176845478996440705066241710213327486796237430229155358200780141165348065647488230615003392068983794766255036549822805329662862117930628430170492402301985719978948836897183043805182174419147660429752437251683435411217038631379411422095295885798060152938752753799030938871683572095760715221900279379292786303637268765822681241993384808166021603722154710143007377537792699069587121289288019052031601285861825494413353820784883465311632650407642428390870121015194231961652268422003711230464300673442064747718021353070124098860353399152667923871101706221865883573781210935179775604425634694999787251125440854522274810914874307259869602040275941178942581281882159952359658979181144077653354321757595255536158128001163846720319346507296807990793963714961774312119402021297573125165253768017359101557338153772001952444543620071848475663415407442328621060997613243487548847434539665981338717466093020535070271952983943271425371155766600025784423031073429551533945060486222764966687624079324353192992639253731076892135352572321080889819339168668278948281170472624501948409700975760920983724090074717973340788141825195842598096241747610138252643955135259311885045636264188300338539652435997416931322894719878308427600401368074703904097238473945834896186539790594118599310356168436869219485382055780395773881360679549900085123259442529724486666766834641402189915944565309423440650667851948417766779470472041958822043295380326310537494883122180391279678446100139726753892195119117836587662528083690053249004597410947068772912328214304635337283519953648274325833119144459017809607782883583730111857543659958982724531925310588115026307542571493943024453931870179923608166611305426253995833897942971602070338767815033010280120095997252222280801423571094760351925544434929986767817891045559063015953809761875920358937341978962358931125983902598310267193304189215109689156225069659119828323455503059081730735195503721665870288053992138576037035377105178021280129566841984140362872725623214428754302210909472721073474134975514190737043318276626177275996888826027225247133683353452816692779591328861381766349857728936900965749562287103024362590772412219094300871755692625758065709912016659622436080242870024547362036394841255954881727272473653467783647201918303998717627037515724649922289467932322693619177641614618795613956699567783068290316589699430767333508234990790624100202506134057344300695745474682175690441651540636584680463692621274211075399042188716127617787014258864825775223889184599523376292377915585744549477361295525952226578636462118377598473700347971408206994145580719080213590732269233100831759510659019121294795408603640757358750205890208704579670007055262505811420663907459215273309406823649441590891009220296680523325266198911311842016291631076894084723564366808182168657219688268358402785500782804043453710183651096951782335743030504852653738073531074185917705610397395062640355442275156101107261779370634723804990666922161971194259120445084641746383589938239946517395509000859479990136026674261494290066467115067175422177038774507673563742154782905911012619157555870238957001405117822646989944917908301795475876760168094100135837613578591356924455647764464178667115391951357696104864922490083446715486383054477914330097680486878348184672733758436892724310447406807685278625585165092088263813233623148733336714764520450876627614950389949504809560460989604329123358348859990294526400284994280878624039811814884767301216754161106629995553668193123287425702063738352020086863691311733469731741219153633246745325630871347302792174956227014687325867891734558379964351358800959350877556356248810493852999007675135513527792412429277488565888566513247302514710210575352516511814850902750476845518252096331899068527614435138213662152368890578786699432288816028377482035506016029894009119713850179871683633744139275973644017007014763706655703504338121113576415018451821413619823495159601064752712575935185304332875537783057509567425442684712219618709178560783936144511383335649103256405733898667178123972237519316430617013859539474367843392670986712452211189690840236327411496601243483098929941738030588417166613073040067588380432111555379440605497721705942821514886165672771240903387727745629097110134885184374118695655449745736845218066982911045058004299887953899027804383596282409421860556287788428802127553884803728640019441614257499904272009595204654170598104989967504511936471172772220436102614079750809686975176600237187748348016120310234680567112644766123747627852190241202569943534716226660893675219833111813511146503854895025120655772636145473604426859498074396932331297127377157347099713952291182653485155587137336629120242714302503763269501350911612952993785864681307226486008270881333538193703682598867893321238327053297625857382790097826460545598555131836688844628265133798491667839409761353766251798258249663458771950124384040359140849209733754642474488176184070023569580177410177696925077814893386672557898564589851056891960924398841569280696983352240225634570497312245269354193837004843183357196516626721575524193401933099018319309196582920969656247667683659647019595754739345514337413708761517323677204227385674279170698204549953095918872434939524094441678998846319845504852393662972079777452814399418256789457795712552426826089940863317371538896262889629402112108884427376568624527612130371017300785135715404533041507959447776143597437803742436646973247138410492124314138903579092416036406314038149831481905251720937103964026808994832572297954564042701757722904173234796073618787889913318305843069394825961318713816423467218730845133877219086975104942843769325024981656673816260615941768252509993741672883951744066932549653403101452225316189009235376486378482881344209870048096227171226407489571939002918573307460104360729190945767994614929290427981687729426487729952858434647775386906950148984133924540394144680263625402118614317031251117577642829914644533408920976961699098372652361768745605894704968170136974909523072082682887890730190018253425805343421705928713931737993142410852647390948284596418093614138475831136130576108462366837237695913492615824516221552134879244145041756848064120636520170386330129532777699023118648020067556905682295016354931992305914246396217025329747573114094220180199368035026495636955866425906762685687372110339156793839895765565193177883000241613539562437777840801748819373095020699900890899328088397430367736595524891300156633294077907139615464534088791510300651321934486673248275907946807879819425019582622320395131252014109960531260696555404248670549986786923021746989009547850725672978794769888831093487464426400718183160331655511534276155622405474473378049246214952133258527698847336269182649174338987824789278468918828054669982303689939783413747587025805716349413568433929396068192061773331791738208562436433635359863494496890781064019674074436583667071586924521182997893804077137501290858646578905771426833582768978554717687184427726120509266486102051535642840632368481807287940717127966820060727559555904040233178749447346454760628189541512139162918444297651066947969354016866010055196077687335396511614930937570968554559381513789569039251014953265628147011998326992200066392875374713135236421589265126204072887716578358405219646054105435443642166562244565042999010256586927279142752931172082793937751326106052881235373451068372939893580871243869385934389175713376300720319760816604464683937725806909237297523486702916910426369262090199605204121024077648190316014085863558427609537086558164273995349346546314504040199528537252004957805254656251154109252437991326262713609099402902262062836752132305065183934057450112099341464918433323646569371725914489324159006242020612885732926133596808726500045628284557574596592120530341310111827501306961509835515632004310784601906565493806542525229161991819959602752327702249855738824899882707465936355768582560518068964285376850772012220347920993936179268206590142165615925306737944568949070853263568196831861772268249911472615732035807646298116244013316737892788689229032593349861797021994981925739617673075834417098559222170171825712777534491508205278430904619460835217402005838672849709411023266953921445461066215006410674740207009189911951376466904481267253691537162290791385403937560077835153374167747942100384002308951850994548779039346122220865060160500351776264831611153325587705073541279249909859373473787081194253055121436979749914951860535920403830235716352727630874693219622190064260886183676103346002255477477813641012691906569686495012688376296907233961276287223041141813610060264044030035996988919945827397624114613744804059697062576764723766065541618574690527229238228275186799156983390747671146103022776606020061246876477728819096791613354019881402757992174167678799231603963569492851513633647219540611171767387372555728522940054361785176502307544693869307873499110352182532929726044553210797887711449898870911511237250604238753734841257086064069052058452122754533848008205302450456517669518576913200042816758054924811780519832646032445792829730129105318385636821206215531288668564956512613892261367064093953334570526986959692350353094224543865278677673027540402702246384483553239914751363441044050092330361271496081355490531539021002299595756583705381261965683144286057956696622154721695620870013727768536960840704833325132793112232507148630206951245395003735723346807094656483089209801534878705633491092366057554050864111521441481434630437273271045027768661953107858323334857840297160925215326092558932655600672124359464255065996771770388445396181632879614460817789272171836908880126778207430106422524634807454300476492885553409062185153654355474125476152769772667769772777058315801412185688011705028365275543214803488004442979998062157904564161957212784508928489806426497427090579129069217807298769477975112447305991406050629946894280931034216416629935614828130998870745292716048433630818404126469637925843094185442216359084576146078558562473814931427078266215185541603870206876980461747400808324343665382354555109449498431093494759944672673665352517662706772194183191977196378015702169933675083760057163454643671776723387588643405644871566964321041282595645349841388412890420682047007615596916843038999348366793542549210328113363184722592305554383058206941675629992013373175489122037230349072681068534454035993561823576312837767640631013125335212141994611869350833176587852047112364331226765129964171325217513553261867681942338790365468908001827135283584888444111761234101179918709236507184857856221021104009776994453121795022479578069506532965940383987369907240797679040826794007618729547835963492793904576973661643405359792219285870574957481696694062334272619733518136626063735982575552496509807260123668283605928341855848026958413772558970883789942910549800331113884603401939166122186696058491571485733568286149500019097591125218800396419762163559375743718011480559442298730418196808085647265713547612831629200449880315402105530597076666362749328308916880932359290081787411985738317192616728834918402429721290434965526942726402559641463525914348400675867690350382320572934132981593533044446496829441367323442158380761694831219333119819061096142952201536170298575105594326461468505452684975764807808009221335811378197749271768545075538328768874474591593731162470601091244609829424841287520224462594477638749491997840446829257360968534549843266536862844489365704111817793806441616531223600214918768769467398407517176307516849856359201486892943105940202457969622924566644881967576294349535326382171613395757790766370764569570259738800438415805894336137106551859987600754924187211714889295221737721146081154344982665479872580056674724051122007383459271575727715218589946948117940644466399432370044291140747218180224825837736017346685300744985564715420036123593397312914458591522887408719508708632218837288262822884631843717261903305777147651564143822306791847386039147683108141358275755853643597721650028277803713422869688787349795096031108899196143386664068450697420787700280509367203387232629637856038653216432348815557557018469089074647879122436375556668678067610544955017260791142930831285761254481944449473244819093795369008206384631678225064809531810406570254327604385703505922818919878065865412184299217273720955103242251079718077833042609086794273428955735559252723805511440438001239041687716445180226491681641927401106451622431101700056691121733189423400547959684669804298017362570406733282129962153684881404102194463424646220745575643960452985313071409084608499653767803793201899140865814662175319337665970114330608625009829566917638846056762972931464911493704624469351984039534449135141193667933301936617663652555149174982307987072280860859626112660504289296966535652516688885572112276802772743708917389639772257564890533401038855931125679991516589025016486961427207005916056166159702451989051832969278935550303934681219761582183980483960562523091462638447386296039848924386187298507775928792722068554807210497817653286210187476766897248841139560349480376727036316921007350834073865261684507482496448597428134936480372426116704266870831925040997615319076855770327421785010006441984124207396400139603601583810565928413684574119102736420274163723488214524101347716529603128408658419787951116511529827814620379139855006399960326591248525308493690313130100799977191362230866011099929142871249388541612038020411340188887219693477904497527454288072803509305828754420755134816660927879353566521255620139988249628478726214432362853676502591450468377635282587652139156480972141929675549384375582600253168536356731379262475878049445944183429172756988376226261846365452743497662411138451305481449836311789784489732076719508784158618879692955819733250699951402601511675529750575437810242238957925786562128432731202200716730574069286869363930186765958251326499145950260917069347519408975357464016830811798846452473618956056479426358070562563281189269663026479535951097127659136233180866921535788607812759910537171402204506186075374866306350591483916467656723205714516886170790984695932236724946737583099607042589220481550799132752088583781117685214269334786921895240622657921043620348852926267984013953216458791151579050460579710838983371864038024417511347226472547010794793996953554669619726763255229914654933499663234185951450360980344092212206712567698723427940708857070474293173329188523896721971353924492426178641188637790962814486917869468177591717150669111480020759432012061969637795103227089029566085562225452602610460736131368869009281721068198618553780982018471154163630326265699283424155023600978046417108525537612728905335045506135684143775854429677977014660294387687225115363801191758154028120818255606485410787933598921064427244898618961629413418001295130683638609294100083136673372153008352696235737175330738653338204842190308186449184093723944033405244909554558016406460761581010301767488475017661908692946098769201691202181688291040870709560951470416921147027413390052253340834812870353031023919699978597413908593605433599697075604460134242453682496098772581311024732798562072126572499003468293886872304895562253204463602639854225258416464324271611419817802482595563544907219226583863662663750835944314877635156145710745528016159677048442714194435183275698407552677926411261765250615965235457187956673170913319358761628255920783080185206890151504713340386100310055914817852110384754542933389188444120517943969970194112695119526564919594189975418393234647424290702718875223534393673633663200307232747037407123982562024662651974090199762452056198557625760008708173083288344381831070054514493545885422678578551915372292379555494333410174420169600090696415612732297770221217951868376359082255128816470021992348864043959153018464004714321186360622527011541122283802778538911098490201342741014121559769965438877197485376431158229838533123071751132961904559007938064276695819014842627991221792947987348901868471676503827328552059082984529806259250352128451925927986593506132961946796252373972565584157853744567558998032405492186962888490332560851455344391660226257775512916200772796852629387937530454181080729285891989715381797343496187232927614747850192611450413274873242970583408471112333746274617274626582415324271059322506255302314738759251724787322881491455915605036334575424233779160374952502493022351481961381162563911415610326844958072508273431765944054098269765269344579863479709743124498271933113863873159636361218623497261409556079920628316999420072054811525353393946076850019909886553861433495781650089961649079678142901148387645682174914075623767618453775144031475411206760160726460556859257799322070337333398916369504346690694828436629980037414527627716547623825546170883189810868806847853705536480469350958818025360529740793538676511195079373282083146268960071075175520614433784114549950136432446328193346389050936545714506900864483440180428363390513578157273973334537284263372174065775771079830517555721036795976901889958494130195999573017901240193908681356585539661941371794487632079868800371607303220547423572266896801882123424391885984168972277652194032493227314793669234004848976059037958094696041754279613782553781223947646147832926976545162290281701100437846038756544151739433960048915318817576650500951697402415644771293656614253949368884230517400129920556854289853897942669956777027089146513736892206104415481662156804219838476730871787590279209175900695273456682026513373111518000181434120962601658629821076663523361774007837783423709152644063054071807843358061072961105550020415131696373046849213356837265400307509829089364612047891114753037049893952833457824082817386441322710002968311940203323456420826473276233830294639378998375836554559919340866235090967961134004867027123176526663710778725111860354037554487418693519733656621772359229396776463251562023487570113795712096237723431370212031004965152111976013176419408203437348512852602913334915125083119802850177855710725373149139215709105130965059885999931560863655477403551898166733535880048214665099741433761182777723351910741217572841592580872591315074606025634903777263373914461377038021318347447301113032670296917335047701632106616227830027269283365584011791419447808748253360714403296252285775009808599609040936312635621328162071453406104224112083010008587264252112262480142647519426184325853386753874054743491072710049754281159466017136122590440158991600229827801796035194080046513534752698777609527839984368086908989197839693532179980139135442552717910225397010810632143048511378291498511381969143043497500189980681644412123273328307192824362406733196554692677851193152775113446468905504248113361434984604849051258345683266441528489713972376040328212660253516693914082049947320486021627759791771234751097502403078935759937715095021751693555827072533911892334070223832077585802137174778378778391015234132098489423459613692340497998279304144463162707214796117456975719681239291913740982925805561955207434243295982898980529233366415419256367380689494201471241340525072204061794355252555225008748790086568314542835167750542294803274783044056438581591952666758282929705226127628711040134801787224801789684052407924360582742467443076721645270313451354167649668901274786801010295133862698649748212118629040337691568576240699296372493097201628707200189835423690364149270236961938547372480329855045112089192879829874467864129159417531675602533435310626745254507114181483239880607297140234725520713490798398982355268723950909365667878992383712578976248755990443228895388377317348941122757071410959790047919301046740750411435381782464630795989555638991884773781341347070246747362112048986226991888517456251732519341352038115863350123913054441910073628447567514161050410973505852762044489190978901984315485280533985777844313933883994310444465669244550885946314081751220331390681596592510546858013133838152176418210433429788826119630443111388796258746090226130900849975430395771243230616906262919403921439740270894777663702488155499322458825979020631257436910946393252806241642476868495455324938017639371615636847859823715902385421265840615367228607131702674740131145261063765383390315921943469817605358380310612887852051546933639241088467632009567089718367490578163085158138161966882222047570437590614338040725853862083565176998426774523195824182683698270160237414938363496629351576854061397342746470899685618170160551104880971554859118617189668025973541705423985135560018720335079060946421271143993196046527424050882225359773481519135438571253258540493946010865793798058620143366078825219717809025817370870916460452727977153509910340736425020386386718220522879694458387652947951048660717390229327455426785669776865939923416834122274663015062155320502655341460995249356050854921756549134830958906536175693817637473644183378974229700703545206663170929607591989627732423090252397443861014263098687733913882518684316501027964911497737582888913450341148865948670215492101084328080783428089417298008983297536940644969903125399863919581601468995220880662285408414864274786281975546629278814621607171381880180840572084715868906836919393381864278454537956719272397972364651667592011057995663962598535512763558768140213409829016296873429850792471846056874828331381259161962476156902875901072733103299140623864608333378638257926302391590003557609032477281338887339178096966601469615031754226751125993315529674213336300222964906480934582008181061802100227664580400278213336758573019011371754672763059044353131319036092489097246427928455549913490005180295707082919052556781889913899625138662319380053611346224294610248954072404857123256628888931722116432947816190554868054943441034090680716088028227959686950133643814268252170472870863010137301155236861416908375675747637239763185757038109443390564564468524183028148107998376918512127201935044041804604721626939445788377090105974693219720558114078775989772072009689382249303236830515862657281114637996983137517937623215111252349734305240622105244234353732905655163406669506165892878218707756794176080712973781335187117931650033155523822487730653444179453415395202424449703410120874072188109388268167512042299404948179449472732894770111574139441228455521828424922240658752689172272780607116754046973008037039618787796694882555614674384392570115829546661358678671897661297311267200072971553613027503556167817765442287442114729881614802705243806817653573275578602505847084013208837932816008769081300492491473682517035382219619039014999523495387105997351143478292339499187936608692301375596368532373806703591144243268561512109404259582639301678017128669239283231057658851714020211196957064799814031505633045141564414623163763809904402816256917576489142569714163598439317433270237812336938043012892626375382667795034169334323607500248175741808750388475094939454896209740485442635637164995949920980884294790363666297526003243856352945844728944547166209297495496616877414120882130477022816116456044007236351581149729739218966737382647204722642221242016560150284971306332795814302516013694825567014780935790889657134926158161346901806965089556310121218491805847922720691871696316330044858020102860657858591269974637661741463934159569539554203314628026518951167938074573315759846086173702687867602943677780500244673391332431669880354073232388281847501051641331189537036488422690270478052742490603492082954755054003457160184072574536938145531175354210726557835615499874447480427323457880061873149341566046352979779455075359304795687209316724536547208381685855606043801977030764246083489876101345709394877002946175792061952549255757109038525171488525265671045349813419803390641529876343695420256080277614421914318921393908834543131769685101840103844472348948869520981943531906506555354617335814045544837884752526253949665869992058417652780125341033896469818642430034146791380619028059607854888010789705516946215228773090104467462497979992627120951684779568482583341402266477210843362437593741610536734041954738964197895425335036301861400951534766961476255651873823292468547356935802896011536791787303553159378363082248615177770541577576561759358512016692943111138863582159667618830326104164651714846979385422621687161400122378213779774131268977266712992025922017408770076956283473932201088159356286281928563571893384958850603853158179760679479840878360975960149733420572704603521790605647603285569276273495182203236144112584182426247712012035776388895974318232827871314608053533574494297621796789034568169889553518504478325616380709476951699086247100019748809205009521943632378719764870339223811540363475488626845956159755193765410115014067001226927474393888589943859730245414801061235908036274585288493563251585384383242493252666087588908318700709100237377106576985056433928854337658342596750653715005333514489908293887737352051459333049626531415141386124437935885070944688045486975358170212908490787347806814366323322819415827345671356443171537967818058195852464840084032909981943781718177302317003989733050495387356116261023999433259780126893432605584710278764901070923443884634011735556865903585244919370181041626208504299258697435817098133894045934471937493877624232409852832762266604942385129709453245586252103600829286649724174919141988966129558076770979594795306013119159011773943104209049079424448868513086844493705909026006120649425744710353547657859242708130410618546219881830090634588187038755856274911587375421064667951346487586771543838018521348281915812462599335160198935595167968932852205824799421034512715877163345222995418839680448835529753361286837225935390079201666941339091168758803988828869216002373257361588207163516271332810518187602104852180675526648673908900907195138058626735124312215691637902277328705410842037841525683288718046987952513073266340278519059417338920358540395677035611329354482585628287610610698229721420961993509331312171187891078766872044548876089410174798647137882462153955933333275562009439580434537919782280590395959927436913793778664940964048777841748336432684026282932406260081908081804390914556351936856063045089142289645219987798849347477729132797266027658401667890136490508741142126861969862044126965282981087045479861559545338021201155646979976785738920186243599326777689454060508218838227909833627167124490026761178498264377033002081844590009717235204331994708242098771514449751017055643029542821819670009202515615844174205933658148134902693111517093872260026458630561325605792560927332265579346280805683443921373688405650434307396574061017779370141424615493070741360805442100295600095663588977899267630517718781943706761498217564186590116160865408635391513039201316805769034172596453692350806417446562351523929050409479953184074862151210561833854566176652606393713658802521666223576132201941701372664966073252010771947931265282763302413805164907174565964853748354669194523580315301969160480994606814904037819829732360930087135760798621425422096419004367905479049930078372421581954535418371129368658430553842717628035279128821129308351575656599944741788438381565148434229858704245592434693295232821803508333726283791830216591836181554217157448465778420134329982594566884558266171979012180849480332448787258183774805522268151011371745368417870280274452442905474518234674919564188551244421337783521423865979925988203287085109338386829906571994614906290257427686038850511032638544540419184958866538545040571323629681069146814847869659166861842756798460041868762298055562963045953227923051616721591968675849523635298935788507746081537321454642984792310511676357749494622952569497660359473962430995343310404994209677883827002714478494069037073249106444151696053256560586778757417472110827435774315194060757983563629143326397812218946287447798119807225646714664054850131009656786314880090303749338875364183165134982546694673316118123364854397649325026179549357204305402182974871251107404011611405899911093062492312813116340549262571356721818628932786138833718028535056503591952741400869510926167541476792668032109237467087213606278332922386413619594121339278036118276324106004740971111048140003623342714514483334641675466354699731494756643423659493496845884551524150756376605086632827424794136062876041290644913828519456402643153225858624043141838669590633245063000392213192647625962691510904457695301444054618037857503036686212462278639752746667870121003392984873375014475600322100622358029343774955032037012738468163061026570300872275462966796880890587127676361066225722352229739206443093524327228100859973095132528630601105497915644791845004618046762408928925680912930592960642357021061524646205023248966593987324933967376952023991760898474571843531936646529125848064480196520162838795189499336759241485626136995945307287254532463291529110128763770605570609531377527751867923292134955245133089867969165129073841302167573238637575820080363575728002754490327953079900799442541108725693188014667935595834676432868876966610097395749967836593397846346959948950610490383647409504695226063858046758073069912290474089879166872117147527644711604401952718169508289733537148530928937046384420893299771125856840846608339934045689026787516008775461267988015465856522061210953490796707365539702576199431376639960606061106406959330828171876426043573425361756943784848495250108266488395159700490598380812105221111091943323951136051446459834210799058082093716464523127704023160072138543723461267260997870385657091998507595634613248460188409850194287687902268734556500519121546544063829253851276317663922050938345204300773017029940362615434001322763910912988327863920412300445551684054889809080779174636092439334912641164240093880746356607262336695842764583698268734815881961058571835767462009650526065929263548291499045768307210893245857073701660717398194485028842603963660746031184786225831056580870870305567595861341700745402965687634774176431051751036732869245558582082372038601781739405175130437994868822320044378043103170921034261674998000073016094814586374488778522273076330495383944345382770608760763542098445008306247630253572781032783461766970544287155315340016497076657195985041748199087201490875686037783591994719343352772947285537925787684832301101859365800717291186967617655053775030293033830706448912811412025506150896411007623824574488655182581058140345320124754723269087547507078577659732542844459353044992070014538748948226556442223696365544194225441338212225477497535494624827680533336983284156138692363443358553868471111430498248398991803165458638289353799130535222833430137953372954016257623228081138499491876144141322933767106563492528814528239506209022357876684650116660097382753660405446941653422239052108314585847035529352219928272760574821266065291385530345549744551470344939486863429459658431024190785923680224560763936784166270518555178702904073557304620639692453307795782245949710420188043000183881429008173039450507342787013124466860092778581811040911511729374873627887874907465285565434748886831064110051023020875107768918781525622735251550379532444857787277617001964853703555167655209119339343762866284619844026295252183678522367475108809781507098978413086245881522660963551401874495836926917799047120726494905737264286005211403581231076006699518536124862746756375896225299116496066876508261734178484789337295056739007878617925351440621045366250640463728815698232317500596261080921955211150859302955654967538862612972339914628358476048627627027309739202001432248707582337354915246085608210328882974183906478869923273691360048837436615223517058437705545210815513361262142911815615301758882573594892507108879262128641392443309383797333867806131795237315266773820858024701433527009243803266951742119507670884326346442749127558907746863582162166042741315170212458586056233631493164646913946562497471741958354218607748711057338458433689939645913740603382159352243594751626239188685307822821763983237306180204246560477527943104796189724299533029792497481684052893791044947004590864991872727345413508101983881864673609392571930511968645601855782450218231065889437986522432050677379966196955472440585922417953006820451795370043472451762893566770508490213107736625751697335527462302943031203596260953423574397249659211010657817826108745318874803187430823573699195156340957162700992444929749105489851519658664740148225106335367949737142510229341882585117371994499115097583746130105505064197721531929354875371191630262030328588658528480193509225875775597425276584011721342323648084027143356367542046375182552524944329657043861387865901965738802868401894087672816714137033661732650120578653915780703088714261519075001492576112927675193096728453971160213606303090542243966320674323582797889332324405779199278484633339777737655901870574806828678347965624146102899508487399692970750432753029972872297327934442988646412725348160603779707298299173029296308695801996312413304939350493325412355071054461182591141116454534710329881047844067780138077131465400099386306481266614330858206811395838319169545558259426895769841428893743467084107946318932539106963955780706021245974898293564613560788983472419979478564362042094613412387613198865352358312996862268948608408456655606876954501274486631405054735351746873009806322780468912246821460806727627708402402266155485024008952891657117617439020337584877842911289623247059191874691042005848326140677333751027195653994697162517248312230633919328707983800748485726516123434933273356664473358556430235280883924348278760886164943289399166399210488307847777048045728491456303353265070029588906265915498509407972767567129795010098229476228961891591441520032283878773485130979081019129267227103778898053964156362364169154985768408398468861684375407065121039062506128107663799047908879674778069738473170475253442156390387201238806323688037017949308954900776331523063548374256816653361606641980030188287123767481898330246836371488309259283375902278942588060087286038859168849730693948020511221766359138251524278670094406942355120201568377778851824670025651708509249623747726813694284350062938814429987905301056217375459182679973217735029368928065210025396268807498092643458011655715886700443503976505323478287327368840863540002740676783821963522226539290939807367391364082898722017776747168118195856133721583119054682936083236976113450281757830202934845982925000895682630271263295866292147653142233351793093387951357095346377183684092444422096319331295620305575517340067973740614162107923633423805646850092037167152642556371853889571416419772387422610596667396997173168169415435095283193556417705668622215217991151355639707143312893657553844648326201206424338016955862698561022460646069330793847858814367407000599769703649019273328826135329363112403650698652160638987250267238087403396744397830258296894256896741864336134979475245526291426522842419243083388103580053787023999542172113686550275341362211693140694669513186928102574795985605145005021715913317751609957865551981886193211282110709442287240442481153406055895958355815232012184605820563592699303478851132068626627588771446035996656108430725696500563064489187599466596772847171539573612108180841547273142661748933134174632662354222072600146012701206934639520564445543291662986660783089068118790090815295063626782075614388815781351134695366303878412092346942868730839320432333872775496805210302821544324723388845215343727250128589747691460808314404125868181540049187772287869801853454537006526655649170915429522756709222217474112062720656622989806032891672068743654948246108697367225547404812889242471854323605753411672850757552057131156697954584887398742228135887985840783135060548290551482785294891121905383195624228719484759407859398047901094194070671764439032730712135887385049993638838205501683402777496070276844880281912220636888636811043569529300652195528261526991271637277388418993287130563464688227398288763198645709836308917786487086676185485680047672552675414742851028145807403152992197814557756843681110185317498167016426647884090262682824448258027532094549915104518517716546311804904567985713257528117913656278158111288816562285876030875974963849435275676612168959261485030785362045274507752950631012480341804584059432926079854435620093708091821523920371790678121992280496069738238743312626730306795943960954957189577217915597300588693646845576676092450906088202212235719254536715191834872587423919410890444115959932760044506556206461164655665487594247369252336955993030355095817626176231849561906494839673002037763874369343999829430209147073618947932692762445186560239559053705128978163455423320114975994896278424327483788032701418676952621180975006405149755889650293004867605208010491537885413909424531691719987628941277221129464568294860281493181560249677887949813777216229359437811004448060797672429276249510784153446429150842764520002042769470698041775832209097020291657347251582904630910359037842977572651720877244740952267166306005469716387943171196873484688738186656751279298575016363411314627530499019135646823804329970695770150789337728658035712790913767420805655493624646

lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-null-max.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-null-max.golden differ

lib/std/compress/deflate/testdata/huffman-null-max.input created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-null-max.input differ

lib/std/compress/deflate/testdata/huffman-null-max.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect differ

lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-pi.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-pi.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-pi.golden differ

lib/std/compress/deflate/testdata/huffman-pi.input created+1

...	@@ -0,0 +1 @@
1	3.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-pi.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-pi.wb.expect differ

lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-rand-1k.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-1k.golden differ

lib/std/compress/deflate/testdata/huffman-rand-1k.input created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-1k.input differ

lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect differ

lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-rand-limit.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-limit.golden differ

lib/std/compress/deflate/testdata/huffman-rand-limit.input created+4

...	@@ -0,0 +1,4 @@
1	aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2	��vH
3	��%�� ɷ��}��>��ls��m�IGH��1Y�4�[�� 0[\|]o#�
4	�-#��ul��pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O��{��k�*�w(��b� ��kQC9/��lu>�5�C.��u�

lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect differ

lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-rand-max.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-max.golden differ

lib/std/compress/deflate/testdata/huffman-rand-max.input created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-rand-max.input differ

lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-shifts.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-shifts.golden differ

lib/std/compress/deflate/testdata/huffman-shifts.input created+2

...	@@ -0,0 +1,2 @@
1	101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2	232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-shifts.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect differ

lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-text-shift.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text-shift.golden differ

lib/std/compress/deflate/testdata/huffman-text-shift.input created+14

...	@@ -0,0 +1,14 @@
1	//Copyright2009ThGoAuthor.Allrightrrvd.
2	//UofthiourccodigovrndbyBSD-tyl
3	//licnthtcnbfoundinthLICENSEfil.
4
5	pckgmin
6
7	import"o"
8
9	funcmin(){
10	vrb=mk([]byt,65535)
11	f,_:=o.Crt("huffmn-null-mx.in")
12	f.Writ(b)
13	}
14	ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect differ

lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-text.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-text.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text.golden differ

lib/std/compress/deflate/testdata/huffman-text.input created+14

...	@@ -0,0 +1,14 @@
1	// zig v0.10.0
2	// create a file filled with 0x00
3	const std = @import("std");
4
5	pub fn main() !void {
6	var b = [1]u8{0} ** 65535;
7	const f = try std.fs.cwd().createFile(
8	"huffman-null-max.in",
9	.{ .read = true },
10	);
11	defer f.close();
12
13	_ = try f.writeAll(b[0..]);
14	}

lib/std/compress/deflate/testdata/huffman-text.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text.wb.expect differ

lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-zero.dyn.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect differ

lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/huffman-zero.golden created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-zero.golden differ

lib/std/compress/deflate/testdata/huffman-zero.input created+1

...	@@ -0,0 +1 @@
1	00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
	\ No newline at end of file

lib/std/compress/deflate/testdata/huffman-zero.wb.expect created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-zero.wb.expect differ

lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput differ

lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput differ

lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput created

Binary files /dev/null and b/lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput differ

lib/std/compress/deflate/testdata/rfc1951.txt created+955

...	@@ -0,0 +1,955 @@
1
2
3
4
5
6
7	Network Working Group P. Deutsch
8	Request for Comments: 1951 Aladdin Enterprises
9	Category: Informational May 1996
10
11
12	DEFLATE Compressed Data Format Specification version 1.3
13
14	Status of This Memo
15
16	This memo provides information for the Internet community. This memo
17	does not specify an Internet standard of any kind. Distribution of
18	this memo is unlimited.
19
20	IESG Note:
21
22	The IESG takes no position on the validity of any Intellectual
23	Property Rights statements contained in this document.
24
25	Notices
26
27	Copyright (c) 1996 L. Peter Deutsch
28
29	Permission is granted to copy and distribute this document for any
30	purpose and without charge, including translations into other
31	languages and incorporation into compilations, provided that the
32	copyright notice and this notice are preserved, and that any
33	substantive changes or deletions from the original are clearly
34	marked.
35
36	A pointer to the latest version of this and related documentation in
37	HTML format can be found at the URL
38	<ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40	Abstract
41
42	This specification defines a lossless compressed data format that
43	compresses data using a combination of the LZ77 algorithm and Huffman
44	coding, with efficiency comparable to the best currently available
45	general-purpose compression methods. The data can be produced or
46	consumed, even for an arbitrarily long sequentially presented input
47	data stream, using only an a priori bounded amount of intermediate
48	storage. The format can be implemented readily in a manner not
49	covered by patents.
50
51
52
53
54
55
56
57
58	Deutsch Informational [Page 1]
59
60	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
61
62
63	Table of Contents
64
65	1. Introduction ................................................... 2
66	1.1. Purpose ................................................... 2
67	1.2. Intended audience ......................................... 3
68	1.3. Scope ..................................................... 3
69	1.4. Compliance ................................................ 3
70	1.5. Definitions of terms and conventions used ................ 3
71	1.6. Changes from previous versions ............................ 4
72	2. Compressed representation overview ............................. 4
73	3. Detailed specification ......................................... 5
74	3.1. Overall conventions ....................................... 5
75	3.1.1. Packing into bytes .................................. 5
76	3.2. Compressed block format ................................... 6
77	3.2.1. Synopsis of prefix and Huffman coding ............... 6
78	3.2.2. Use of Huffman coding in the "deflate" format ....... 7
79	3.2.3. Details of block format ............................. 9
80	3.2.4. Non-compressed blocks (BTYPE=00) ................... 11
81	3.2.5. Compressed blocks (length and distance codes) ...... 11
82	3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12
83	3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13
84	3.3. Compliance ............................................... 14
85	4. Compression algorithm details ................................. 14
86	5. References .................................................... 16
87	6. Security Considerations ....................................... 16
88	7. Source code ................................................... 16
89	8. Acknowledgements .............................................. 16
90	9. Author's Address .............................................. 17
91
92	1. Introduction
93
94	1.1. Purpose
95
96	The purpose of this specification is to define a lossless
97	compressed data format that:
98	* Is independent of CPU type, operating system, file system,
99	and character set, and hence can be used for interchange;
100	* Can be produced or consumed, even for an arbitrarily long
101	sequentially presented input data stream, using only an a
102	priori bounded amount of intermediate storage, and hence
103	can be used in data communications or similar structures
104	such as Unix filters;
105	* Compresses data with efficiency comparable to the best
106	currently available general-purpose compression methods,
107	and in particular considerably better than the "compress"
108	program;
109	* Can be implemented readily in a manner not covered by
110	patents, and hence can be practiced freely;
111
112
113
114	Deutsch Informational [Page 2]
115
116	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
117
118
119	* Is compatible with the file format produced by the current
120	widely used gzip utility, in that conforming decompressors
121	will be able to read data produced by the existing gzip
122	compressor.
123
124	The data format defined by this specification does not attempt to:
125
126	* Allow random access to compressed data;
127	* Compress specialized data (e.g., raster graphics) as well
128	as the best currently available specialized algorithms.
129
130	A simple counting argument shows that no lossless compression
131	algorithm can compress every possible input data set. For the
132	format defined here, the worst case expansion is 5 bytes per 32K-
133	byte block, i.e., a size increase of 0.015% for large data sets.
134	English text usually compresses by a factor of 2.5 to 3;
135	executable files usually compress somewhat less; graphical data
136	such as raster images may compress much more.
137
138	1.2. Intended audience
139
140	This specification is intended for use by implementors of software
141	to compress data into "deflate" format and/or decompress data from
142	"deflate" format.
143
144	The text of the specification assumes a basic background in
145	programming at the level of bits and other primitive data
146	representations. Familiarity with the technique of Huffman coding
147	is helpful but not required.
148
149	1.3. Scope
150
151	The specification specifies a method for representing a sequence
152	of bytes as a (usually shorter) sequence of bits, and a method for
153	packing the latter bit sequence into bytes.
154
155	1.4. Compliance
156
157	Unless otherwise indicated below, a compliant decompressor must be
158	able to accept and decompress any data set that conforms to all
159	the specifications presented here; a compliant compressor must
160	produce data sets that conform to all the specifications presented
161	here.
162
163	1.5. Definitions of terms and conventions used
164
165	Byte: 8 bits stored or transmitted as a unit (same as an octet).
166	For this specification, a byte is exactly 8 bits, even on machines
167
168
169
170	Deutsch Informational [Page 3]
171
172	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
173
174
175	which store a character on a number of bits different from eight.
176	See below, for the numbering of bits within a byte.
177
178	String: a sequence of arbitrary bytes.
179
180	1.6. Changes from previous versions
181
182	There have been no technical changes to the deflate format since
183	version 1.1 of this specification. In version 1.2, some
184	terminology was changed. Version 1.3 is a conversion of the
185	specification to RFC style.
186
187	2. Compressed representation overview
188
189	A compressed data set consists of a series of blocks, corresponding
190	to successive blocks of input data. The block sizes are arbitrary,
191	except that non-compressible blocks are limited to 65,535 bytes.
192
193	Each block is compressed using a combination of the LZ77 algorithm
194	and Huffman coding. The Huffman trees for each block are independent
195	of those for previous or subsequent blocks; the LZ77 algorithm may
196	use a reference to a duplicated string occurring in a previous block,
197	up to 32K input bytes before.
198
199	Each block consists of two parts: a pair of Huffman code trees that
200	describe the representation of the compressed data part, and a
201	compressed data part. (The Huffman trees themselves are compressed
202	using Huffman encoding.) The compressed data consists of a series of
203	elements of two types: literal bytes (of strings that have not been
204	detected as duplicated within the previous 32K input bytes), and
205	pointers to duplicated strings, where a pointer is represented as a
206	pair <length, backward distance>. The representation used in the
207	"deflate" format limits distances to 32K bytes and lengths to 258
208	bytes, but does not limit the size of a block, except for
209	uncompressible blocks, which are limited as noted above.
210
211	Each type of value (literals, distances, and lengths) in the
212	compressed data is represented using a Huffman code, using one code
213	tree for literals and lengths and a separate code tree for distances.
214	The code trees for each block appear in a compact form just before
215	the compressed data for that block.
216
217
218
219
220
221
222
223
224
225
226	Deutsch Informational [Page 4]
227
228	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
229
230
231	3. Detailed specification
232
233	3.1. Overall conventions In the diagrams below, a box like this:
234
235	+---+
236	\| \| <-- the vertical bars might be missing
237	+---+
238
239	represents one byte; a box like this:
240
241	+==============+
242	\| \|
243	+==============+
244
245	represents a variable number of bytes.
246
247	Bytes stored within a computer do not have a "bit order", since
248	they are always treated as a unit. However, a byte considered as
249	an integer between 0 and 255 does have a most- and least-
250	significant bit, and since we write numbers with the most-
251	significant digit on the left, we also write bytes with the most-
252	significant bit on the left. In the diagrams below, we number the
253	bits of a byte so that bit 0 is the least-significant bit, i.e.,
254	the bits are numbered:
255
256	+--------+
257	\|76543210\|
258	+--------+
259
260	Within a computer, a number may occupy multiple bytes. All
261	multi-byte numbers in the format described here are stored with
262	the least-significant byte first (at the lower memory address).
263	For example, the decimal number 520 is stored as:
264
265	0 1
266	+--------+--------+
267	\|00001000\|00000010\|
268	+--------+--------+
269	^ ^
270	\| \|
271	\| + more significant byte = 2 x 256
272	+ less significant byte = 8
273
274	3.1.1. Packing into bytes
275
276	This document does not address the issue of the order in which
277	bits of a byte are transmitted on a bit-sequential medium,
278	since the final data format described here is byte- rather than
279
280
281
282	Deutsch Informational [Page 5]
283
284	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
285
286
287	bit-oriented. However, we describe the compressed block format
288	in below, as a sequence of data elements of various bit
289	lengths, not a sequence of bytes. We must therefore specify
290	how to pack these data elements into bytes to form the final
291	compressed byte sequence:
292
293	* Data elements are packed into bytes in order of
294	increasing bit number within the byte, i.e., starting
295	with the least-significant bit of the byte.
296	* Data elements other than Huffman codes are packed
297	starting with the least-significant bit of the data
298	element.
299	* Huffman codes are packed starting with the most-
300	significant bit of the code.
301
302	In other words, if one were to print out the compressed data as
303	a sequence of bytes, starting with the first byte at the
304	right margin and proceeding to the left, with the most-
305	significant bit of each byte on the left as usual, one would be
306	able to parse the result from right to left, with fixed-width
307	elements in the correct MSB-to-LSB order and Huffman codes in
308	bit-reversed order (i.e., with the first bit of the code in the
309	relative LSB position).
310
311	3.2. Compressed block format
312
313	3.2.1. Synopsis of prefix and Huffman coding
314
315	Prefix coding represents symbols from an a priori known
316	alphabet by bit sequences (codes), one code for each symbol, in
317	a manner such that different symbols may be represented by bit
318	sequences of different lengths, but a parser can always parse
319	an encoded string unambiguously symbol-by-symbol.
320
321	We define a prefix code in terms of a binary tree in which the
322	two edges descending from each non-leaf node are labeled 0 and
323	1 and in which the leaf nodes correspond one-for-one with (are
324	labeled with) the symbols of the alphabet; then the code for a
325	symbol is the sequence of 0's and 1's on the edges leading from
326	the root to the leaf labeled with that symbol. For example:
327
328
329
330
331
332
333
334
335
336
337
338	Deutsch Informational [Page 6]
339
340	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
341
342
343	/\ Symbol Code
344	0 1 ------ ----
345	/ \ A 00
346	/\ B B 1
347	0 1 C 011
348	/ \ D 010
349	A /\
350	0 1
351	/ \
352	D C
353
354	A parser can decode the next symbol from an encoded input
355	stream by walking down the tree from the root, at each step
356	choosing the edge corresponding to the next input bit.
357
358	Given an alphabet with known symbol frequencies, the Huffman
359	algorithm allows the construction of an optimal prefix code
360	(one which represents strings with those symbol frequencies
361	using the fewest bits of any possible prefix codes for that
362	alphabet). Such a code is called a Huffman code. (See
363	reference [1] in Chapter 5, references for additional
364	information on Huffman codes.)
365
366	Note that in the "deflate" format, the Huffman codes for the
367	various alphabets must not exceed certain maximum code lengths.
368	This constraint complicates the algorithm for computing code
369	lengths from symbol frequencies. Again, see Chapter 5,
370	references for details.
371
372	3.2.2. Use of Huffman coding in the "deflate" format
373
374	The Huffman codes used for each alphabet in the "deflate"
375	format have two additional rules:
376
377	* All codes of a given bit length have lexicographically
378	consecutive values, in the same order as the symbols
379	they represent;
380
381	* Shorter codes lexicographically precede longer codes.
382
383
384
385
386
387
388
389
390
391
392
393
394	Deutsch Informational [Page 7]
395
396	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
397
398
399	We could recode the example above to follow this rule as
400	follows, assuming that the order of the alphabet is ABCD:
401
402	Symbol Code
403	------ ----
404	A 10
405	B 0
406	C 110
407	D 111
408
409	I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are
410	lexicographically consecutive.
411
412	Given this rule, we can define the Huffman code for an alphabet
413	just by giving the bit lengths of the codes for each symbol of
414	the alphabet in order; this is sufficient to determine the
415	actual codes. In our example, the code is completely defined
416	by the sequence of bit lengths (2, 1, 3, 3). The following
417	algorithm generates the codes as integers, intended to be read
418	from most- to least-significant bit. The code lengths are
419	initially in tree[I].Len; the codes are produced in
420	tree[I].Code.
421
422	1) Count the number of codes for each code length. Let
423	bl_count[N] be the number of codes of length N, N >= 1.
424
425	2) Find the numerical value of the smallest code for each
426	code length:
427
428	code = 0;
429	bl_count[0] = 0;
430	for (bits = 1; bits <= MAX_BITS; bits++) {
431	code = (code + bl_count[bits-1]) << 1;
432	next_code[bits] = code;
433	}
434
435	3) Assign numerical values to all codes, using consecutive
436	values for all codes of the same length with the base
437	values determined at step 2. Codes that are never used
438	(which have a bit length of zero) must not be assigned a
439	value.
440
441	for (n = 0; n <= max_code; n++) {
442	len = tree[n].Len;
443	if (len != 0) {
444	tree[n].Code = next_code[len];
445	next_code[len]++;
446	}
447
448
449
450	Deutsch Informational [Page 8]
451
452	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
453
454
455	}
456
457	Example:
458
459	Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3,
460	3, 2, 4, 4). After step 1, we have:
461
462	N bl_count[N]
463	- -----------
464	2 1
465	3 5
466	4 2
467
468	Step 2 computes the following next_code values:
469
470	N next_code[N]
471	- ------------
472	1 0
473	2 0
474	3 2
475	4 14
476
477	Step 3 produces the following code values:
478
479	Symbol Length Code
480	------ ------ ----
481	A 3 010
482	B 3 011
483	C 3 100
484	D 3 101
485	E 3 110
486	F 2 00
487	G 4 1110
488	H 4 1111
489
490	3.2.3. Details of block format
491
492	Each block of compressed data begins with 3 header bits
493	containing the following data:
494
495	first bit BFINAL
496	next 2 bits BTYPE
497
498	Note that the header bits do not necessarily begin on a byte
499	boundary, since a block does not necessarily occupy an integral
500	number of bytes.
501
502
503
504
505
506	Deutsch Informational [Page 9]
507
508	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
509
510
511	BFINAL is set if and only if this is the last block of the data
512	set.
513
514	BTYPE specifies how the data are compressed, as follows:
515
516	00 - no compression
517	01 - compressed with fixed Huffman codes
518	10 - compressed with dynamic Huffman codes
519	11 - reserved (error)
520
521	The only difference between the two compressed cases is how the
522	Huffman codes for the literal/length and distance alphabets are
523	defined.
524
525	In all cases, the decoding algorithm for the actual data is as
526	follows:
527
528	do
529	read block header from input stream.
530	if stored with no compression
531	skip any remaining bits in current partially
532	processed byte
533	read LEN and NLEN (see next section)
534	copy LEN bytes of data to output
535	otherwise
536	if compressed with dynamic Huffman codes
537	read representation of code trees (see
538	subsection below)
539	loop (until end of block code recognized)
540	decode literal/length value from input stream
541	if value < 256
542	copy value (literal byte) to output stream
543	otherwise
544	if value = end of block (256)
545	break from loop
546	otherwise (value = 257..285)
547	decode distance from input stream
548
549	move backwards distance bytes in the output
550	stream, and copy length bytes from this
551	position to the output stream.
552	end loop
553	while not last block
554
555	Note that a duplicated string reference may refer to a string
556	in a previous block; i.e., the backward distance may cross one
557	or more block boundaries. However a distance cannot refer past
558	the beginning of the output stream. (An application using a
559
560
561
562	Deutsch Informational [Page 10]
563
564	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
565
566
567	preset dictionary might discard part of the output stream; a
568	distance can refer to that part of the output stream anyway)
569	Note also that the referenced string may overlap the current
570	position; for example, if the last 2 bytes decoded have values
571	X and Y, a string reference with <length = 5, distance = 2>
572	adds X,Y,X,Y,X to the output stream.
573
574	We now specify each compression method in turn.
575
576	3.2.4. Non-compressed blocks (BTYPE=00)
577
578	Any bits of input up to the next byte boundary are ignored.
579	The rest of the block consists of the following information:
580
581	0 1 2 3 4...
582	+---+---+---+---+================================+
583	\| LEN \| NLEN \|... LEN bytes of literal data...\|
584	+---+---+---+---+================================+
585
586	LEN is the number of data bytes in the block. NLEN is the
587	one's complement of LEN.
588
589	3.2.5. Compressed blocks (length and distance codes)
590
591	As noted above, encoded data blocks in the "deflate" format
592	consist of sequences of symbols drawn from three conceptually
593	distinct alphabets: either literal bytes, from the alphabet of
594	byte values (0..255), or <length, backward distance> pairs,
595	where the length is drawn from (3..258) and the distance is
596	drawn from (1..32,768). In fact, the literal and length
597	alphabets are merged into a single alphabet (0..285), where
598	values 0..255 represent literal bytes, the value 256 indicates
599	end-of-block, and values 257..285 represent length codes
600	(possibly in conjunction with extra bits following the symbol
601	code) as follows:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618	Deutsch Informational [Page 11]
619
620	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
621
622
623	Extra Extra Extra
624	Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
625	---- ---- ------ ---- ---- ------- ---- ---- -------
626	257 0 3 267 1 15,16 277 4 67-82
627	258 0 4 268 1 17,18 278 4 83-98
628	259 0 5 269 2 19-22 279 4 99-114
629	260 0 6 270 2 23-26 280 4 115-130
630	261 0 7 271 2 27-30 281 5 131-162
631	262 0 8 272 2 31-34 282 5 163-194
632	263 0 9 273 3 35-42 283 5 195-226
633	264 0 10 274 3 43-50 284 5 227-257
634	265 1 11,12 275 3 51-58 285 0 258
635	266 1 13,14 276 3 59-66
636
637	The extra bits should be interpreted as a machine integer
638	stored with the most-significant bit first, e.g., bits 1110
639	represent the value 14.
640
641	Extra Extra Extra
642	Code Bits Dist Code Bits Dist Code Bits Distance
643	---- ---- ---- ---- ---- ------ ---- ---- --------
644	0 0 1 10 4 33-48 20 9 1025-1536
645	1 0 2 11 4 49-64 21 9 1537-2048
646	2 0 3 12 5 65-96 22 10 2049-3072
647	3 0 4 13 5 97-128 23 10 3073-4096
648	4 1 5,6 14 6 129-192 24 11 4097-6144
649	5 1 7,8 15 6 193-256 25 11 6145-8192
650	6 2 9-12 16 7 257-384 26 12 8193-12288
651	7 2 13-16 17 7 385-512 27 12 12289-16384
652	8 3 17-24 18 8 513-768 28 13 16385-24576
653	9 3 25-32 19 8 769-1024 29 13 24577-32768
654
655	3.2.6. Compression with fixed Huffman codes (BTYPE=01)
656
657	The Huffman codes for the two alphabets are fixed, and are not
658	represented explicitly in the data. The Huffman code lengths
659	for the literal/length alphabet are:
660
661	Lit Value Bits Codes
662	--------- ---- -----
663	0 - 143 8 00110000 through
664	10111111
665	144 - 255 9 110010000 through
666	111111111
667	256 - 279 7 0000000 through
668	0010111
669	280 - 287 8 11000000 through
670	11000111
671
672
673
674	Deutsch Informational [Page 12]
675
676	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
677
678
679	The code lengths are sufficient to generate the actual codes,
680	as described above; we show the codes in the table for added
681	clarity. Literal/length values 286-287 will never actually
682	occur in the compressed data, but participate in the code
683	construction.
684
685	Distance codes 0-31 are represented by (fixed-length) 5-bit
686	codes, with possible additional bits as shown in the table
687	shown in Paragraph 3.2.5, above. Note that distance codes 30-
688	31 will never actually occur in the compressed data.
689
690	3.2.7. Compression with dynamic Huffman codes (BTYPE=10)
691
692	The Huffman codes for the two alphabets appear in the block
693	immediately after the header bits and before the actual
694	compressed data, first the literal/length code and then the
695	distance code. Each code is defined by a sequence of code
696	lengths, as discussed in Paragraph 3.2.2, above. For even
697	greater compactness, the code length sequences themselves are
698	compressed using a Huffman code. The alphabet for code lengths
699	is as follows:
700
701	0 - 15: Represent code lengths of 0 - 15
702	16: Copy the previous code length 3 - 6 times.
703	The next 2 bits indicate repeat length
704	(0 = 3, ... , 3 = 6)
705	Example: Codes 8, 16 (+2 bits 11),
706	16 (+2 bits 10) will expand to
707	12 code lengths of 8 (1 + 6 + 5)
708	17: Repeat a code length of 0 for 3 - 10 times.
709	(3 bits of length)
710	18: Repeat a code length of 0 for 11 - 138 times
711	(7 bits of length)
712
713	A code length of 0 indicates that the corresponding symbol in
714	the literal/length or distance alphabet will not occur in the
715	block, and should not participate in the Huffman code
716	construction algorithm given earlier. If only one distance
717	code is used, it is encoded using one bit, not zero bits; in
718	this case there is a single code length of one, with one unused
719	code. One distance code of zero bits means that there are no
720	distance codes used at all (the data is all literals).
721
722	We can now define the format of the block:
723
724	5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286)
725	5 Bits: HDIST, # of Distance codes - 1 (1 - 32)
726	4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19)
727
728
729
730	Deutsch Informational [Page 13]
731
732	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
733
734
735	(HCLEN + 4) x 3 bits: code lengths for the code length
736	alphabet given just above, in the order: 16, 17, 18,
737	0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
738
739	These code lengths are interpreted as 3-bit integers
740	(0-7); as above, a code length of 0 means the
741	corresponding symbol (literal/length or distance code
742	length) is not used.
743
744	HLIT + 257 code lengths for the literal/length alphabet,
745	encoded using the code length Huffman code
746
747	HDIST + 1 code lengths for the distance alphabet,
748	encoded using the code length Huffman code
749
750	The actual compressed data of the block,
751	encoded using the literal/length and distance Huffman
752	codes
753
754	The literal/length symbol 256 (end of data),
755	encoded using the literal/length Huffman code
756
757	The code length repeat codes can cross from HLIT + 257 to the
758	HDIST + 1 code lengths. In other words, all code lengths form
759	a single sequence of HLIT + HDIST + 258 values.
760
761	3.3. Compliance
762
763	A compressor may limit further the ranges of values specified in
764	the previous section and still be compliant; for example, it may
765	limit the range of backward pointers to some value smaller than
766	32K. Similarly, a compressor may limit the size of blocks so that
767	a compressible block fits in memory.
768
769	A compliant decompressor must accept the full range of possible
770	values defined in the previous section, and must accept blocks of
771	arbitrary size.
772
773	4. Compression algorithm details
774
775	While it is the intent of this document to define the "deflate"
776	compressed data format without reference to any particular
777	compression algorithm, the format is related to the compressed
778	formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below);
779	since many variations of LZ77 are patented, it is strongly
780	recommended that the implementor of a compressor follow the general
781	algorithm presented here, which is known not to be patented per se.
782	The material in this section is not part of the definition of the
783
784
785
786	Deutsch Informational [Page 14]
787
788	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
789
790
791	specification per se, and a compressor need not follow it in order to
792	be compliant.
793
794	The compressor terminates a block when it determines that starting a
795	new block with fresh trees would be useful, or when the block size
796	fills up the compressor's block buffer.
797
798	The compressor uses a chained hash table to find duplicated strings,
799	using a hash function that operates on 3-byte sequences. At any
800	given point during compression, let XYZ be the next 3 input bytes to
801	be examined (not necessarily all different, of course). First, the
802	compressor examines the hash chain for XYZ. If the chain is empty,
803	the compressor simply writes out X as a literal byte and advances one
804	byte in the input. If the hash chain is not empty, indicating that
805	the sequence XYZ (or, if we are unlucky, some other 3 bytes with the
806	same hash function value) has occurred recently, the compressor
807	compares all strings on the XYZ hash chain with the actual input data
808	sequence starting at the current point, and selects the longest
809	match.
810
811	The compressor searches the hash chains starting with the most recent
812	strings, to favor small distances and thus take advantage of the
813	Huffman encoding. The hash chains are singly linked. There are no
814	deletions from the hash chains; the algorithm simply discards matches
815	that are too old. To avoid a worst-case situation, very long hash
816	chains are arbitrarily truncated at a certain length, determined by a
817	run-time parameter.
818
819	To improve overall compression, the compressor optionally defers the
820	selection of matches ("lazy matching"): after a match of length N has
821	been found, the compressor searches for a longer match starting at
822	the next input byte. If it finds a longer match, it truncates the
823	previous match to a length of one (thus producing a single literal
824	byte) and then emits the longer match. Otherwise, it emits the
825	original match, and, as described above, advances N bytes before
826	continuing.
827
828	Run-time parameters also control this "lazy match" procedure. If
829	compression ratio is most important, the compressor attempts a
830	complete second search regardless of the length of the first match.
831	In the normal case, if the current match is "long enough", the
832	compressor reduces the search for a longer match, thus speeding up
833	the process. If speed is most important, the compressor inserts new
834	strings in the hash table only when no match was found, or when the
835	match is not "too long". This degrades the compression ratio but
836	saves time since there are both fewer insertions and fewer searches.
837
838
839
840
841
842	Deutsch Informational [Page 15]
843
844	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
845
846
847	5. References
848
849	[1] Huffman, D. A., "A Method for the Construction of Minimum
850	Redundancy Codes", Proceedings of the Institute of Radio
851	Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101.
852
853	[2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data
854	Compression", IEEE Transactions on Information Theory, Vol. 23,
855	No. 3, pp. 337-343.
856
857	[3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources,
858	available in ftp://ftp.uu.net/pub/archiving/zip/doc/
859
860	[4] Gailly, J.-L., and Adler, M., GZIP documentation and sources,
861	available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/
862
863	[5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix
864	encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169.
865
866	[6] Hirschberg and Lelewer, "Efficient decoding of prefix codes,"
867	Comm. ACM, 33,4, April 1990, pp. 449-459.
868
869	6. Security Considerations
870
871	Any data compression method involves the reduction of redundancy in
872	the data. Consequently, any corruption of the data is likely to have
873	severe effects and be difficult to correct. Uncompressed text, on
874	the other hand, will probably still be readable despite the presence
875	of some corrupted bytes.
876
877	It is recommended that systems using this data format provide some
878	means of validating the integrity of the compressed data. See
879	reference [3], for example.
880
881	7. Source code
882
883	Source code for a C language implementation of a "deflate" compliant
884	compressor and decompressor is available within the zlib package at
885	ftp://ftp.uu.net/pub/archiving/zip/zlib/.
886
887	8. Acknowledgements
888
889	Trademarks cited in this document are the property of their
890	respective owners.
891
892	Phil Katz designed the deflate format. Jean-Loup Gailly and Mark
893	Adler wrote the related software described in this specification.
894	Glenn Randers-Pehrson converted this document to RFC and HTML format.
895
896
897
898	Deutsch Informational [Page 16]
899
900	RFC 1951 DEFLATE Compressed Data Format Specification May 1996
901
902
903	9. Author's Address
904
905	L. Peter Deutsch
906	Aladdin Enterprises
907	203 Santa Margarita Ave.
908	Menlo Park, CA 94025
909
910	Phone: (415) 322-0103 (AM only)
911	FAX: (415) 322-1734
912	EMail: <ghost@aladdin.com>
913
914	Questions about the technical content of this specification can be
915	sent by email to:
916
917	Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
918	Mark Adler <madler@alumni.caltech.edu>
919
920	Editorial comments on this specification can be sent by email to:
921
922	L. Peter Deutsch <ghost@aladdin.com> and
923	Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954	Deutsch Informational [Page 17]
955

lib/std/compress/deflate/token.zig created+104

...	@@ -0,0 +1,104 @@
1	// 2 bits: type, can be 0 (literal), 1 (EOF), 2 (Match) or 3 (Unused).
2	// 8 bits: xlength (length - MIN_MATCH_LENGTH).
3	// 22 bits: xoffset (offset - MIN_OFFSET_SIZE), or literal.
4	const length_shift = 22;
5	const offset_mask = (1 << length_shift) - 1; // 4_194_303
6	const literal_type = 0 << 30; // 0
7	pub const match_type = 1 << 30; // 1_073_741_824
8
9	// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
10	// is length_codes[length - MIN_MATCH_LENGTH]
11	var length_codes = [_]u32{
12	0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
13	9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
14	13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
15	15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
16	17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
17	18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
18	19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
19	20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20	21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21	21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
22	22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
23	22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
24	23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
25	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
26	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
27	24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
28	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
29	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
30	25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
31	25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
32	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
33	26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
34	26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
35	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
36	27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
37	27, 27, 27, 27, 27, 28,
38	};
39
40	var offset_codes = [_]u32{
41	0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
42	8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
43	10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
44	11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
45	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
46	12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
47	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
48	13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
49	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
50	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
51	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
52	14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
53	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
54	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
55	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
56	15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
57	};
58
59	pub const Token = u32;
60
61	// Convert a literal into a literal token.
62	pub fn literalToken(lit: u32) Token {
63	return literal_type + lit;
64	}
65
66	// Convert a < xlength, xoffset > pair into a match token.
67	pub fn matchToken(xlength: u32, xoffset: u32) Token {
68	return match_type + (xlength << length_shift) + xoffset;
69	}
70
71	// Returns the literal of a literal token
72	pub fn literal(t: Token) u32 {
73	return @intCast(u32, t - literal_type);
74	}
75
76	// Returns the extra offset of a match token
77	pub fn offset(t: Token) u32 {
78	return @intCast(u32, t) & offset_mask;
79	}
80
81	pub fn length(t: Token) u32 {
82	return @intCast(u32, (t - match_type) >> length_shift);
83	}
84
85	pub fn lengthCode(len: u32) u32 {
86	return length_codes[len];
87	}
88
89	// Returns the offset code corresponding to a specific offset
90	pub fn offsetCode(off: u32) u32 {
91	if (off < @intCast(u32, offset_codes.len)) {
92	return offset_codes[off];
93	}
94	if (off >> 7 < @intCast(u32, offset_codes.len)) {
95	return offset_codes[off >> 7] + 14;
96	}
97	return offset_codes[off >> 14] + 28;
98	}
99
100	test {
101	const std = @import("std");
102	const expect = std.testing.expect;
103	try expect(matchToken(555, 555) == 3_401_581_099);
104	}

lib/std/compress/gzip.zig+4-10

 @@ -20,15 +20,14 @@ pub fn GzipStream(comptime ReaderType: type) type {
         const Self = @This();
         pub const Error = ReaderType.Error ||
-            deflate.InflateStream(ReaderType).Error ||
+            deflate.Decompressor(ReaderType).Error ||
             error{ CorruptedData, WrongChecksum };
         pub const Reader = io.Reader(*Self, Error, read);
         allocator: mem.Allocator,
-        inflater: deflate.InflateStream(ReaderType),
+        inflater: deflate.Decompressor(ReaderType),
         in_reader: ReaderType,
         hasher: std.hash.Crc32,
-        window_slice: []u8,
         read_amt: usize,
         info: struct {
 @@ -93,16 +92,11 @@ pub fn GzipStream(comptime ReaderType: type) type {
                 _ = try source.readIntLittle(u16);
             }
-            // The RFC doesn't say anything about the DEFLATE window size to be
-            // used, default to 32K.
-            var window_slice = try allocator.alloc(u8, 32 * 1024);
             return Self{
                 .allocator = allocator,
-                .inflater = deflate.inflateStream(source, window_slice),
+                .inflater = try deflate.decompressor(allocator, source, null),
                 .in_reader = source,
                 .hasher = std.hash.Crc32.init(),
-                .window_slice = window_slice,
                 .info = .{
                     .filename = filename,
                     .comment = comment,
 @@ -113,7 +107,7 @@ pub fn GzipStream(comptime ReaderType: type) type {
         }
         pub fn deinit(self: *Self) void {
-            self.allocator.free(self.window_slice);
+            self.inflater.deinit();
             if (self.info.filename) |filename|
                 self.allocator.free(filename);
             if (self.info.comment) |comment|

lib/std/compress/zlib.zig+6-10

 @@ -13,15 +13,14 @@ pub fn ZlibStream(comptime ReaderType: type) type {
         const Self = @This();
         pub const Error = ReaderType.Error ||
-            deflate.InflateStream(ReaderType).Error ||
+            deflate.Decompressor(ReaderType).Error ||
             error{ WrongChecksum, Unsupported };
         pub const Reader = io.Reader(*Self, Error, read);
         allocator: mem.Allocator,
-        inflater: deflate.InflateStream(ReaderType),
+        inflater: deflate.Decompressor(ReaderType),
         in_reader: ReaderType,
         hasher: std.hash.Adler32,
-        window_slice: []u8,
         fn init(allocator: mem.Allocator, source: ReaderType) !Self {
             // Zlib header format is specified in RFC1950
 @@ -38,28 +37,25 @@ pub fn ZlibStream(comptime ReaderType: type) type {
             // The CM field must be 8 to indicate the use of DEFLATE
             if (CM != 8) return error.InvalidCompression;
-            // CINFO is the base-2 logarithm of the window size, minus 8.
+            // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.
             // Values above 7 are unspecified and therefore rejected.
             if (CINFO > 7) return error.InvalidWindowSize;
-            const window_size: u16 = @as(u16, 1) << (CINFO + 8);
+            const dictionary = null;
             // TODO: Support this case
             if (FDICT != 0)
                 return error.Unsupported;
-            var window_slice = try allocator.alloc(u8, window_size);
             return Self{
                 .allocator = allocator,
-                .inflater = deflate.inflateStream(source, window_slice),
+                .inflater = try deflate.decompressor(allocator, source, dictionary),
                 .in_reader = source,
                 .hasher = std.hash.Adler32.init(),
-                .window_slice = window_slice,
             };
         }
         pub fn deinit(self: *Self) void {
-            self.allocator.free(self.window_slice);
+            self.inflater.deinit();
         }
         // Implements the io.Reader interface

author	Hadrien Dorio <hadrien.dorio@gmail.com> 2022-01-09 02:12:00+01:00
committer	Hadrien Dorio <hadrien.dorio@gmail.com> 2022-01-23 19:30:06+01:00
log	490f067de82c8fd53029ddde0c73a242f4c04758
tree	4edb14646e6cfecc2f79a336bc764593fdd8c164
parent	dba04a272aba81f3be4d3419f1392e2c5a7e5dfb
signature	Commit is signed but in an unrecognized format.

compress: add a deflate compressor