authorgravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-02-14 22:34:13+01:00
committergravatar for igor.anic@gmail.comIgor Anić <igor.anic@gmail.com> 2024-02-14 22:34:13+01:00
log2457b68b2f008e7dfac6829c4ad7cfbb96e94c68
tree76b763d22a06084c3cb5be8efe0a0b52d2834796
parente20080be1312da92e441aabc6627fa2721282bad

remove v1 deflate implementation

78 files changed, 1 insertions(+), 9725 deletions(-)

build.zig+1-2
......@@ -150,8 +150,7 @@ pub fn build(b: *std.Build) !void {
150150 "rfc1951.txt",
151151 "rfc1952.txt",
152152 "rfc8478.txt",
153 // exclude files from lib/std/compress/deflate/testdata
154 // and lib/std/compress/flate/testdata
153 // exclude files from lib/std/compress/flate/testdata
155154 ".expect",
156155 ".expect-noinput",
157156 ".golden",
lib/std/compress.zig-10
......@@ -9,13 +9,6 @@ pub const flate = @import("compress/flate/root.zig").flate;
99pub const gzip = @import("compress/flate/root.zig").gzip;
1010pub const zlib = @import("compress/flate/root.zig").zlib;
1111
12// Version 1 interface
13pub const v1 = struct {
14 pub const deflate = @import("compress/deflate.zig");
15 pub const gzip = @import("compress/gzip.zig");
16 pub const zlib = @import("compress/zlib.zig");
17};
18
1912pub fn HashedReader(
2013 comptime ReaderType: anytype,
2114 comptime HasherType: anytype,
......@@ -77,12 +70,9 @@ pub fn hashedWriter(
7770}
7871
7972test {
80 _ = v1.deflate;
81 _ = v1.gzip;
8273 _ = lzma;
8374 _ = lzma2;
8475 _ = xz;
85 _ = v1.zlib;
8676 _ = zstd;
8777 _ = flate;
8878 _ = gzip;
lib/std/compress/deflate.zig deleted-44
......@@ -1,44 +0,0 @@
1//! The deflate package is a translation of the Go code of the compress/flate package from
2//! https://go.googlesource.com/go/+/refs/tags/go1.17/src/compress/flate/
3
4const deflate = @import("deflate/compressor.zig");
5const inflate = @import("deflate/decompressor.zig");
6
7pub const Compression = deflate.Compression;
8pub const CompressorOptions = deflate.CompressorOptions;
9pub const Compressor = deflate.Compressor;
10pub const Decompressor = inflate.Decompressor;
11
12pub const compressor = deflate.compressor;
13pub const decompressor = inflate.decompressor;
14
15/// Copies elements from a source `src` slice into a destination `dst` slice.
16/// The copy never returns an error but might not be complete if the destination is too small.
17/// Returns the number of elements copied, which will be the minimum of `src.len` and `dst.len`.
18/// TODO: remove this smelly function
19pub fn copy(dst: []u8, src: []const u8) usize {
20 if (dst.len <= src.len) {
21 @memcpy(dst, src[0..dst.len]);
22 return dst.len;
23 } else {
24 @memcpy(dst[0..src.len], src);
25 return src.len;
26 }
27}
28
29test {
30 _ = @import("deflate/token.zig");
31 _ = @import("deflate/bits_utils.zig");
32 _ = @import("deflate/dict_decoder.zig");
33
34 _ = @import("deflate/huffman_code.zig");
35 _ = @import("deflate/huffman_bit_writer.zig");
36
37 _ = @import("deflate/compressor.zig");
38 _ = @import("deflate/compressor_test.zig");
39
40 _ = @import("deflate/deflate_fast.zig");
41 _ = @import("deflate/deflate_fast_test.zig");
42
43 _ = @import("deflate/decompressor.zig");
44}
lib/std/compress/deflate/bits_utils.zig deleted-33
......@@ -1,33 +0,0 @@
1const math = @import("std").math;
2
3// Reverse bit-by-bit a N-bit code.
4pub fn bitReverse(comptime T: type, value: T, N: usize) T {
5 const r = @bitReverse(value);
6 return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).Int.bits - N));
7}
8
9test "bitReverse" {
10 const std = @import("std");
11
12 const ReverseBitsTest = struct {
13 in: u16,
14 bit_count: u5,
15 out: u16,
16 };
17
18 const reverse_bits_tests = [_]ReverseBitsTest{
19 .{ .in = 1, .bit_count = 1, .out = 1 },
20 .{ .in = 1, .bit_count = 2, .out = 2 },
21 .{ .in = 1, .bit_count = 3, .out = 4 },
22 .{ .in = 1, .bit_count = 4, .out = 8 },
23 .{ .in = 1, .bit_count = 5, .out = 16 },
24 .{ .in = 17, .bit_count = 5, .out = 17 },
25 .{ .in = 257, .bit_count = 9, .out = 257 },
26 .{ .in = 29, .bit_count = 5, .out = 23 },
27 };
28
29 for (reverse_bits_tests) |h| {
30 const v = bitReverse(u16, h.in, h.bit_count);
31 try std.testing.expectEqual(h.out, v);
32 }
33}
lib/std/compress/deflate/compressor.zig deleted-1110
......@@ -1,1110 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const fmt = std.fmt;
4const io = std.io;
5const math = std.math;
6const mem = std.mem;
7
8const Allocator = std.mem.Allocator;
9
10const deflate_const = @import("deflate_const.zig");
11const fast = @import("deflate_fast.zig");
12const hm_bw = @import("huffman_bit_writer.zig");
13const token = @import("token.zig");
14
15pub const Compression = enum(i5) {
16 /// huffman_only disables Lempel-Ziv match searching and only performs Huffman
17 /// entropy encoding. This mode is useful in compressing data that has
18 /// already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
19 /// that lacks an entropy encoder. Compression gains are achieved when
20 /// certain bytes in the input stream occur more frequently than others.
21 ///
22 /// Note that huffman_only produces a compressed output that is
23 /// RFC 1951 compliant. That is, any valid DEFLATE decompressor will
24 /// continue to be able to decompress this output.
25 huffman_only = -2,
26 /// Same as level_6
27 default_compression = -1,
28 /// Does not attempt any compression; only adds the necessary DEFLATE framing.
29 no_compression = 0,
30 /// Prioritizes speed over output size, based on Snappy's LZ77-style encoder
31 best_speed = 1,
32 level_2 = 2,
33 level_3 = 3,
34 level_4 = 4,
35 level_5 = 5,
36 level_6 = 6,
37 level_7 = 7,
38 level_8 = 8,
39 /// Prioritizes smaller output size over speed
40 best_compression = 9,
41};
42
43const log_window_size = 15;
44const window_size = 1 << log_window_size;
45const window_mask = window_size - 1;
46
47// The LZ77 step produces a sequence of literal tokens and <length, offset>
48// pair tokens. The offset is also known as distance. The underlying wire
49// format limits the range of lengths and offsets. For example, there are
50// 256 legitimate lengths: those in the range [3, 258]. This package's
51// compressor uses a higher minimum match length, enabling optimizations
52// such as finding matches via 32-bit loads and compares.
53const base_match_length = deflate_const.base_match_length; // The smallest match length per the RFC section 3.2.5
54const min_match_length = 4; // The smallest match length that the compressor actually emits
55const max_match_length = deflate_const.max_match_length;
56const base_match_offset = deflate_const.base_match_offset; // The smallest match offset
57const max_match_offset = deflate_const.max_match_offset; // The largest match offset
58
59// The maximum number of tokens we put into a single flate block, just to
60// stop things from getting too large.
61const max_flate_block_tokens = 1 << 14;
62const max_store_block_size = deflate_const.max_store_block_size;
63const hash_bits = 17; // After 17 performance degrades
64const hash_size = 1 << hash_bits;
65const hash_mask = (1 << hash_bits) - 1;
66const max_hash_offset = 1 << 24;
67
68const skip_never = math.maxInt(u32);
69
70const CompressionLevel = struct {
71 good: u16,
72 lazy: u16,
73 nice: u16,
74 chain: u16,
75 fast_skip_hashshing: u32,
76};
77
78fn levels(compression: Compression) CompressionLevel {
79 switch (compression) {
80 .no_compression,
81 .best_speed, // best_speed uses a custom algorithm; see deflate_fast.zig
82 .huffman_only,
83 => return .{
84 .good = 0,
85 .lazy = 0,
86 .nice = 0,
87 .chain = 0,
88 .fast_skip_hashshing = 0,
89 },
90 // For levels 2-3 we don't bother trying with lazy matches.
91 .level_2 => return .{
92 .good = 4,
93 .lazy = 0,
94 .nice = 16,
95 .chain = 8,
96 .fast_skip_hashshing = 5,
97 },
98 .level_3 => return .{
99 .good = 4,
100 .lazy = 0,
101 .nice = 32,
102 .chain = 32,
103 .fast_skip_hashshing = 6,
104 },
105
106 // Levels 4-9 use increasingly more lazy matching and increasingly stringent conditions for
107 // "good enough".
108 .level_4 => return .{
109 .good = 4,
110 .lazy = 4,
111 .nice = 16,
112 .chain = 16,
113 .fast_skip_hashshing = skip_never,
114 },
115 .level_5 => return .{
116 .good = 8,
117 .lazy = 16,
118 .nice = 32,
119 .chain = 32,
120 .fast_skip_hashshing = skip_never,
121 },
122 .default_compression,
123 .level_6,
124 => return .{
125 .good = 8,
126 .lazy = 16,
127 .nice = 128,
128 .chain = 128,
129 .fast_skip_hashshing = skip_never,
130 },
131 .level_7 => return .{
132 .good = 8,
133 .lazy = 32,
134 .nice = 128,
135 .chain = 256,
136 .fast_skip_hashshing = skip_never,
137 },
138 .level_8 => return .{
139 .good = 32,
140 .lazy = 128,
141 .nice = 258,
142 .chain = 1024,
143 .fast_skip_hashshing = skip_never,
144 },
145 .best_compression => return .{
146 .good = 32,
147 .lazy = 258,
148 .nice = 258,
149 .chain = 4096,
150 .fast_skip_hashshing = skip_never,
151 },
152 }
153}
154
155// matchLen returns the number of matching bytes in a and b
156// up to length 'max'. Both slices must be at least 'max'
157// bytes in size.
158fn matchLen(a: []u8, b: []u8, max: u32) u32 {
159 const bounded_a = a[0..max];
160 const bounded_b = b[0..max];
161 for (bounded_a, 0..) |av, i| {
162 if (bounded_b[i] != av) {
163 return @as(u32, @intCast(i));
164 }
165 }
166 return max;
167}
168
169const hash_mul = 0x1e35a7bd;
170
171// hash4 returns a hash representation of the first 4 bytes
172// of the supplied slice.
173// The caller must ensure that b.len >= 4.
174fn hash4(b: []u8) u32 {
175 return ((@as(u32, b[3]) |
176 @as(u32, b[2]) << 8 |
177 @as(u32, b[1]) << 16 |
178 @as(u32, b[0]) << 24) *% hash_mul) >> (32 - hash_bits);
179}
180
181// bulkHash4 will compute hashes using the same
182// algorithm as hash4
183fn bulkHash4(b: []u8, dst: []u32) u32 {
184 if (b.len < min_match_length) {
185 return 0;
186 }
187 var hb =
188 @as(u32, b[3]) |
189 @as(u32, b[2]) << 8 |
190 @as(u32, b[1]) << 16 |
191 @as(u32, b[0]) << 24;
192
193 dst[0] = (hb *% hash_mul) >> (32 - hash_bits);
194 const end = b.len - min_match_length + 1;
195 var i: u32 = 1;
196 while (i < end) : (i += 1) {
197 hb = (hb << 8) | @as(u32, b[i + 3]);
198 dst[i] = (hb *% hash_mul) >> (32 - hash_bits);
199 }
200
201 return hb;
202}
203
204pub const CompressorOptions = struct {
205 level: Compression = .default_compression,
206 dictionary: ?[]const u8 = null,
207};
208
209/// Returns a new Compressor compressing data at the given level.
210/// Following zlib, levels range from 1 (best_speed) to 9 (best_compression);
211/// higher levels typically run slower but compress more. Level 0
212/// (no_compression) does not attempt any compression; it only adds the
213/// necessary DEFLATE framing.
214/// Level -1 (default_compression) uses the default compression level.
215/// Level -2 (huffman_only) will use Huffman compression only, giving
216/// a very fast compression for all types of input, but sacrificing considerable
217/// compression efficiency.
218///
219/// `dictionary` is optional and initializes the new `Compressor` with a preset dictionary.
220/// The returned Compressor behaves as if the dictionary had been written to it without producing
221/// any compressed output. The compressed data written to hm_bw can only be decompressed by a
222/// Decompressor initialized with the same dictionary.
223///
224/// The compressed data will be passed to the provided `writer`, see `writer()` and `write()`.
225pub fn compressor(
226 allocator: Allocator,
227 writer: anytype,
228 options: CompressorOptions,
229) !Compressor(@TypeOf(writer)) {
230 return Compressor(@TypeOf(writer)).init(allocator, writer, options);
231}
232
233pub fn Compressor(comptime WriterType: anytype) type {
234 return struct {
235 const Self = @This();
236
237 /// A Writer takes data written to it and writes the compressed
238 /// form of that data to an underlying writer.
239 pub const Writer = io.Writer(*Self, Error, write);
240
241 /// Returns a Writer that takes data written to it and writes the compressed
242 /// form of that data to an underlying writer.
243 pub fn writer(self: *Self) Writer {
244 return .{ .context = self };
245 }
246
247 pub const Error = WriterType.Error;
248
249 allocator: Allocator,
250
251 compression: Compression,
252 compression_level: CompressionLevel,
253
254 // Inner writer wrapped in a HuffmanBitWriter
255 hm_bw: hm_bw.HuffmanBitWriter(WriterType) = undefined,
256 bulk_hasher: *const fn ([]u8, []u32) u32,
257
258 sync: bool, // requesting flush
259 best_speed_enc: *fast.DeflateFast, // Encoder for best_speed
260
261 // Input hash chains
262 // hash_head[hashValue] contains the largest inputIndex with the specified hash value
263 // If hash_head[hashValue] is within the current window, then
264 // hash_prev[hash_head[hashValue] & window_mask] contains the previous index
265 // with the same hash value.
266 chain_head: u32,
267 hash_head: []u32, // [hash_size]u32,
268 hash_prev: []u32, // [window_size]u32,
269 hash_offset: u32,
270
271 // input window: unprocessed data is window[index..window_end]
272 index: u32,
273 window: []u8,
274 window_end: usize,
275 block_start: usize, // window index where current tokens start
276 byte_available: bool, // if true, still need to process window[index-1].
277
278 // queued output tokens
279 tokens: []token.Token,
280 tokens_count: u16,
281
282 // deflate state
283 length: u32,
284 offset: u32,
285 hash: u32,
286 max_insert_index: usize,
287 err: bool,
288
289 // hash_match must be able to contain hashes for the maximum match length.
290 hash_match: []u32, // [max_match_length - 1]u32,
291
292 // dictionary
293 dictionary: ?[]const u8,
294
295 fn fillDeflate(self: *Self, b: []const u8) u32 {
296 if (self.index >= 2 * window_size - (min_match_length + max_match_length)) {
297 // shift the window by window_size
298 mem.copyForwards(u8, self.window, self.window[window_size .. 2 * window_size]);
299 self.index -= window_size;
300 self.window_end -= window_size;
301 if (self.block_start >= window_size) {
302 self.block_start -= window_size;
303 } else {
304 self.block_start = math.maxInt(u32);
305 }
306 self.hash_offset += window_size;
307 if (self.hash_offset > max_hash_offset) {
308 const delta = self.hash_offset - 1;
309 self.hash_offset -= delta;
310 self.chain_head -|= delta;
311
312 // Iterate over slices instead of arrays to avoid copying
313 // the entire table onto the stack (https://golang.org/issue/18625).
314 for (self.hash_prev, 0..) |v, i| {
315 if (v > delta) {
316 self.hash_prev[i] = @as(u32, @intCast(v - delta));
317 } else {
318 self.hash_prev[i] = 0;
319 }
320 }
321 for (self.hash_head, 0..) |v, i| {
322 if (v > delta) {
323 self.hash_head[i] = @as(u32, @intCast(v - delta));
324 } else {
325 self.hash_head[i] = 0;
326 }
327 }
328 }
329 }
330 const n = std.compress.v1.deflate.copy(self.window[self.window_end..], b);
331 self.window_end += n;
332 return @as(u32, @intCast(n));
333 }
334
335 fn writeBlock(self: *Self, tokens: []token.Token, index: usize) !void {
336 if (index > 0) {
337 var window: ?[]u8 = null;
338 if (self.block_start <= index) {
339 window = self.window[self.block_start..index];
340 }
341 self.block_start = index;
342 try self.hm_bw.writeBlock(tokens, false, window);
343 return;
344 }
345 return;
346 }
347
348 // fillWindow will fill the current window with the supplied
349 // dictionary and calculate all hashes.
350 // This is much faster than doing a full encode.
351 // Should only be used after a reset.
352 fn fillWindow(self: *Self, in_b: []const u8) void {
353 var b = in_b;
354 // Do not fill window if we are in store-only mode (look at the fill() function to see
355 // Compressions which use fillStore() instead of fillDeflate()).
356 if (self.compression == .no_compression or
357 self.compression == .huffman_only or
358 self.compression == .best_speed)
359 {
360 return;
361 }
362
363 // fillWindow() must not be called with stale data
364 assert(self.index == 0 and self.window_end == 0);
365
366 // If we are given too much, cut it.
367 if (b.len > window_size) {
368 b = b[b.len - window_size ..];
369 }
370 // Add all to window.
371 @memcpy(self.window[0..b.len], b);
372 const n = b.len;
373
374 // Calculate 256 hashes at the time (more L1 cache hits)
375 const loops = (n + 256 - min_match_length) / 256;
376 var j: usize = 0;
377 while (j < loops) : (j += 1) {
378 const index = j * 256;
379 var end = index + 256 + min_match_length - 1;
380 if (end > n) {
381 end = n;
382 }
383 const to_check = self.window[index..end];
384 const dst_size = to_check.len - min_match_length + 1;
385
386 if (dst_size <= 0) {
387 continue;
388 }
389
390 const dst = self.hash_match[0..dst_size];
391 _ = self.bulk_hasher(to_check, dst);
392 var new_h: u32 = 0;
393 for (dst, 0..) |val, i| {
394 const di = i + index;
395 new_h = val;
396 const hh = &self.hash_head[new_h & hash_mask];
397 // Get previous value with the same hash.
398 // Our chain should point to the previous value.
399 self.hash_prev[di & window_mask] = hh.*;
400 // Set the head of the hash chain to us.
401 hh.* = @as(u32, @intCast(di + self.hash_offset));
402 }
403 self.hash = new_h;
404 }
405 // Update window information.
406 self.window_end = n;
407 self.index = @as(u32, @intCast(n));
408 }
409
410 const Match = struct {
411 length: u32,
412 offset: u32,
413 ok: bool,
414 };
415
416 // Try to find a match starting at pos whose length is greater than prev_length.
417 // We only look at self.compression_level.chain possibilities before giving up.
418 fn findMatch(
419 self: *Self,
420 pos: u32,
421 prev_head: u32,
422 prev_length: u32,
423 lookahead: u32,
424 ) Match {
425 var length: u32 = 0;
426 var offset: u32 = 0;
427 var ok: bool = false;
428
429 var min_match_look: u32 = max_match_length;
430 if (lookahead < min_match_look) {
431 min_match_look = lookahead;
432 }
433
434 var win = self.window[0 .. pos + min_match_look];
435
436 // We quit when we get a match that's at least nice long
437 var nice = win.len - pos;
438 if (self.compression_level.nice < nice) {
439 nice = self.compression_level.nice;
440 }
441
442 // If we've got a match that's good enough, only look in 1/4 the chain.
443 var tries = self.compression_level.chain;
444 length = prev_length;
445 if (length >= self.compression_level.good) {
446 tries >>= 2;
447 }
448
449 var w_end = win[pos + length];
450 const w_pos = win[pos..];
451 const min_index = pos -| window_size;
452
453 var i = prev_head;
454 while (tries > 0) : (tries -= 1) {
455 if (w_end == win[i + length]) {
456 const n = matchLen(win[i..], w_pos, min_match_look);
457
458 if (n > length and (n > min_match_length or pos - i <= 4096)) {
459 length = n;
460 offset = pos - i;
461 ok = true;
462 if (n >= nice) {
463 // The match is good enough that we don't try to find a better one.
464 break;
465 }
466 w_end = win[pos + n];
467 }
468 }
469 if (i == min_index) {
470 // hash_prev[i & window_mask] has already been overwritten, so stop now.
471 break;
472 }
473
474 if (@as(u32, @intCast(self.hash_prev[i & window_mask])) < self.hash_offset) {
475 break;
476 }
477
478 i = @as(u32, @intCast(self.hash_prev[i & window_mask])) - self.hash_offset;
479 if (i < min_index) {
480 break;
481 }
482 }
483
484 return Match{ .length = length, .offset = offset, .ok = ok };
485 }
486
487 fn writeStoredBlock(self: *Self, buf: []u8) !void {
488 try self.hm_bw.writeStoredHeader(buf.len, false);
489 try self.hm_bw.writeBytes(buf);
490 }
491
492 // encSpeed will compress and store the currently added data,
493 // if enough has been accumulated or we at the end of the stream.
494 fn encSpeed(self: *Self) !void {
495 // We only compress if we have max_store_block_size.
496 if (self.window_end < max_store_block_size) {
497 if (!self.sync) {
498 return;
499 }
500
501 // Handle small sizes.
502 if (self.window_end < 128) {
503 switch (self.window_end) {
504 0 => return,
505 1...16 => {
506 try self.writeStoredBlock(self.window[0..self.window_end]);
507 },
508 else => {
509 try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
510 self.err = self.hm_bw.err;
511 },
512 }
513 self.window_end = 0;
514 self.best_speed_enc.reset();
515 return;
516 }
517 }
518 // Encode the block.
519 self.tokens_count = 0;
520 self.best_speed_enc.encode(
521 self.tokens,
522 &self.tokens_count,
523 self.window[0..self.window_end],
524 );
525
526 // If we removed less than 1/16th, Huffman compress the block.
527 if (self.tokens_count > self.window_end - (self.window_end >> 4)) {
528 try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
529 } else {
530 try self.hm_bw.writeBlockDynamic(
531 self.tokens[0..self.tokens_count],
532 false,
533 self.window[0..self.window_end],
534 );
535 }
536 self.err = self.hm_bw.err;
537 self.window_end = 0;
538 }
539
540 fn initDeflate(self: *Self) !void {
541 self.window = try self.allocator.alloc(u8, 2 * window_size);
542 self.hash_offset = 1;
543 self.tokens = try self.allocator.alloc(token.Token, max_flate_block_tokens);
544 self.tokens_count = 0;
545 @memset(self.tokens, 0);
546 self.length = min_match_length - 1;
547 self.offset = 0;
548 self.byte_available = false;
549 self.index = 0;
550 self.hash = 0;
551 self.chain_head = 0;
552 self.bulk_hasher = bulkHash4;
553 }
554
555 fn deflate(self: *Self) !void {
556 if (self.window_end - self.index < min_match_length + max_match_length and !self.sync) {
557 return;
558 }
559
560 self.max_insert_index = self.window_end -| (min_match_length - 1);
561 if (self.index < self.max_insert_index) {
562 self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
563 }
564
565 while (true) {
566 assert(self.index <= self.window_end);
567
568 const lookahead = self.window_end -| self.index;
569 if (lookahead < min_match_length + max_match_length) {
570 if (!self.sync) {
571 break;
572 }
573 assert(self.index <= self.window_end);
574
575 if (lookahead == 0) {
576 // Flush current output block if any.
577 if (self.byte_available) {
578 // There is still one pending token that needs to be flushed
579 self.tokens[self.tokens_count] = token.literalToken(@as(u32, @intCast(self.window[self.index - 1])));
580 self.tokens_count += 1;
581 self.byte_available = false;
582 }
583 if (self.tokens.len > 0) {
584 try self.writeBlock(self.tokens[0..self.tokens_count], self.index);
585 self.tokens_count = 0;
586 }
587 break;
588 }
589 }
590 if (self.index < self.max_insert_index) {
591 // Update the hash
592 self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
593 const hh = &self.hash_head[self.hash & hash_mask];
594 self.chain_head = @as(u32, @intCast(hh.*));
595 self.hash_prev[self.index & window_mask] = @as(u32, @intCast(self.chain_head));
596 hh.* = @as(u32, @intCast(self.index + self.hash_offset));
597 }
598 const prev_length = self.length;
599 const prev_offset = self.offset;
600 self.length = min_match_length - 1;
601 self.offset = 0;
602 const min_index = self.index -| window_size;
603
604 if (self.hash_offset <= self.chain_head and
605 self.chain_head - self.hash_offset >= min_index and
606 (self.compression_level.fast_skip_hashshing != skip_never and
607 lookahead > min_match_length - 1 or
608 self.compression_level.fast_skip_hashshing == skip_never and
609 lookahead > prev_length and
610 prev_length < self.compression_level.lazy))
611 {
612 {
613 const fmatch = self.findMatch(
614 self.index,
615 self.chain_head -| self.hash_offset,
616 min_match_length - 1,
617 @as(u32, @intCast(lookahead)),
618 );
619 if (fmatch.ok) {
620 self.length = fmatch.length;
621 self.offset = fmatch.offset;
622 }
623 }
624 }
625 if (self.compression_level.fast_skip_hashshing != skip_never and
626 self.length >= min_match_length or
627 self.compression_level.fast_skip_hashshing == skip_never and
628 prev_length >= min_match_length and
629 self.length <= prev_length)
630 {
631 // There was a match at the previous step, and the current match is
632 // not better. Output the previous match.
633 if (self.compression_level.fast_skip_hashshing != skip_never) {
634 self.tokens[self.tokens_count] = token.matchToken(@as(u32, @intCast(self.length - base_match_length)), @as(u32, @intCast(self.offset - base_match_offset)));
635 self.tokens_count += 1;
636 } else {
637 self.tokens[self.tokens_count] = token.matchToken(
638 @as(u32, @intCast(prev_length - base_match_length)),
639 @as(u32, @intCast(prev_offset -| base_match_offset)),
640 );
641 self.tokens_count += 1;
642 }
643 // Insert in the hash table all strings up to the end of the match.
644 // index and index-1 are already inserted. If there is not enough
645 // lookahead, the last two strings are not inserted into the hash
646 // table.
647 if (self.length <= self.compression_level.fast_skip_hashshing) {
648 var newIndex: u32 = 0;
649 if (self.compression_level.fast_skip_hashshing != skip_never) {
650 newIndex = self.index + self.length;
651 } else {
652 newIndex = self.index + prev_length - 1;
653 }
654 var index = self.index;
655 index += 1;
656 while (index < newIndex) : (index += 1) {
657 if (index < self.max_insert_index) {
658 self.hash = hash4(self.window[index .. index + min_match_length]);
659 // Get previous value with the same hash.
660 // Our chain should point to the previous value.
661 const hh = &self.hash_head[self.hash & hash_mask];
662 self.hash_prev[index & window_mask] = hh.*;
663 // Set the head of the hash chain to us.
664 hh.* = @as(u32, @intCast(index + self.hash_offset));
665 }
666 }
667 self.index = index;
668
669 if (self.compression_level.fast_skip_hashshing == skip_never) {
670 self.byte_available = false;
671 self.length = min_match_length - 1;
672 }
673 } else {
674 // For matches this long, we don't bother inserting each individual
675 // item into the table.
676 self.index += self.length;
677 if (self.index < self.max_insert_index) {
678 self.hash = hash4(self.window[self.index .. self.index + min_match_length]);
679 }
680 }
681 if (self.tokens_count == max_flate_block_tokens) {
682 // The block includes the current character
683 try self.writeBlock(self.tokens[0..self.tokens_count], self.index);
684 self.tokens_count = 0;
685 }
686 } else {
687 if (self.compression_level.fast_skip_hashshing != skip_never or self.byte_available) {
688 var i = self.index -| 1;
689 if (self.compression_level.fast_skip_hashshing != skip_never) {
690 i = self.index;
691 }
692 self.tokens[self.tokens_count] = token.literalToken(@as(u32, @intCast(self.window[i])));
693 self.tokens_count += 1;
694 if (self.tokens_count == max_flate_block_tokens) {
695 try self.writeBlock(self.tokens[0..self.tokens_count], i + 1);
696 self.tokens_count = 0;
697 }
698 }
699 self.index += 1;
700 if (self.compression_level.fast_skip_hashshing == skip_never) {
701 self.byte_available = true;
702 }
703 }
704 }
705 }
706
707 fn fillStore(self: *Self, b: []const u8) u32 {
708 const n = std.compress.v1.deflate.copy(self.window[self.window_end..], b);
709 self.window_end += n;
710 return @as(u32, @intCast(n));
711 }
712
713 fn store(self: *Self) !void {
714 if (self.window_end > 0 and (self.window_end == max_store_block_size or self.sync)) {
715 try self.writeStoredBlock(self.window[0..self.window_end]);
716 self.window_end = 0;
717 }
718 }
719
720 // storeHuff compresses and stores the currently added data
721 // when the self.window is full or we are at the end of the stream.
722 fn storeHuff(self: *Self) !void {
723 if (self.window_end < self.window.len and !self.sync or self.window_end == 0) {
724 return;
725 }
726 try self.hm_bw.writeBlockHuff(false, self.window[0..self.window_end]);
727 self.err = self.hm_bw.err;
728 self.window_end = 0;
729 }
730
731 pub fn bytesWritten(self: *Self) usize {
732 return self.hm_bw.bytes_written;
733 }
734
735 /// Writes the compressed form of `input` to the underlying writer.
736 pub fn write(self: *Self, input: []const u8) Error!usize {
737 var buf = input;
738
739 // writes data to hm_bw, which will eventually write the
740 // compressed form of data to its underlying writer.
741 while (buf.len > 0) {
742 try self.step();
743 const filled = self.fill(buf);
744 buf = buf[filled..];
745 }
746
747 return input.len;
748 }
749
750 /// Flushes any pending data to the underlying writer.
751 /// It is useful mainly in compressed network protocols, to ensure that
752 /// a remote reader has enough data to reconstruct a packet.
753 /// Flush does not return until the data has been written.
754 /// Calling `flush()` when there is no pending data still causes the Writer
755 /// to emit a sync marker of at least 4 bytes.
756 /// If the underlying writer returns an error, `flush()` returns that error.
757 ///
758 /// In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
759 pub fn flush(self: *Self) Error!void {
760 self.sync = true;
761 try self.step();
762 try self.hm_bw.writeStoredHeader(0, false);
763 try self.hm_bw.flush();
764 self.sync = false;
765 return;
766 }
767
768 fn step(self: *Self) !void {
769 switch (self.compression) {
770 .no_compression => return self.store(),
771 .huffman_only => return self.storeHuff(),
772 .best_speed => return self.encSpeed(),
773 .default_compression,
774 .level_2,
775 .level_3,
776 .level_4,
777 .level_5,
778 .level_6,
779 .level_7,
780 .level_8,
781 .best_compression,
782 => return self.deflate(),
783 }
784 }
785
786 fn fill(self: *Self, b: []const u8) u32 {
787 switch (self.compression) {
788 .no_compression => return self.fillStore(b),
789 .huffman_only => return self.fillStore(b),
790 .best_speed => return self.fillStore(b),
791 .default_compression,
792 .level_2,
793 .level_3,
794 .level_4,
795 .level_5,
796 .level_6,
797 .level_7,
798 .level_8,
799 .best_compression,
800 => return self.fillDeflate(b),
801 }
802 }
803
804 fn init(
805 allocator: Allocator,
806 in_writer: WriterType,
807 options: CompressorOptions,
808 ) !Self {
809 var s = Self{
810 .allocator = undefined,
811 .compression = undefined,
812 .compression_level = undefined,
813 .hm_bw = undefined, // HuffmanBitWriter
814 .bulk_hasher = undefined,
815 .sync = false,
816 .best_speed_enc = undefined, // Best speed encoder
817 .chain_head = 0,
818 .hash_head = undefined,
819 .hash_prev = undefined, // previous hash
820 .hash_offset = 0,
821 .index = 0,
822 .window = undefined,
823 .window_end = 0,
824 .block_start = 0,
825 .byte_available = false,
826 .tokens = undefined,
827 .tokens_count = 0,
828 .length = 0,
829 .offset = 0,
830 .hash = 0,
831 .max_insert_index = 0,
832 .err = false, // Error
833 .hash_match = undefined,
834 .dictionary = options.dictionary,
835 };
836
837 s.hm_bw = try hm_bw.huffmanBitWriter(allocator, in_writer);
838 s.allocator = allocator;
839
840 s.hash_head = try allocator.alloc(u32, hash_size);
841 s.hash_prev = try allocator.alloc(u32, window_size);
842 s.hash_match = try allocator.alloc(u32, max_match_length - 1);
843 @memset(s.hash_head, 0);
844 @memset(s.hash_prev, 0);
845 @memset(s.hash_match, 0);
846
847 switch (options.level) {
848 .no_compression => {
849 s.compression = options.level;
850 s.compression_level = levels(options.level);
851 s.window = try allocator.alloc(u8, max_store_block_size);
852 s.tokens = try allocator.alloc(token.Token, 0);
853 },
854 .huffman_only => {
855 s.compression = options.level;
856 s.compression_level = levels(options.level);
857 s.window = try allocator.alloc(u8, max_store_block_size);
858 s.tokens = try allocator.alloc(token.Token, 0);
859 },
860 .best_speed => {
861 s.compression = options.level;
862 s.compression_level = levels(options.level);
863 s.window = try allocator.alloc(u8, max_store_block_size);
864 s.tokens = try allocator.alloc(token.Token, max_store_block_size);
865 s.best_speed_enc = try allocator.create(fast.DeflateFast);
866 s.best_speed_enc.* = fast.deflateFast();
867 try s.best_speed_enc.init(allocator);
868 },
869 .default_compression => {
870 s.compression = .level_6;
871 s.compression_level = levels(.level_6);
872 try s.initDeflate();
873 if (options.dictionary != null) {
874 s.fillWindow(options.dictionary.?);
875 }
876 },
877 .level_2,
878 .level_3,
879 .level_4,
880 .level_5,
881 .level_6,
882 .level_7,
883 .level_8,
884 .best_compression,
885 => {
886 s.compression = options.level;
887 s.compression_level = levels(options.level);
888 try s.initDeflate();
889 if (options.dictionary != null) {
890 s.fillWindow(options.dictionary.?);
891 }
892 },
893 }
894 return s;
895 }
896
897 /// Release all allocated memory.
898 pub fn deinit(self: *Self) void {
899 self.hm_bw.deinit();
900 self.allocator.free(self.window);
901 self.allocator.free(self.tokens);
902 self.allocator.free(self.hash_head);
903 self.allocator.free(self.hash_prev);
904 self.allocator.free(self.hash_match);
905 if (self.compression == .best_speed) {
906 self.best_speed_enc.deinit();
907 self.allocator.destroy(self.best_speed_enc);
908 }
909 }
910
911 /// Reset discards the inner writer's state and replace the inner writer with new_writer.
912 /// new_writer must be of the same type as the previous writer.
913 pub fn reset(self: *Self, new_writer: WriterType) void {
914 self.hm_bw.reset(new_writer);
915 self.sync = false;
916 switch (self.compression) {
917 // Reset window
918 .no_compression => self.window_end = 0,
919 // Reset window, tokens, and encoder
920 .best_speed => {
921 self.window_end = 0;
922 self.tokens_count = 0;
923 self.best_speed_enc.reset();
924 },
925 // Reset everything and reinclude the dictionary if there is one
926 .huffman_only,
927 .default_compression,
928 .level_2,
929 .level_3,
930 .level_4,
931 .level_5,
932 .level_6,
933 .level_7,
934 .level_8,
935 .best_compression,
936 => {
937 self.chain_head = 0;
938 @memset(self.hash_head, 0);
939 @memset(self.hash_prev, 0);
940 self.hash_offset = 1;
941 self.index = 0;
942 self.window_end = 0;
943 self.block_start = 0;
944 self.byte_available = false;
945 self.tokens_count = 0;
946 self.length = min_match_length - 1;
947 self.offset = 0;
948 self.hash = 0;
949 self.max_insert_index = 0;
950
951 if (self.dictionary != null) {
952 self.fillWindow(self.dictionary.?);
953 }
954 },
955 }
956 }
957
958 /// Writes any pending data to the underlying writer.
959 pub fn close(self: *Self) Error!void {
960 self.sync = true;
961 try self.step();
962 try self.hm_bw.writeStoredHeader(0, true);
963 try self.hm_bw.flush();
964 return;
965 }
966 };
967}
968
969// tests
970
971const expect = std.testing.expect;
972const testing = std.testing;
973
974const ArrayList = std.ArrayList;
975
976const DeflateTest = struct {
977 in: []const u8,
978 level: Compression,
979 out: []const u8,
980};
981
982var deflate_tests = [_]DeflateTest{
983 // Level 0
984 .{
985 .in = &[_]u8{},
986 .level = .no_compression,
987 .out = &[_]u8{ 1, 0, 0, 255, 255 },
988 },
989
990 // Level -1
991 .{
992 .in = &[_]u8{0x11},
993 .level = .default_compression,
994 .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
995 },
996 .{
997 .in = &[_]u8{0x11},
998 .level = .level_6,
999 .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1000 },
1001
1002 // Level 4
1003 .{
1004 .in = &[_]u8{0x11},
1005 .level = .level_4,
1006 .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1007 },
1008
1009 // Level 0
1010 .{
1011 .in = &[_]u8{0x11},
1012 .level = .no_compression,
1013 .out = &[_]u8{ 0, 1, 0, 254, 255, 17, 1, 0, 0, 255, 255 },
1014 },
1015 .{
1016 .in = &[_]u8{ 0x11, 0x12 },
1017 .level = .no_compression,
1018 .out = &[_]u8{ 0, 2, 0, 253, 255, 17, 18, 1, 0, 0, 255, 255 },
1019 },
1020 .{
1021 .in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1022 .level = .no_compression,
1023 .out = &[_]u8{ 0, 8, 0, 247, 255, 17, 17, 17, 17, 17, 17, 17, 17, 1, 0, 0, 255, 255 },
1024 },
1025
1026 // Level 2
1027 .{
1028 .in = &[_]u8{},
1029 .level = .level_2,
1030 .out = &[_]u8{ 1, 0, 0, 255, 255 },
1031 },
1032 .{
1033 .in = &[_]u8{0x11},
1034 .level = .level_2,
1035 .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1036 },
1037 .{
1038 .in = &[_]u8{ 0x11, 0x12 },
1039 .level = .level_2,
1040 .out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 },
1041 },
1042 .{
1043 .in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1044 .level = .level_2,
1045 .out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 },
1046 },
1047
1048 // Level 9
1049 .{
1050 .in = &[_]u8{},
1051 .level = .best_compression,
1052 .out = &[_]u8{ 1, 0, 0, 255, 255 },
1053 },
1054 .{
1055 .in = &[_]u8{0x11},
1056 .level = .best_compression,
1057 .out = &[_]u8{ 18, 4, 4, 0, 0, 255, 255 },
1058 },
1059 .{
1060 .in = &[_]u8{ 0x11, 0x12 },
1061 .level = .best_compression,
1062 .out = &[_]u8{ 18, 20, 2, 4, 0, 0, 255, 255 },
1063 },
1064 .{
1065 .in = &[_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 },
1066 .level = .best_compression,
1067 .out = &[_]u8{ 18, 132, 2, 64, 0, 0, 0, 255, 255 },
1068 },
1069};
1070
1071test "deflate" {
1072 for (deflate_tests) |dt| {
1073 var output = ArrayList(u8).init(testing.allocator);
1074 defer output.deinit();
1075
1076 var comp = try compressor(testing.allocator, output.writer(), .{ .level = dt.level });
1077 _ = try comp.write(dt.in);
1078 try comp.close();
1079 comp.deinit();
1080
1081 try testing.expectEqualSlices(u8, dt.out, output.items);
1082 }
1083}
1084
1085test "bulkHash4" {
1086 for (deflate_tests) |x| {
1087 if (x.out.len < min_match_length) {
1088 continue;
1089 }
1090 // double the test data
1091 var out = try testing.allocator.alloc(u8, x.out.len * 2);
1092 defer testing.allocator.free(out);
1093 @memcpy(out[0..x.out.len], x.out);
1094 @memcpy(out[x.out.len..], x.out);
1095
1096 var j: usize = 4;
1097 while (j < out.len) : (j += 1) {
1098 var y = out[0..j];
1099
1100 const dst = try testing.allocator.alloc(u32, y.len - min_match_length + 1);
1101 defer testing.allocator.free(dst);
1102
1103 _ = bulkHash4(y, dst);
1104 for (dst, 0..) |got, i| {
1105 const want = hash4(y[i..]);
1106 try testing.expectEqual(want, got);
1107 }
1108 }
1109 }
1110}
lib/std/compress/deflate/compressor_test.zig deleted-531
......@@ -1,531 +0,0 @@
1const std = @import("std");
2const expect = std.testing.expect;
3const fifo = std.fifo;
4const io = std.io;
5const math = std.math;
6const mem = std.mem;
7const testing = std.testing;
8
9const ArrayList = std.ArrayList;
10
11const deflate = @import("compressor.zig");
12const inflate = @import("decompressor.zig");
13
14const compressor = deflate.compressor;
15const decompressor = inflate.decompressor;
16const huffman_only = deflate.huffman_only;
17
18fn testSync(level: deflate.Compression, input: []const u8) !void {
19 if (input.len == 0) {
20 return;
21 }
22
23 var divided_buf = fifo
24 .LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
25 .init(testing.allocator);
26 defer divided_buf.deinit();
27 var whole_buf = std.ArrayList(u8).init(testing.allocator);
28 defer whole_buf.deinit();
29
30 const multi_writer = io.multiWriter(.{
31 divided_buf.writer(),
32 whole_buf.writer(),
33 }).writer();
34
35 var comp = try compressor(
36 testing.allocator,
37 multi_writer,
38 .{ .level = level },
39 );
40 defer comp.deinit();
41
42 {
43 var decomp = try decompressor(
44 testing.allocator,
45 divided_buf.reader(),
46 null,
47 );
48 defer decomp.deinit();
49
50 // Write first half of the input and flush()
51 const half: usize = (input.len + 1) / 2;
52 var half_len: usize = half - 0;
53 {
54 _ = try comp.writer().writeAll(input[0..half]);
55
56 // Flush
57 try comp.flush();
58
59 // Read back
60 const decompressed = try testing.allocator.alloc(u8, half_len);
61 defer testing.allocator.free(decompressed);
62
63 const read = try decomp.reader().readAll(decompressed); // read at least half
64 try testing.expectEqual(half_len, read);
65 try testing.expectEqualSlices(u8, input[0..half], decompressed);
66 }
67
68 // Write last half of the input and close()
69 half_len = input.len - half;
70 {
71 _ = try comp.writer().writeAll(input[half..]);
72
73 // Close
74 try comp.close();
75
76 // Read back
77 const decompressed = try testing.allocator.alloc(u8, half_len);
78 defer testing.allocator.free(decompressed);
79
80 var read = try decomp.reader().readAll(decompressed);
81 try testing.expectEqual(half_len, read);
82 try testing.expectEqualSlices(u8, input[half..], decompressed);
83
84 // Extra read
85 var final: [10]u8 = undefined;
86 read = try decomp.reader().readAll(&final);
87 try testing.expectEqual(@as(usize, 0), read); // expect ended stream to return 0 bytes
88
89 try decomp.close();
90 }
91 }
92
93 _ = try comp.writer().writeAll(input);
94 try comp.close();
95
96 // stream should work for ordinary reader too (reading whole_buf in one go)
97 const whole_buf_reader = io.fixedBufferStream(whole_buf.items).reader();
98 var decomp = try decompressor(testing.allocator, whole_buf_reader, null);
99 defer decomp.deinit();
100
101 const decompressed = try testing.allocator.alloc(u8, input.len);
102 defer testing.allocator.free(decompressed);
103
104 _ = try decomp.reader().readAll(decompressed);
105 try decomp.close();
106
107 try testing.expectEqualSlices(u8, input, decompressed);
108}
109
110fn testToFromWithLevelAndLimit(level: deflate.Compression, input: []const u8, limit: u32) !void {
111 var compressed = std.ArrayList(u8).init(testing.allocator);
112 defer compressed.deinit();
113
114 var comp = try compressor(testing.allocator, compressed.writer(), .{ .level = level });
115 defer comp.deinit();
116
117 try comp.writer().writeAll(input);
118 try comp.close();
119
120 if (limit > 0) {
121 try expect(compressed.items.len <= limit);
122 }
123
124 var fib = io.fixedBufferStream(compressed.items);
125 var decomp = try decompressor(testing.allocator, fib.reader(), null);
126 defer decomp.deinit();
127
128 const decompressed = try testing.allocator.alloc(u8, input.len);
129 defer testing.allocator.free(decompressed);
130
131 const read: usize = try decomp.reader().readAll(decompressed);
132 try testing.expectEqual(input.len, read);
133 try testing.expectEqualSlices(u8, input, decompressed);
134
135 if (false) {
136 // TODO: this test has regressed
137 try testSync(level, input);
138 }
139}
140
141fn testToFromWithLimit(input: []const u8, limit: [11]u32) !void {
142 try testToFromWithLevelAndLimit(.no_compression, input, limit[0]);
143 try testToFromWithLevelAndLimit(.best_speed, input, limit[1]);
144 try testToFromWithLevelAndLimit(.level_2, input, limit[2]);
145 try testToFromWithLevelAndLimit(.level_3, input, limit[3]);
146 try testToFromWithLevelAndLimit(.level_4, input, limit[4]);
147 try testToFromWithLevelAndLimit(.level_5, input, limit[5]);
148 try testToFromWithLevelAndLimit(.level_6, input, limit[6]);
149 try testToFromWithLevelAndLimit(.level_7, input, limit[7]);
150 try testToFromWithLevelAndLimit(.level_8, input, limit[8]);
151 try testToFromWithLevelAndLimit(.best_compression, input, limit[9]);
152 try testToFromWithLevelAndLimit(.huffman_only, input, limit[10]);
153}
154
155test "deflate/inflate" {
156 const limits = [_]u32{0} ** 11;
157
158 var test0 = [_]u8{};
159 var test1 = [_]u8{0x11};
160 var test2 = [_]u8{ 0x11, 0x12 };
161 var test3 = [_]u8{ 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11 };
162 var test4 = [_]u8{ 0x11, 0x10, 0x13, 0x41, 0x21, 0x21, 0x41, 0x13, 0x87, 0x78, 0x13 };
163
164 try testToFromWithLimit(&test0, limits);
165 try testToFromWithLimit(&test1, limits);
166 try testToFromWithLimit(&test2, limits);
167 try testToFromWithLimit(&test3, limits);
168 try testToFromWithLimit(&test4, limits);
169
170 var large_data_chunk = try testing.allocator.alloc(u8, 100_000);
171 defer testing.allocator.free(large_data_chunk);
172 // fill with random data
173 for (large_data_chunk, 0..) |_, i| {
174 large_data_chunk[i] = @as(u8, @truncate(i)) *% @as(u8, @truncate(i));
175 }
176 try testToFromWithLimit(large_data_chunk, limits);
177}
178
179test "very long sparse chunk" {
180 // A SparseReader returns a stream consisting of 0s ending with 65,536 (1<<16) 1s.
181 // This tests missing hash references in a very large input.
182 const SparseReader = struct {
183 l: usize, // length
184 cur: usize, // current position
185
186 const Self = @This();
187 const Error = error{};
188
189 pub const Reader = io.Reader(*Self, Error, read);
190
191 pub fn reader(self: *Self) Reader {
192 return .{ .context = self };
193 }
194
195 fn read(s: *Self, b: []u8) Error!usize {
196 var n: usize = 0; // amount read
197
198 if (s.cur >= s.l) {
199 return 0;
200 }
201 n = b.len;
202 var cur = s.cur + n;
203 if (cur > s.l) {
204 n -= cur - s.l;
205 cur = s.l;
206 }
207 for (b[0..n], 0..) |_, i| {
208 if (s.cur + i >= s.l -| (1 << 16)) {
209 b[i] = 1;
210 } else {
211 b[i] = 0;
212 }
213 }
214 s.cur = cur;
215 return n;
216 }
217 };
218
219 var comp = try compressor(
220 testing.allocator,
221 io.null_writer,
222 .{ .level = .best_speed },
223 );
224 defer comp.deinit();
225 var writer = comp.writer();
226
227 var sparse = SparseReader{ .l = 0x23e8, .cur = 0 };
228 var reader = sparse.reader();
229
230 var read: usize = 1;
231 var written: usize = 0;
232 while (read > 0) {
233 var buf: [1 << 15]u8 = undefined; // 32,768 bytes buffer
234 read = try reader.read(&buf);
235 written += try writer.write(buf[0..read]);
236 }
237 try testing.expectEqual(@as(usize, 0x23e8), written);
238}
239
240test "compressor reset" {
241 for (std.enums.values(deflate.Compression)) |c| {
242 try testWriterReset(c, null);
243 try testWriterReset(c, "dict");
244 try testWriterReset(c, "hello");
245 }
246}
247
248fn testWriterReset(level: deflate.Compression, dict: ?[]const u8) !void {
249 const filler = struct {
250 fn writeData(c: anytype) !void {
251 const msg = "all your base are belong to us";
252 try c.writer().writeAll(msg);
253 try c.flush();
254
255 const hello = "hello world";
256 var i: usize = 0;
257 while (i < 1024) : (i += 1) {
258 try c.writer().writeAll(hello);
259 }
260
261 i = 0;
262 while (i < 65000) : (i += 1) {
263 try c.writer().writeAll("x");
264 }
265 }
266 };
267
268 var buf1 = ArrayList(u8).init(testing.allocator);
269 defer buf1.deinit();
270 var buf2 = ArrayList(u8).init(testing.allocator);
271 defer buf2.deinit();
272
273 var comp = try compressor(
274 testing.allocator,
275 buf1.writer(),
276 .{ .level = level, .dictionary = dict },
277 );
278 defer comp.deinit();
279
280 try filler.writeData(&comp);
281 try comp.close();
282
283 comp.reset(buf2.writer());
284 try filler.writeData(&comp);
285 try comp.close();
286
287 try testing.expectEqualSlices(u8, buf1.items, buf2.items);
288}
289
290test "decompressor dictionary" {
291 const dict = "hello world"; // dictionary
292 const text = "hello again world";
293
294 var compressed = fifo
295 .LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
296 .init(testing.allocator);
297 defer compressed.deinit();
298
299 var comp = try compressor(
300 testing.allocator,
301 compressed.writer(),
302 .{
303 .level = .level_5,
304 .dictionary = null, // no dictionary
305 },
306 );
307 defer comp.deinit();
308
309 // imitate a compressor with a dictionary
310 try comp.writer().writeAll(dict);
311 try comp.flush();
312 compressed.discard(compressed.readableLength()); // empty the output
313 try comp.writer().writeAll(text);
314 try comp.close();
315
316 const decompressed = try testing.allocator.alloc(u8, text.len);
317 defer testing.allocator.free(decompressed);
318
319 var decomp = try decompressor(
320 testing.allocator,
321 compressed.reader(),
322 dict,
323 );
324 defer decomp.deinit();
325
326 _ = try decomp.reader().readAll(decompressed);
327 try testing.expectEqualSlices(u8, "hello again world", decompressed);
328}
329
330test "compressor dictionary" {
331 const dict = "hello world";
332 const text = "hello again world";
333
334 var compressed_nd = fifo
335 .LinearFifo(u8, fifo.LinearFifoBufferType.Dynamic)
336 .init(testing.allocator); // compressed with no dictionary
337 defer compressed_nd.deinit();
338
339 var compressed_d = ArrayList(u8).init(testing.allocator); // compressed with a dictionary
340 defer compressed_d.deinit();
341
342 // imitate a compressor with a dictionary
343 var comp_nd = try compressor(
344 testing.allocator,
345 compressed_nd.writer(),
346 .{
347 .level = .level_5,
348 .dictionary = null, // no dictionary
349 },
350 );
351 defer comp_nd.deinit();
352 try comp_nd.writer().writeAll(dict);
353 try comp_nd.flush();
354 compressed_nd.discard(compressed_nd.readableLength()); // empty the output
355 try comp_nd.writer().writeAll(text);
356 try comp_nd.close();
357
358 // use a compressor with a dictionary
359 var comp_d = try compressor(
360 testing.allocator,
361 compressed_d.writer(),
362 .{
363 .level = .level_5,
364 .dictionary = dict, // with a dictionary
365 },
366 );
367 defer comp_d.deinit();
368 try comp_d.writer().writeAll(text);
369 try comp_d.close();
370
371 try testing.expectEqualSlices(u8, compressed_d.items, compressed_nd.readableSlice(0));
372}
373
374// Update the hash for best_speed only if d.index < d.maxInsertIndex
375// See https://golang.org/issue/2508
376test "Go non-regression test for 2508" {
377 var comp = try compressor(
378 testing.allocator,
379 io.null_writer,
380 .{ .level = .best_speed },
381 );
382 defer comp.deinit();
383
384 var buf = [_]u8{0} ** 1024;
385
386 var i: usize = 0;
387 while (i < 131_072) : (i += 1) {
388 try comp.writer().writeAll(&buf);
389 try comp.close();
390 }
391}
392
393test "deflate/inflate string" {
394 const StringTest = struct {
395 filename: []const u8,
396 limit: [11]u32,
397 };
398
399 const deflate_inflate_string_tests = [_]StringTest{
400 .{
401 .filename = "compress-e.txt",
402 .limit = [11]u32{
403 100_018, // no_compression
404 50_650, // best_speed
405 50_960, // 2
406 51_150, // 3
407 50_930, // 4
408 50_790, // 5
409 50_790, // 6
410 50_790, // 7
411 50_790, // 8
412 50_790, // best_compression
413 43_683, // huffman_only
414 },
415 },
416 .{
417 .filename = "rfc1951.txt",
418 .limit = [11]u32{
419 36_954, // no_compression
420 12_952, // best_speed
421 12_228, // 2
422 12_016, // 3
423 11_466, // 4
424 11_191, // 5
425 11_129, // 6
426 11_120, // 7
427 11_112, // 8
428 11_109, // best_compression
429 20_273, // huffman_only
430 },
431 },
432 };
433
434 inline for (deflate_inflate_string_tests) |t| {
435 const golden = @embedFile("testdata/" ++ t.filename);
436 try testToFromWithLimit(golden, t.limit);
437 }
438}
439
440test "inflate reset" {
441 const strings = [_][]const u8{
442 "lorem ipsum izzle fo rizzle",
443 "the quick brown fox jumped over",
444 };
445
446 var compressed_strings = [_]ArrayList(u8){
447 ArrayList(u8).init(testing.allocator),
448 ArrayList(u8).init(testing.allocator),
449 };
450 defer compressed_strings[0].deinit();
451 defer compressed_strings[1].deinit();
452
453 for (strings, 0..) |s, i| {
454 var comp = try compressor(
455 testing.allocator,
456 compressed_strings[i].writer(),
457 .{ .level = .level_6 },
458 );
459 defer comp.deinit();
460
461 try comp.writer().writeAll(s);
462 try comp.close();
463 }
464
465 var fib = io.fixedBufferStream(compressed_strings[0].items);
466 var decomp = try decompressor(testing.allocator, fib.reader(), null);
467 defer decomp.deinit();
468
469 const decompressed_0: []u8 = try decomp.reader()
470 .readAllAlloc(testing.allocator, math.maxInt(usize));
471 defer testing.allocator.free(decompressed_0);
472
473 fib = io.fixedBufferStream(compressed_strings[1].items);
474 try decomp.reset(fib.reader(), null);
475
476 const decompressed_1: []u8 = try decomp.reader()
477 .readAllAlloc(testing.allocator, math.maxInt(usize));
478 defer testing.allocator.free(decompressed_1);
479
480 try decomp.close();
481
482 try testing.expectEqualSlices(u8, strings[0], decompressed_0);
483 try testing.expectEqualSlices(u8, strings[1], decompressed_1);
484}
485
486test "inflate reset dictionary" {
487 const dict = "the lorem fox";
488 const strings = [_][]const u8{
489 "lorem ipsum izzle fo rizzle",
490 "the quick brown fox jumped over",
491 };
492
493 var compressed_strings = [_]ArrayList(u8){
494 ArrayList(u8).init(testing.allocator),
495 ArrayList(u8).init(testing.allocator),
496 };
497 defer compressed_strings[0].deinit();
498 defer compressed_strings[1].deinit();
499
500 for (strings, 0..) |s, i| {
501 var comp = try compressor(
502 testing.allocator,
503 compressed_strings[i].writer(),
504 .{ .level = .level_6 },
505 );
506 defer comp.deinit();
507
508 try comp.writer().writeAll(s);
509 try comp.close();
510 }
511
512 var fib = io.fixedBufferStream(compressed_strings[0].items);
513 var decomp = try decompressor(testing.allocator, fib.reader(), dict);
514 defer decomp.deinit();
515
516 const decompressed_0: []u8 = try decomp.reader()
517 .readAllAlloc(testing.allocator, math.maxInt(usize));
518 defer testing.allocator.free(decompressed_0);
519
520 fib = io.fixedBufferStream(compressed_strings[1].items);
521 try decomp.reset(fib.reader(), dict);
522
523 const decompressed_1: []u8 = try decomp.reader()
524 .readAllAlloc(testing.allocator, math.maxInt(usize));
525 defer testing.allocator.free(decompressed_1);
526
527 try decomp.close();
528
529 try testing.expectEqualSlices(u8, strings[0], decompressed_0);
530 try testing.expectEqualSlices(u8, strings[1], decompressed_1);
531}
lib/std/compress/deflate/decompressor.zig deleted-1119
......@@ -1,1119 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const math = std.math;
4const mem = std.mem;
5
6const Allocator = std.mem.Allocator;
7const ArrayList = std.ArrayList;
8
9const bu = @import("bits_utils.zig");
10const ddec = @import("dict_decoder.zig");
11const deflate_const = @import("deflate_const.zig");
12
13const max_match_offset = deflate_const.max_match_offset;
14const end_block_marker = deflate_const.end_block_marker;
15
16const max_code_len = 16; // max length of Huffman code
17// The next three numbers come from the RFC section 3.2.7, with the
18// additional proviso in section 3.2.5 which implies that distance codes
19// 30 and 31 should never occur in compressed data.
20const max_num_lit = 286;
21const max_num_dist = 30;
22const num_codes = 19; // number of codes in Huffman meta-code
23
24var corrupt_input_error_offset: u64 = undefined;
25
26const InflateError = error{
27 CorruptInput, // A CorruptInput error reports the presence of corrupt input at a given offset.
28 BadInternalState, // An BadInternalState reports an error in the flate code itself.
29 BadReaderState, // An error was encountered while accessing the inner reader
30 UnexpectedEndOfStream,
31 EndOfStreamWithNoError,
32};
33
34// The data structure for decoding Huffman tables is based on that of
35// zlib. There is a lookup table of a fixed bit width (huffman_chunk_bits),
36// For codes smaller than the table width, there are multiple entries
37// (each combination of trailing bits has the same value). For codes
38// larger than the table width, the table contains a link to an overflow
39// table. The width of each entry in the link table is the maximum code
40// size minus the chunk width.
41//
42// Note that you can do a lookup in the table even without all bits
43// filled. Since the extra bits are zero, and the DEFLATE Huffman codes
44// have the property that shorter codes come before longer ones, the
45// bit length estimate in the result is a lower bound on the actual
46// number of bits.
47//
48// See the following:
49// https://github.com/madler/zlib/raw/master/doc/algorithm.txt
50
51// chunk & 15 is number of bits
52// chunk >> 4 is value, including table link
53
54const huffman_chunk_bits = 9;
55const huffman_num_chunks = 1 << huffman_chunk_bits; // 512
56const huffman_count_mask = 15; // 0b1111
57const huffman_value_shift = 4;
58
59const HuffmanDecoder = struct {
60 const Self = @This();
61
62 allocator: Allocator = undefined,
63
64 min: u32 = 0, // the minimum code length
65 chunks: [huffman_num_chunks]u16 = [1]u16{0} ** huffman_num_chunks, // chunks as described above
66 links: [][]u16 = undefined, // overflow links
67 link_mask: u32 = 0, // mask the width of the link table
68 initialized: bool = false,
69 sub_chunks: ArrayList(u32) = undefined,
70
71 // Initialize Huffman decoding tables from array of code lengths.
72 // Following this function, self is guaranteed to be initialized into a complete
73 // tree (i.e., neither over-subscribed nor under-subscribed). The exception is a
74 // degenerate case where the tree has only a single symbol with length 1. Empty
75 // trees are permitted.
76 fn init(self: *Self, allocator: Allocator, lengths: []u32) !bool {
77
78 // Sanity enables additional runtime tests during Huffman
79 // table construction. It's intended to be used during
80 // development and debugging
81 const sanity = false;
82
83 if (self.min != 0) {
84 self.* = HuffmanDecoder{};
85 }
86
87 self.allocator = allocator;
88
89 // Count number of codes of each length,
90 // compute min and max length.
91 var count: [max_code_len]u32 = [1]u32{0} ** max_code_len;
92 var min: u32 = 0;
93 var max: u32 = 0;
94 for (lengths) |n| {
95 if (n == 0) {
96 continue;
97 }
98 if (min == 0) {
99 min = n;
100 }
101 min = @min(n, min);
102 max = @max(n, max);
103 count[n] += 1;
104 }
105
106 // Empty tree. The decompressor.huffSym function will fail later if the tree
107 // is used. Technically, an empty tree is only valid for the HDIST tree and
108 // not the HCLEN and HLIT tree. However, a stream with an empty HCLEN tree
109 // is guaranteed to fail since it will attempt to use the tree to decode the
110 // codes for the HLIT and HDIST trees. Similarly, an empty HLIT tree is
111 // guaranteed to fail later since the compressed data section must be
112 // composed of at least one symbol (the end-of-block marker).
113 if (max == 0) {
114 return true;
115 }
116
117 var next_code: [max_code_len]u32 = [1]u32{0} ** max_code_len;
118 var code: u32 = 0;
119 {
120 var i = min;
121 while (i <= max) : (i += 1) {
122 code <<= 1;
123 next_code[i] = code;
124 code += count[i];
125 }
126 }
127
128 // Check that the coding is complete (i.e., that we've
129 // assigned all 2-to-the-max possible bit sequences).
130 // Exception: To be compatible with zlib, we also need to
131 // accept degenerate single-code codings. See also
132 // TestDegenerateHuffmanCoding.
133 if (code != @as(u32, 1) << @as(u5, @intCast(max)) and !(code == 1 and max == 1)) {
134 return false;
135 }
136
137 self.min = min;
138 if (max > huffman_chunk_bits) {
139 const num_links = @as(u32, 1) << @as(u5, @intCast(max - huffman_chunk_bits));
140 self.link_mask = @as(u32, @intCast(num_links - 1));
141
142 // create link tables
143 const link = next_code[huffman_chunk_bits + 1] >> 1;
144 self.links = try self.allocator.alloc([]u16, huffman_num_chunks - link);
145 self.sub_chunks = ArrayList(u32).init(self.allocator);
146 self.initialized = true;
147 var j = @as(u32, @intCast(link));
148 while (j < huffman_num_chunks) : (j += 1) {
149 var reverse = @as(u32, @intCast(bu.bitReverse(u16, @as(u16, @intCast(j)), 16)));
150 reverse >>= @as(u32, @intCast(16 - huffman_chunk_bits));
151 const off = j - @as(u32, @intCast(link));
152 if (sanity) {
153 // check we are not overwriting an existing chunk
154 assert(self.chunks[reverse] == 0);
155 }
156 self.chunks[reverse] = @as(u16, @intCast(off << huffman_value_shift | (huffman_chunk_bits + 1)));
157 self.links[off] = try self.allocator.alloc(u16, num_links);
158 if (sanity) {
159 // initialize to a known invalid chunk code (0) to see if we overwrite
160 // this value later on
161 @memset(self.links[off], 0);
162 }
163 try self.sub_chunks.append(off);
164 }
165 }
166
167 for (lengths, 0..) |n, li| {
168 if (n == 0) {
169 continue;
170 }
171 const ncode = next_code[n];
172 next_code[n] += 1;
173 const chunk = @as(u16, @intCast((li << huffman_value_shift) | n));
174 var reverse = @as(u16, @intCast(bu.bitReverse(u16, @as(u16, @intCast(ncode)), 16)));
175 reverse >>= @as(u4, @intCast(16 - n));
176 if (n <= huffman_chunk_bits) {
177 var off = reverse;
178 while (off < self.chunks.len) : (off += @as(u16, 1) << @as(u4, @intCast(n))) {
179 // We should never need to overwrite
180 // an existing chunk. Also, 0 is
181 // never a valid chunk, because the
182 // lower 4 "count" bits should be
183 // between 1 and 15.
184 if (sanity) {
185 assert(self.chunks[off] == 0);
186 }
187 self.chunks[off] = chunk;
188 }
189 } else {
190 const j = reverse & (huffman_num_chunks - 1);
191 if (sanity) {
192 // Expect an indirect chunk
193 assert(self.chunks[j] & huffman_count_mask == huffman_chunk_bits + 1);
194 // Longer codes should have been
195 // associated with a link table above.
196 }
197 const value = self.chunks[j] >> huffman_value_shift;
198 var link_tab = self.links[value];
199 reverse >>= huffman_chunk_bits;
200 var off = reverse;
201 while (off < link_tab.len) : (off += @as(u16, 1) << @as(u4, @intCast(n - huffman_chunk_bits))) {
202 if (sanity) {
203 // check we are not overwriting an existing chunk
204 assert(link_tab[off] == 0);
205 }
206 link_tab[off] = @as(u16, @intCast(chunk));
207 }
208 }
209 }
210
211 if (sanity) {
212 // Above we've sanity checked that we never overwrote
213 // an existing entry. Here we additionally check that
214 // we filled the tables completely.
215 for (self.chunks, 0..) |chunk, i| {
216 // As an exception, in the degenerate
217 // single-code case, we allow odd
218 // chunks to be missing.
219 if (code == 1 and i % 2 == 1) {
220 continue;
221 }
222
223 // Assert we are not missing a chunk.
224 // All chunks should have been written once
225 // thus losing their initial value of 0
226 assert(chunk != 0);
227 }
228
229 if (self.initialized) {
230 for (self.links) |link_tab| {
231 for (link_tab) |chunk| {
232 // Assert we are not missing a chunk.
233 assert(chunk != 0);
234 }
235 }
236 }
237 }
238
239 return true;
240 }
241
242 /// Release all allocated memory.
243 pub fn deinit(self: *Self) void {
244 if (self.initialized and self.links.len > 0) {
245 for (self.sub_chunks.items) |off| {
246 self.allocator.free(self.links[off]);
247 }
248 self.allocator.free(self.links);
249 self.sub_chunks.deinit();
250 self.initialized = false;
251 }
252 }
253};
254
255var fixed_huffman_decoder: ?HuffmanDecoder = null;
256
257fn fixedHuffmanDecoderInit(allocator: Allocator) !HuffmanDecoder {
258 if (fixed_huffman_decoder != null) {
259 return fixed_huffman_decoder.?;
260 }
261
262 // These come from the RFC section 3.2.6.
263 var bits: [288]u32 = undefined;
264 var i: u32 = 0;
265 while (i < 144) : (i += 1) {
266 bits[i] = 8;
267 }
268 while (i < 256) : (i += 1) {
269 bits[i] = 9;
270 }
271 while (i < 280) : (i += 1) {
272 bits[i] = 7;
273 }
274 while (i < 288) : (i += 1) {
275 bits[i] = 8;
276 }
277
278 fixed_huffman_decoder = HuffmanDecoder{};
279 _ = try fixed_huffman_decoder.?.init(allocator, &bits);
280 return fixed_huffman_decoder.?;
281}
282
283const DecompressorState = enum {
284 init,
285 dict,
286};
287
288/// Returns a new Decompressor that can be used to read the uncompressed version of `reader`.
289/// `dictionary` is optional and initializes the Decompressor with a preset dictionary.
290/// The returned Decompressor behaves as if the uncompressed data stream started with the given
291/// dictionary, which has already been read. Use the same `dictionary` as the compressor used to
292/// compress the data.
293/// This decompressor may use at most 300 KiB of heap memory from the provided allocator.
294/// The uncompressed data will be written into the provided buffer, see `reader()` and `read()`.
295pub fn decompressor(allocator: Allocator, reader: anytype, dictionary: ?[]const u8) !Decompressor(@TypeOf(reader)) {
296 return Decompressor(@TypeOf(reader)).init(allocator, reader, dictionary);
297}
298
299pub fn Decompressor(comptime ReaderType: type) type {
300 return struct {
301 const Self = @This();
302
303 pub const Error =
304 ReaderType.Error ||
305 error{EndOfStream} ||
306 InflateError ||
307 Allocator.Error;
308 pub const Reader = io.Reader(*Self, Error, read);
309
310 allocator: Allocator,
311
312 // Input source.
313 inner_reader: ReaderType,
314 roffset: u64,
315
316 // Input bits, in top of b.
317 b: u32,
318 nb: u32,
319
320 // Huffman decoders for literal/length, distance.
321 hd1: HuffmanDecoder,
322 hd2: HuffmanDecoder,
323
324 // Length arrays used to define Huffman codes.
325 bits: *[max_num_lit + max_num_dist]u32,
326 codebits: *[num_codes]u32,
327
328 // Output history, buffer.
329 dict: ddec.DictDecoder,
330
331 // Temporary buffer (avoids repeated allocation).
332 buf: [4]u8,
333
334 // Next step in the decompression,
335 // and decompression state.
336 step: *const fn (*Self) Error!void,
337 step_state: DecompressorState,
338 final: bool,
339 err: ?Error,
340 to_read: []u8,
341 // Huffman states for the lit/length values
342 hl: ?*HuffmanDecoder,
343 // Huffman states for the distance values.
344 hd: ?*HuffmanDecoder,
345 copy_len: u32,
346 copy_dist: u32,
347
348 /// Returns a Reader that reads compressed data from an underlying reader and outputs
349 /// uncompressed data.
350 pub fn reader(self: *Self) Reader {
351 return .{ .context = self };
352 }
353
354 fn init(allocator: Allocator, in_reader: ReaderType, dict: ?[]const u8) !Self {
355 fixed_huffman_decoder = try fixedHuffmanDecoderInit(allocator);
356
357 const bits = try allocator.create([max_num_lit + max_num_dist]u32);
358 const codebits = try allocator.create([num_codes]u32);
359
360 var dd = ddec.DictDecoder{};
361 try dd.init(allocator, max_match_offset, dict);
362
363 return Self{
364 .allocator = allocator,
365
366 // Input source.
367 .inner_reader = in_reader,
368 .roffset = 0,
369
370 // Input bits, in top of b.
371 .b = 0,
372 .nb = 0,
373
374 // Huffman decoders for literal/length, distance.
375 .hd1 = HuffmanDecoder{},
376 .hd2 = HuffmanDecoder{},
377
378 // Length arrays used to define Huffman codes.
379 .bits = bits,
380 .codebits = codebits,
381
382 // Output history, buffer.
383 .dict = dd,
384
385 // Temporary buffer (avoids repeated allocation).
386 .buf = [_]u8{0} ** 4,
387
388 // Next step in the decompression and decompression state.
389 .step = nextBlock,
390 .step_state = .init,
391 .final = false,
392 .err = null,
393 .to_read = &[0]u8{},
394 .hl = null,
395 .hd = null,
396 .copy_len = 0,
397 .copy_dist = 0,
398 };
399 }
400
401 /// Release all allocated memory.
402 pub fn deinit(self: *Self) void {
403 self.hd2.deinit();
404 self.hd1.deinit();
405 self.dict.deinit();
406 self.allocator.destroy(self.codebits);
407 self.allocator.destroy(self.bits);
408 }
409
410 fn nextBlock(self: *Self) Error!void {
411 while (self.nb < 1 + 2) {
412 self.moreBits() catch |e| {
413 self.err = e;
414 return e;
415 };
416 }
417 self.final = self.b & 1 == 1;
418 self.b >>= 1;
419 const typ = self.b & 3;
420 self.b >>= 2;
421 self.nb -= 1 + 2;
422 switch (typ) {
423 0 => try self.dataBlock(),
424 1 => {
425 // compressed, fixed Huffman tables
426 self.hl = &fixed_huffman_decoder.?;
427 self.hd = null;
428 try self.huffmanBlock();
429 },
430 2 => {
431 // compressed, dynamic Huffman tables
432 self.hd2.deinit();
433 self.hd1.deinit();
434 try self.readHuffman();
435 self.hl = &self.hd1;
436 self.hd = &self.hd2;
437 try self.huffmanBlock();
438 },
439 else => {
440 // 3 is reserved.
441 corrupt_input_error_offset = self.roffset;
442 self.err = InflateError.CorruptInput;
443 return InflateError.CorruptInput;
444 },
445 }
446 }
447
448 /// Reads compressed data from the underlying reader and outputs uncompressed data into
449 /// `output`.
450 pub fn read(self: *Self, output: []u8) Error!usize {
451 while (true) {
452 if (self.to_read.len > 0) {
453 const n = std.compress.v1.deflate.copy(output, self.to_read);
454 self.to_read = self.to_read[n..];
455 if (self.to_read.len == 0 and
456 self.err != null)
457 {
458 if (self.err.? == InflateError.EndOfStreamWithNoError) {
459 return n;
460 }
461 return self.err.?;
462 }
463 return n;
464 }
465 if (self.err != null) {
466 if (self.err.? == InflateError.EndOfStreamWithNoError) {
467 return 0;
468 }
469 return self.err.?;
470 }
471 self.step(self) catch |e| {
472 self.err = e;
473 if (self.to_read.len == 0) {
474 self.to_read = self.dict.readFlush(); // Flush what's left in case of error
475 }
476 };
477 }
478 }
479
480 pub fn close(self: *Self) Error!void {
481 if (self.err) |err| {
482 if (err != error.EndOfStreamWithNoError) return err;
483 }
484 }
485
486 // RFC 1951 section 3.2.7.
487 // Compression with dynamic Huffman codes
488
489 const code_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
490
491 fn readHuffman(self: *Self) Error!void {
492 // HLIT[5], HDIST[5], HCLEN[4].
493 while (self.nb < 5 + 5 + 4) {
494 try self.moreBits();
495 }
496 const nlit = @as(u32, @intCast(self.b & 0x1F)) + 257;
497 if (nlit > max_num_lit) {
498 corrupt_input_error_offset = self.roffset;
499 self.err = InflateError.CorruptInput;
500 return InflateError.CorruptInput;
501 }
502 self.b >>= 5;
503 const ndist = @as(u32, @intCast(self.b & 0x1F)) + 1;
504 if (ndist > max_num_dist) {
505 corrupt_input_error_offset = self.roffset;
506 self.err = InflateError.CorruptInput;
507 return InflateError.CorruptInput;
508 }
509 self.b >>= 5;
510 const nclen = @as(u32, @intCast(self.b & 0xF)) + 4;
511 // num_codes is 19, so nclen is always valid.
512 self.b >>= 4;
513 self.nb -= 5 + 5 + 4;
514
515 // (HCLEN+4)*3 bits: code lengths in the magic code_order order.
516 var i: u32 = 0;
517 while (i < nclen) : (i += 1) {
518 while (self.nb < 3) {
519 try self.moreBits();
520 }
521 self.codebits[code_order[i]] = @as(u32, @intCast(self.b & 0x7));
522 self.b >>= 3;
523 self.nb -= 3;
524 }
525 i = nclen;
526 while (i < code_order.len) : (i += 1) {
527 self.codebits[code_order[i]] = 0;
528 }
529 if (!try self.hd1.init(self.allocator, self.codebits[0..])) {
530 corrupt_input_error_offset = self.roffset;
531 self.err = InflateError.CorruptInput;
532 return InflateError.CorruptInput;
533 }
534
535 // HLIT + 257 code lengths, HDIST + 1 code lengths,
536 // using the code length Huffman code.
537 i = 0;
538 const n = nlit + ndist;
539 while (i < n) {
540 const x = try self.huffSym(&self.hd1);
541 if (x < 16) {
542 // Actual length.
543 self.bits[i] = x;
544 i += 1;
545 continue;
546 }
547 // Repeat previous length or zero.
548 var rep: u32 = 0;
549 var nb: u32 = 0;
550 var b: u32 = 0;
551 switch (x) {
552 16 => {
553 rep = 3;
554 nb = 2;
555 if (i == 0) {
556 corrupt_input_error_offset = self.roffset;
557 self.err = InflateError.CorruptInput;
558 return InflateError.CorruptInput;
559 }
560 b = self.bits[i - 1];
561 },
562 17 => {
563 rep = 3;
564 nb = 3;
565 b = 0;
566 },
567 18 => {
568 rep = 11;
569 nb = 7;
570 b = 0;
571 },
572 else => return error.BadInternalState, // unexpected length code
573 }
574 while (self.nb < nb) {
575 try self.moreBits();
576 }
577 rep += @as(u32, @intCast(self.b & (@as(u32, 1) << @as(u5, @intCast(nb))) - 1));
578 self.b >>= @as(u5, @intCast(nb));
579 self.nb -= nb;
580 if (i + rep > n) {
581 corrupt_input_error_offset = self.roffset;
582 self.err = InflateError.CorruptInput;
583 return InflateError.CorruptInput;
584 }
585 var j: u32 = 0;
586 while (j < rep) : (j += 1) {
587 self.bits[i] = b;
588 i += 1;
589 }
590 }
591
592 if (!try self.hd1.init(self.allocator, self.bits[0..nlit]) or
593 !try self.hd2.init(self.allocator, self.bits[nlit..][0..ndist]))
594 {
595 corrupt_input_error_offset = self.roffset;
596 self.err = InflateError.CorruptInput;
597 return InflateError.CorruptInput;
598 }
599
600 // As an optimization, we can initialize the min bits to read at a time
601 // for the HLIT tree to the length of the EOB marker since we know that
602 // every block must terminate with one. This preserves the property that
603 // we never read any extra bytes after the end of the DEFLATE stream.
604 if (self.hd1.min < self.bits[end_block_marker]) {
605 self.hd1.min = self.bits[end_block_marker];
606 }
607
608 return;
609 }
610
611 // Decode a single Huffman block.
612 // hl and hd are the Huffman states for the lit/length values
613 // and the distance values, respectively. If hd == null, using the
614 // fixed distance encoding associated with fixed Huffman blocks.
615 fn huffmanBlock(self: *Self) Error!void {
616 while (true) {
617 switch (self.step_state) {
618 .init => {
619 // Read literal and/or (length, distance) according to RFC section 3.2.3.
620 const v = try self.huffSym(self.hl.?);
621 var n: u32 = 0; // number of bits extra
622 var length: u32 = 0;
623 switch (v) {
624 0...255 => {
625 self.dict.writeByte(@as(u8, @intCast(v)));
626 if (self.dict.availWrite() == 0) {
627 self.to_read = self.dict.readFlush();
628 self.step = huffmanBlock;
629 self.step_state = .init;
630 return;
631 }
632 self.step_state = .init;
633 continue;
634 },
635 256 => {
636 self.finishBlock();
637 return;
638 },
639 // otherwise, reference to older data
640 257...264 => {
641 length = v - (257 - 3);
642 n = 0;
643 },
644 265...268 => {
645 length = v * 2 - (265 * 2 - 11);
646 n = 1;
647 },
648 269...272 => {
649 length = v * 4 - (269 * 4 - 19);
650 n = 2;
651 },
652 273...276 => {
653 length = v * 8 - (273 * 8 - 35);
654 n = 3;
655 },
656 277...280 => {
657 length = v * 16 - (277 * 16 - 67);
658 n = 4;
659 },
660 281...284 => {
661 length = v * 32 - (281 * 32 - 131);
662 n = 5;
663 },
664 max_num_lit - 1 => { // 285
665 length = 258;
666 n = 0;
667 },
668 else => {
669 corrupt_input_error_offset = self.roffset;
670 self.err = InflateError.CorruptInput;
671 return InflateError.CorruptInput;
672 },
673 }
674 if (n > 0) {
675 while (self.nb < n) {
676 try self.moreBits();
677 }
678 length += @as(u32, @intCast(self.b)) & ((@as(u32, 1) << @as(u5, @intCast(n))) - 1);
679 self.b >>= @as(u5, @intCast(n));
680 self.nb -= n;
681 }
682
683 var dist: u32 = 0;
684 if (self.hd == null) {
685 while (self.nb < 5) {
686 try self.moreBits();
687 }
688 dist = @as(
689 u32,
690 @intCast(bu.bitReverse(u8, @as(u8, @intCast((self.b & 0x1F) << 3)), 8)),
691 );
692 self.b >>= 5;
693 self.nb -= 5;
694 } else {
695 dist = try self.huffSym(self.hd.?);
696 }
697
698 switch (dist) {
699 0...3 => dist += 1,
700 4...max_num_dist - 1 => { // 4...29
701 const nb = @as(u32, @intCast(dist - 2)) >> 1;
702 // have 1 bit in bottom of dist, need nb more.
703 var extra = (dist & 1) << @as(u5, @intCast(nb));
704 while (self.nb < nb) {
705 try self.moreBits();
706 }
707 extra |= @as(u32, @intCast(self.b & (@as(u32, 1) << @as(u5, @intCast(nb))) - 1));
708 self.b >>= @as(u5, @intCast(nb));
709 self.nb -= nb;
710 dist = (@as(u32, 1) << @as(u5, @intCast(nb + 1))) + 1 + extra;
711 },
712 else => {
713 corrupt_input_error_offset = self.roffset;
714 self.err = InflateError.CorruptInput;
715 return InflateError.CorruptInput;
716 },
717 }
718
719 // No check on length; encoding can be prescient.
720 if (dist > self.dict.histSize()) {
721 corrupt_input_error_offset = self.roffset;
722 self.err = InflateError.CorruptInput;
723 return InflateError.CorruptInput;
724 }
725
726 self.copy_len = length;
727 self.copy_dist = dist;
728 self.step_state = .dict;
729 },
730
731 .dict => {
732 // Perform a backwards copy according to RFC section 3.2.3.
733 var cnt = self.dict.tryWriteCopy(self.copy_dist, self.copy_len);
734 if (cnt == 0) {
735 cnt = self.dict.writeCopy(self.copy_dist, self.copy_len);
736 }
737 self.copy_len -= cnt;
738
739 if (self.dict.availWrite() == 0 or self.copy_len > 0) {
740 self.to_read = self.dict.readFlush();
741 self.step = huffmanBlock; // We need to continue this work
742 self.step_state = .dict;
743 return;
744 }
745 self.step_state = .init;
746 },
747 }
748 }
749 }
750
751 // Copy a single uncompressed data block from input to output.
752 fn dataBlock(self: *Self) Error!void {
753 // Uncompressed.
754 // Discard current half-byte.
755 self.nb = 0;
756 self.b = 0;
757
758 // Length then ones-complement of length.
759 const nr: u32 = 4;
760 self.inner_reader.readNoEof(self.buf[0..nr]) catch {
761 self.err = InflateError.UnexpectedEndOfStream;
762 return InflateError.UnexpectedEndOfStream;
763 };
764 self.roffset += @as(u64, @intCast(nr));
765 const n = @as(u32, @intCast(self.buf[0])) | @as(u32, @intCast(self.buf[1])) << 8;
766 const nn = @as(u32, @intCast(self.buf[2])) | @as(u32, @intCast(self.buf[3])) << 8;
767 if (@as(u16, @intCast(nn)) != @as(u16, @truncate(~n))) {
768 corrupt_input_error_offset = self.roffset;
769 self.err = InflateError.CorruptInput;
770 return InflateError.CorruptInput;
771 }
772
773 if (n == 0) {
774 self.to_read = self.dict.readFlush();
775 self.finishBlock();
776 return;
777 }
778
779 self.copy_len = n;
780 try self.copyData();
781 }
782
783 // copyData copies self.copy_len bytes from the underlying reader into self.hist.
784 // It pauses for reads when self.hist is full.
785 fn copyData(self: *Self) Error!void {
786 var buf = self.dict.writeSlice();
787 if (buf.len > self.copy_len) {
788 buf = buf[0..self.copy_len];
789 }
790
791 const cnt = try self.inner_reader.read(buf);
792 if (cnt < buf.len) {
793 self.err = InflateError.UnexpectedEndOfStream;
794 }
795 self.roffset += @as(u64, @intCast(cnt));
796 self.copy_len -= @as(u32, @intCast(cnt));
797 self.dict.writeMark(@as(u32, @intCast(cnt)));
798 if (self.err != null) {
799 return InflateError.UnexpectedEndOfStream;
800 }
801
802 if (self.dict.availWrite() == 0 or self.copy_len > 0) {
803 self.to_read = self.dict.readFlush();
804 self.step = copyData;
805 return;
806 }
807 self.finishBlock();
808 }
809
810 fn finishBlock(self: *Self) void {
811 if (self.final) {
812 if (self.dict.availRead() > 0) {
813 self.to_read = self.dict.readFlush();
814 }
815 self.err = InflateError.EndOfStreamWithNoError;
816 }
817 self.step = nextBlock;
818 }
819
820 fn moreBits(self: *Self) InflateError!void {
821 const c = self.inner_reader.readByte() catch |e| {
822 if (e == error.EndOfStream) {
823 return InflateError.UnexpectedEndOfStream;
824 }
825 return InflateError.BadReaderState;
826 };
827 self.roffset += 1;
828 self.b |= @as(u32, c) << @as(u5, @intCast(self.nb));
829 self.nb += 8;
830 return;
831 }
832
833 // Read the next Huffman-encoded symbol according to h.
834 fn huffSym(self: *Self, h: *HuffmanDecoder) InflateError!u32 {
835 // Since a HuffmanDecoder can be empty or be composed of a degenerate tree
836 // with single element, huffSym must error on these two edge cases. In both
837 // cases, the chunks slice will be 0 for the invalid sequence, leading it
838 // satisfy the n == 0 check below.
839 var n: u32 = h.min;
840 // Optimization. Go compiler isn't smart enough to keep self.b, self.nb in registers,
841 // but is smart enough to keep local variables in registers, so use nb and b,
842 // inline call to moreBits and reassign b, nb back to self on return.
843 var nb = self.nb;
844 var b = self.b;
845 while (true) {
846 while (nb < n) {
847 const c = self.inner_reader.readByte() catch |e| {
848 self.b = b;
849 self.nb = nb;
850 if (e == error.EndOfStream) {
851 return error.UnexpectedEndOfStream;
852 }
853 return InflateError.BadReaderState;
854 };
855 self.roffset += 1;
856 b |= @as(u32, @intCast(c)) << @as(u5, @intCast(nb & 31));
857 nb += 8;
858 }
859 var chunk = h.chunks[b & (huffman_num_chunks - 1)];
860 n = @as(u32, @intCast(chunk & huffman_count_mask));
861 if (n > huffman_chunk_bits) {
862 chunk = h.links[chunk >> huffman_value_shift][(b >> huffman_chunk_bits) & h.link_mask];
863 n = @as(u32, @intCast(chunk & huffman_count_mask));
864 }
865 if (n <= nb) {
866 if (n == 0) {
867 self.b = b;
868 self.nb = nb;
869 corrupt_input_error_offset = self.roffset;
870 self.err = InflateError.CorruptInput;
871 return InflateError.CorruptInput;
872 }
873 self.b = b >> @as(u5, @intCast(n & 31));
874 self.nb = nb - n;
875 return @as(u32, @intCast(chunk >> huffman_value_shift));
876 }
877 }
878 }
879
880 /// Replaces the inner reader and dictionary with new_reader and new_dict.
881 /// new_reader must be of the same type as the reader being replaced.
882 pub fn reset(s: *Self, new_reader: ReaderType, new_dict: ?[]const u8) Error!void {
883 s.inner_reader = new_reader;
884 s.step = nextBlock;
885 s.err = null;
886 s.nb = 0;
887
888 s.dict.deinit();
889 try s.dict.init(s.allocator, max_match_offset, new_dict);
890
891 return;
892 }
893 };
894}
895
896// tests
897const expectError = std.testing.expectError;
898const io = std.io;
899const testing = std.testing;
900
901test "confirm decompressor resets" {
902 var compressed = std.ArrayList(u8).init(std.testing.allocator);
903 defer compressed.deinit();
904
905 inline for (.{
906 &[_]u8{ 0x5d, 0xc0, 0x21, 0x01, 0x00, 0x00, 0x00, 0x80, 0x20, 0xff, 0xaf, 0xa6, 0x4b, 0x03 },
907 &[_]u8{ 0x55, 0xc1, 0x41, 0x0d, 0x00, 0x00, 0x00, 0x02, 0xa1, 0x94, 0x96, 0x34, 0x25, 0xef, 0x1b, 0x5f, 0x01 },
908 }) |data| {
909 try compressed.writer().writeAll(data);
910 }
911
912 var stream = std.io.fixedBufferStream(compressed.items);
913 var decomp = try decompressor(std.testing.allocator, stream.reader(), null);
914 defer decomp.deinit();
915
916 while (true) {
917 if (try stream.getPos() == try stream.getEndPos()) break;
918
919 const buf = try decomp.reader().readAllAlloc(std.testing.allocator, 1024 * 100);
920 defer std.testing.allocator.free(buf);
921
922 try decomp.close();
923
924 try decomp.reset(stream.reader(), null);
925 }
926}
927
928test "truncated input" {
929 const TruncatedTest = struct {
930 input: []const u8,
931 output: []const u8,
932 };
933
934 const tests = [_]TruncatedTest{
935 .{ .input = "\x00", .output = "" },
936 .{ .input = "\x00\x0c", .output = "" },
937 .{ .input = "\x00\x0c\x00", .output = "" },
938 .{ .input = "\x00\x0c\x00\xf3\xff", .output = "" },
939 .{ .input = "\x00\x0c\x00\xf3\xffhello", .output = "hello" },
940 .{ .input = "\x00\x0c\x00\xf3\xffhello, world", .output = "hello, world" },
941 .{ .input = "\x02", .output = "" },
942 .{ .input = "\xf2H\xcd", .output = "He" },
943 .{ .input = "\xf2H͙0a\u{0084}\t", .output = "Hel\x90\x90\x90\x90\x90" },
944 .{ .input = "\xf2H͙0a\u{0084}\t\x00", .output = "Hel\x90\x90\x90\x90\x90" },
945 };
946
947 for (tests) |t| {
948 var fib = io.fixedBufferStream(t.input);
949 const r = fib.reader();
950 var z = try decompressor(testing.allocator, r, null);
951 defer z.deinit();
952 var zr = z.reader();
953
954 var output = [1]u8{0} ** 12;
955 try expectError(error.UnexpectedEndOfStream, zr.readAll(&output));
956 try testing.expectEqualSlices(u8, t.output, output[0..t.output.len]);
957 }
958}
959
960test "Go non-regression test for 9842" {
961 // See https://golang.org/issue/9842
962
963 const Test = struct {
964 err: ?anyerror,
965 input: []const u8,
966 };
967
968 const tests = [_]Test{
969 .{ .err = error.UnexpectedEndOfStream, .input = ("\x95\x90=o\xc20\x10\x86\xf30") },
970 .{ .err = error.CorruptInput, .input = ("\x950\x00\x0000000") },
971
972 // Huffman.construct errors
973
974 // lencode
975 .{ .err = error.CorruptInput, .input = ("\x950000") },
976 .{ .err = error.CorruptInput, .input = ("\x05000") },
977 // hlen
978 .{ .err = error.CorruptInput, .input = ("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00") },
979 // hdist
980 .{ .err = error.CorruptInput, .input = ("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf.") },
981
982 // like the "empty distance alphabet" test but for ndist instead of nlen
983 .{ .err = error.CorruptInput, .input = ("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce") },
984 .{ .err = null, .input = "\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0" },
985 };
986
987 for (tests) |t| {
988 var fib = std.io.fixedBufferStream(t.input);
989 const reader = fib.reader();
990 var decomp = try decompressor(testing.allocator, reader, null);
991 defer decomp.deinit();
992
993 var output: [10]u8 = undefined;
994 if (t.err != null) {
995 try expectError(t.err.?, decomp.reader().read(&output));
996 } else {
997 _ = try decomp.reader().read(&output);
998 }
999 }
1000}
1001
1002test "inflate A Tale of Two Cities (1859) intro" {
1003 const compressed = [_]u8{
1004 0x74, 0xeb, 0xcd, 0x0d, 0x80, 0x20, 0x0c, 0x47, 0x71, 0xdc, 0x9d, 0xa2, 0x03, 0xb8, 0x88,
1005 0x63, 0xf0, 0xf1, 0x47, 0x9a, 0x00, 0x35, 0xb4, 0x86, 0xf5, 0x0d, 0x27, 0x63, 0x82, 0xe7,
1006 0xdf, 0x7b, 0x87, 0xd1, 0x70, 0x4a, 0x96, 0x41, 0x1e, 0x6a, 0x24, 0x89, 0x8c, 0x2b, 0x74,
1007 0xdf, 0xf8, 0x95, 0x21, 0xfd, 0x8f, 0xdc, 0x89, 0x09, 0x83, 0x35, 0x4a, 0x5d, 0x49, 0x12,
1008 0x29, 0xac, 0xb9, 0x41, 0xbf, 0x23, 0x2e, 0x09, 0x79, 0x06, 0x1e, 0x85, 0x91, 0xd6, 0xc6,
1009 0x2d, 0x74, 0xc4, 0xfb, 0xa1, 0x7b, 0x0f, 0x52, 0x20, 0x84, 0x61, 0x28, 0x0c, 0x63, 0xdf,
1010 0x53, 0xf4, 0x00, 0x1e, 0xc3, 0xa5, 0x97, 0x88, 0xf4, 0xd9, 0x04, 0xa5, 0x2d, 0x49, 0x54,
1011 0xbc, 0xfd, 0x90, 0xa5, 0x0c, 0xae, 0xbf, 0x3f, 0x84, 0x77, 0x88, 0x3f, 0xaf, 0xc0, 0x40,
1012 0xd6, 0x5b, 0x14, 0x8b, 0x54, 0xf6, 0x0f, 0x9b, 0x49, 0xf7, 0xbf, 0xbf, 0x36, 0x54, 0x5a,
1013 0x0d, 0xe6, 0x3e, 0xf0, 0x9e, 0x29, 0xcd, 0xa1, 0x41, 0x05, 0x36, 0x48, 0x74, 0x4a, 0xe9,
1014 0x46, 0x66, 0x2a, 0x19, 0x17, 0xf4, 0x71, 0x8e, 0xcb, 0x15, 0x5b, 0x57, 0xe4, 0xf3, 0xc7,
1015 0xe7, 0x1e, 0x9d, 0x50, 0x08, 0xc3, 0x50, 0x18, 0xc6, 0x2a, 0x19, 0xa0, 0xdd, 0xc3, 0x35,
1016 0x82, 0x3d, 0x6a, 0xb0, 0x34, 0x92, 0x16, 0x8b, 0xdb, 0x1b, 0xeb, 0x7d, 0xbc, 0xf8, 0x16,
1017 0xf8, 0xc2, 0xe1, 0xaf, 0x81, 0x7e, 0x58, 0xf4, 0x9f, 0x74, 0xf8, 0xcd, 0x39, 0xd3, 0xaa,
1018 0x0f, 0x26, 0x31, 0xcc, 0x8d, 0x9a, 0xd2, 0x04, 0x3e, 0x51, 0xbe, 0x7e, 0xbc, 0xc5, 0x27,
1019 0x3d, 0xa5, 0xf3, 0x15, 0x63, 0x94, 0x42, 0x75, 0x53, 0x6b, 0x61, 0xc8, 0x01, 0x13, 0x4d,
1020 0x23, 0xba, 0x2a, 0x2d, 0x6c, 0x94, 0x65, 0xc7, 0x4b, 0x86, 0x9b, 0x25, 0x3e, 0xba, 0x01,
1021 0x10, 0x84, 0x81, 0x28, 0x80, 0x55, 0x1c, 0xc0, 0xa5, 0xaa, 0x36, 0xa6, 0x09, 0xa8, 0xa1,
1022 0x85, 0xf9, 0x7d, 0x45, 0xbf, 0x80, 0xe4, 0xd1, 0xbb, 0xde, 0xb9, 0x5e, 0xf1, 0x23, 0x89,
1023 0x4b, 0x00, 0xd5, 0x59, 0x84, 0x85, 0xe3, 0xd4, 0xdc, 0xb2, 0x66, 0xe9, 0xc1, 0x44, 0x0b,
1024 0x1e, 0x84, 0xec, 0xe6, 0xa1, 0xc7, 0x42, 0x6a, 0x09, 0x6d, 0x9a, 0x5e, 0x70, 0xa2, 0x36,
1025 0x94, 0x29, 0x2c, 0x85, 0x3f, 0x24, 0x39, 0xf3, 0xae, 0xc3, 0xca, 0xca, 0xaf, 0x2f, 0xce,
1026 0x8e, 0x58, 0x91, 0x00, 0x25, 0xb5, 0xb3, 0xe9, 0xd4, 0xda, 0xef, 0xfa, 0x48, 0x7b, 0x3b,
1027 0xe2, 0x63, 0x12, 0x00, 0x00, 0x20, 0x04, 0x80, 0x70, 0x36, 0x8c, 0xbd, 0x04, 0x71, 0xff,
1028 0xf6, 0x0f, 0x66, 0x38, 0xcf, 0xa1, 0x39, 0x11, 0x0f,
1029 };
1030
1031 const expected =
1032 \\It was the best of times,
1033 \\it was the worst of times,
1034 \\it was the age of wisdom,
1035 \\it was the age of foolishness,
1036 \\it was the epoch of belief,
1037 \\it was the epoch of incredulity,
1038 \\it was the season of Light,
1039 \\it was the season of Darkness,
1040 \\it was the spring of hope,
1041 \\it was the winter of despair,
1042 \\
1043 \\we had everything before us, we had nothing before us, we were all going direct to Heaven, we were all going direct the other way---in short, the period was so far like the present period, that some of its noisiest authorities insisted on its being received, for good or for evil, in the superlative degree of comparison only.
1044 \\
1045 ;
1046
1047 var fib = std.io.fixedBufferStream(&compressed);
1048 const reader = fib.reader();
1049 var decomp = try decompressor(testing.allocator, reader, null);
1050 defer decomp.deinit();
1051
1052 var got: [700]u8 = undefined;
1053 const got_len = try decomp.reader().read(&got);
1054 try testing.expectEqual(@as(usize, 616), got_len);
1055 try testing.expectEqualSlices(u8, expected, got[0..expected.len]);
1056}
1057
1058test "lengths overflow" {
1059 // malformed final dynamic block, tries to write 321 code lengths (MAXCODES is 316)
1060 // f dy hlit hdist hclen 16 17 18 0 (18) x138 (18) x138 (18) x39 (16) x6
1061 // 1 10 11101 11101 0000 010 010 010 010 (11) 1111111 (11) 1111111 (11) 0011100 (01) 11
1062 const stream = [_]u8{
1063 0b11101101, 0b00011101, 0b00100100, 0b11101001, 0b11111111, 0b11111111, 0b00111001,
1064 0b00001110,
1065 };
1066 try expectError(error.CorruptInput, decompress(stream[0..]));
1067}
1068
1069test "empty distance alphabet" {
1070 // dynamic block with empty distance alphabet is valid if only literals and end of data symbol are used
1071 // f dy hlit hdist hclen 16 17 18 0 8 7 9 6 10 5 11 4 12 3 13 2 14 1 15 (18) x128 (18) x128 (1) ( 0) (256)
1072 // 1 10 00000 00000 1111 000 000 010 010 000 000 000 000 000 000 000 000 000 000 000 000 000 001 000 (11) 1110101 (11) 1110101 (0) (10) (0)
1073 const stream = [_]u8{
1074 0b00000101, 0b11100000, 0b00000001, 0b00001001, 0b00000000, 0b00000000,
1075 0b00000000, 0b00000000, 0b00010000, 0b01011100, 0b10111111, 0b00101110,
1076 };
1077 try decompress(stream[0..]);
1078}
1079
1080test "distance past beginning of output stream" {
1081 // f fx ('A') ('B') ('C') <len=4, dist=4> (end)
1082 // 1 01 (01110001) (01110010) (01110011) (0000010) (00011) (0000000)
1083 const stream = [_]u8{ 0b01110011, 0b01110100, 0b01110010, 0b00000110, 0b01100001, 0b00000000 };
1084 try std.testing.expectError(error.CorruptInput, decompress(stream[0..]));
1085}
1086
1087test "fuzzing" {
1088 const compressed = [_]u8{
1089 0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05, 0xfc, 0x30, 0x0b, 0x0a, 0x08, 0x50, 0xeb, 0x25, 0x05,
1090 } ++ [_]u8{0xe1} ** 15 ++ [_]u8{0x30} ++ [_]u8{0xe1} ** 1481;
1091 try expectError(error.UnexpectedEndOfStream, decompress(&compressed));
1092
1093 // see https://github.com/ziglang/zig/issues/9842
1094 try expectError(error.UnexpectedEndOfStream, decompress("\x95\x90=o\xc20\x10\x86\xf30"));
1095 try expectError(error.CorruptInput, decompress("\x950\x00\x0000000"));
1096
1097 // Huffman errors
1098 // lencode
1099 try expectError(error.CorruptInput, decompress("\x950000"));
1100 try expectError(error.CorruptInput, decompress("\x05000"));
1101 // hlen
1102 try expectError(error.CorruptInput, decompress("\x05\xea\x01\t\x00\x00\x00\x01\x00\\\xbf.\t\x00"));
1103 // hdist
1104 try expectError(error.CorruptInput, decompress("\x05\xe0\x01A\x00\x00\x00\x00\x10\\\xbf."));
1105
1106 // like the "empty distance alphabet" test but for ndist instead of nlen
1107 try expectError(error.CorruptInput, decompress("\x05\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf\xce"));
1108 try decompress("\x15\xe0\x01\t\x00\x00\x00\x00\x10\\\xbf.0");
1109}
1110
1111fn decompress(input: []const u8) !void {
1112 const allocator = testing.allocator;
1113 var fib = std.io.fixedBufferStream(input);
1114 const reader = fib.reader();
1115 var decomp = try decompressor(allocator, reader, null);
1116 defer decomp.deinit();
1117 const output = try decomp.reader().readAllAlloc(allocator, math.maxInt(usize));
1118 defer std.testing.allocator.free(output);
1119}
lib/std/compress/deflate/deflate_const.zig deleted-28
......@@ -1,28 +0,0 @@
1// Deflate
2
3// Biggest block size for uncompressed block.
4pub const max_store_block_size = 65535;
5// The special code used to mark the end of a block.
6pub const end_block_marker = 256;
7
8// LZ77
9
10// The smallest match length per the RFC section 3.2.5
11pub const base_match_length = 3;
12// The smallest match offset.
13pub const base_match_offset = 1;
14// The largest match length.
15pub const max_match_length = 258;
16// The largest match offset.
17pub const max_match_offset = 1 << 15;
18
19// Huffman Codes
20
21// The largest offset code.
22pub const offset_code_count = 30;
23// Max number of frequencies used for a Huffman Code
24// Possible lengths are codegenCodeCount (19), offset_code_count (30) and max_num_lit (286).
25// The largest of these is max_num_lit.
26pub const max_num_frequencies = max_num_lit;
27// Maximum number of literals.
28pub const max_num_lit = 286;
lib/std/compress/deflate/deflate_fast.zig deleted-728
......@@ -1,728 +0,0 @@
1// This encoding algorithm, which prioritizes speed over output size, is
2// based on Snappy's LZ77-style encoder: github.com/golang/snappy
3
4const std = @import("std");
5const math = std.math;
6const mem = std.mem;
7
8const Allocator = std.mem.Allocator;
9
10const deflate_const = @import("deflate_const.zig");
11const deflate = @import("compressor.zig");
12const token = @import("token.zig");
13
14const base_match_length = deflate_const.base_match_length;
15const base_match_offset = deflate_const.base_match_offset;
16const max_match_length = deflate_const.max_match_length;
17const max_match_offset = deflate_const.max_match_offset;
18const max_store_block_size = deflate_const.max_store_block_size;
19
20const table_bits = 14; // Bits used in the table.
21const table_mask = table_size - 1; // Mask for table indices. Redundant, but can eliminate bounds checks.
22const table_shift = 32 - table_bits; // Right-shift to get the table_bits most significant bits of a uint32.
23const table_size = 1 << table_bits; // Size of the table.
24
25// Reset the buffer offset when reaching this.
26// Offsets are stored between blocks as i32 values.
27// Since the offset we are checking against is at the beginning
28// of the buffer, we need to subtract the current and input
29// buffer to not risk overflowing the i32.
30const buffer_reset = math.maxInt(i32) - max_store_block_size * 2;
31
32fn load32(b: []u8, i: i32) u32 {
33 const s = b[@as(usize, @intCast(i)) .. @as(usize, @intCast(i)) + 4];
34 return @as(u32, @intCast(s[0])) |
35 @as(u32, @intCast(s[1])) << 8 |
36 @as(u32, @intCast(s[2])) << 16 |
37 @as(u32, @intCast(s[3])) << 24;
38}
39
40fn load64(b: []u8, i: i32) u64 {
41 const s = b[@as(usize, @intCast(i))..@as(usize, @intCast(i + 8))];
42 return @as(u64, @intCast(s[0])) |
43 @as(u64, @intCast(s[1])) << 8 |
44 @as(u64, @intCast(s[2])) << 16 |
45 @as(u64, @intCast(s[3])) << 24 |
46 @as(u64, @intCast(s[4])) << 32 |
47 @as(u64, @intCast(s[5])) << 40 |
48 @as(u64, @intCast(s[6])) << 48 |
49 @as(u64, @intCast(s[7])) << 56;
50}
51
52fn hash(u: u32) u32 {
53 return (u *% 0x1e35a7bd) >> table_shift;
54}
55
56// These constants are defined by the Snappy implementation so that its
57// assembly implementation can fast-path some 16-bytes-at-a-time copies.
58// They aren't necessary in the pure Go implementation, and may not be
59// necessary in Zig, but using the same thresholds doesn't really hurt.
60const input_margin = 16 - 1;
61const min_non_literal_block_size = 1 + 1 + input_margin;
62
63const TableEntry = struct {
64 val: u32, // Value at destination
65 offset: i32,
66};
67
68pub fn deflateFast() DeflateFast {
69 return DeflateFast{
70 .table = [_]TableEntry{.{ .val = 0, .offset = 0 }} ** table_size,
71 .prev = undefined,
72 .prev_len = 0,
73 .cur = max_store_block_size,
74 .allocator = undefined,
75 };
76}
77
78// DeflateFast maintains the table for matches,
79// and the previous byte block for cross block matching.
80pub const DeflateFast = struct {
81 table: [table_size]TableEntry,
82 prev: []u8, // Previous block, zero length if unknown.
83 prev_len: u32, // Previous block length
84 cur: i32, // Current match offset.
85 allocator: Allocator,
86
87 const Self = @This();
88
89 pub fn init(self: *Self, allocator: Allocator) !void {
90 self.allocator = allocator;
91 self.prev = try allocator.alloc(u8, max_store_block_size);
92 self.prev_len = 0;
93 }
94
95 pub fn deinit(self: *Self) void {
96 self.allocator.free(self.prev);
97 self.prev_len = 0;
98 }
99
100 // Encodes a block given in `src` and appends tokens to `dst` and returns the result.
101 pub fn encode(self: *Self, dst: []token.Token, tokens_count: *u16, src: []u8) void {
102
103 // Ensure that self.cur doesn't wrap.
104 if (self.cur >= buffer_reset) {
105 self.shiftOffsets();
106 }
107
108 // This check isn't in the Snappy implementation, but there, the caller
109 // instead of the callee handles this case.
110 if (src.len < min_non_literal_block_size) {
111 self.cur += max_store_block_size;
112 self.prev_len = 0;
113 emitLiteral(dst, tokens_count, src);
114 return;
115 }
116
117 // s_limit is when to stop looking for offset/length copies. The input_margin
118 // lets us use a fast path for emitLiteral in the main loop, while we are
119 // looking for copies.
120 const s_limit = @as(i32, @intCast(src.len - input_margin));
121
122 // next_emit is where in src the next emitLiteral should start from.
123 var next_emit: i32 = 0;
124 var s: i32 = 0;
125 var cv: u32 = load32(src, s);
126 var next_hash: u32 = hash(cv);
127
128 outer: while (true) {
129 // Copied from the C++ snappy implementation:
130 //
131 // Heuristic match skipping: If 32 bytes are scanned with no matches
132 // found, start looking only at every other byte. If 32 more bytes are
133 // scanned (or skipped), look at every third byte, etc.. When a match
134 // is found, immediately go back to looking at every byte. This is a
135 // small loss (~5% performance, ~0.1% density) for compressible data
136 // due to more bookkeeping, but for non-compressible data (such as
137 // JPEG) it's a huge win since the compressor quickly "realizes" the
138 // data is incompressible and doesn't bother looking for matches
139 // everywhere.
140 //
141 // The "skip" variable keeps track of how many bytes there are since
142 // the last match; dividing it by 32 (ie. right-shifting by five) gives
143 // the number of bytes to move ahead for each iteration.
144 var skip: i32 = 32;
145
146 var next_s: i32 = s;
147 var candidate: TableEntry = undefined;
148 while (true) {
149 s = next_s;
150 const bytes_between_hash_lookups = skip >> 5;
151 next_s = s + bytes_between_hash_lookups;
152 skip += bytes_between_hash_lookups;
153 if (next_s > s_limit) {
154 break :outer;
155 }
156 candidate = self.table[next_hash & table_mask];
157 const now = load32(src, next_s);
158 self.table[next_hash & table_mask] = .{ .offset = s + self.cur, .val = cv };
159 next_hash = hash(now);
160
161 const offset = s - (candidate.offset - self.cur);
162 if (offset > max_match_offset or cv != candidate.val) {
163 // Out of range or not matched.
164 cv = now;
165 continue;
166 }
167 break;
168 }
169
170 // A 4-byte match has been found. We'll later see if more than 4 bytes
171 // match. But, prior to the match, src[next_emit..s] are unmatched. Emit
172 // them as literal bytes.
173 emitLiteral(dst, tokens_count, src[@as(usize, @intCast(next_emit))..@as(usize, @intCast(s))]);
174
175 // Call emitCopy, and then see if another emitCopy could be our next
176 // move. Repeat until we find no match for the input immediately after
177 // what was consumed by the last emitCopy call.
178 //
179 // If we exit this loop normally then we need to call emitLiteral next,
180 // though we don't yet know how big the literal will be. We handle that
181 // by proceeding to the next iteration of the main loop. We also can
182 // exit this loop via goto if we get close to exhausting the input.
183 while (true) {
184 // Invariant: we have a 4-byte match at s, and no need to emit any
185 // literal bytes prior to s.
186
187 // Extend the 4-byte match as long as possible.
188 //
189 s += 4;
190 const t = candidate.offset - self.cur + 4;
191 const l = self.matchLen(s, t, src);
192
193 // matchToken is flate's equivalent of Snappy's emitCopy. (length,offset)
194 dst[tokens_count.*] = token.matchToken(
195 @as(u32, @intCast(l + 4 - base_match_length)),
196 @as(u32, @intCast(s - t - base_match_offset)),
197 );
198 tokens_count.* += 1;
199 s += l;
200 next_emit = s;
201 if (s >= s_limit) {
202 break :outer;
203 }
204
205 // We could immediately start working at s now, but to improve
206 // compression we first update the hash table at s-1 and at s. If
207 // another emitCopy is not our next move, also calculate next_hash
208 // at s+1. At least on amd64 architecture, these three hash calculations
209 // are faster as one load64 call (with some shifts) instead of
210 // three load32 calls.
211 var x = load64(src, s - 1);
212 const prev_hash = hash(@as(u32, @truncate(x)));
213 self.table[prev_hash & table_mask] = TableEntry{
214 .offset = self.cur + s - 1,
215 .val = @as(u32, @truncate(x)),
216 };
217 x >>= 8;
218 const curr_hash = hash(@as(u32, @truncate(x)));
219 candidate = self.table[curr_hash & table_mask];
220 self.table[curr_hash & table_mask] = TableEntry{
221 .offset = self.cur + s,
222 .val = @as(u32, @truncate(x)),
223 };
224
225 const offset = s - (candidate.offset - self.cur);
226 if (offset > max_match_offset or @as(u32, @truncate(x)) != candidate.val) {
227 cv = @as(u32, @truncate(x >> 8));
228 next_hash = hash(cv);
229 s += 1;
230 break;
231 }
232 }
233 }
234
235 if (@as(u32, @intCast(next_emit)) < src.len) {
236 emitLiteral(dst, tokens_count, src[@as(usize, @intCast(next_emit))..]);
237 }
238 self.cur += @as(i32, @intCast(src.len));
239 self.prev_len = @as(u32, @intCast(src.len));
240 @memcpy(self.prev[0..self.prev_len], src);
241 return;
242 }
243
244 fn emitLiteral(dst: []token.Token, tokens_count: *u16, lit: []u8) void {
245 for (lit) |v| {
246 dst[tokens_count.*] = token.literalToken(@as(u32, @intCast(v)));
247 tokens_count.* += 1;
248 }
249 return;
250 }
251
252 // matchLen returns the match length between src[s..] and src[t..].
253 // t can be negative to indicate the match is starting in self.prev.
254 // We assume that src[s-4 .. s] and src[t-4 .. t] already match.
255 fn matchLen(self: *Self, s: i32, t: i32, src: []u8) i32 {
256 var s1 = @as(u32, @intCast(s)) + max_match_length - 4;
257 if (s1 > src.len) {
258 s1 = @as(u32, @intCast(src.len));
259 }
260
261 // If we are inside the current block
262 if (t >= 0) {
263 var b = src[@as(usize, @intCast(t))..];
264 const a = src[@as(usize, @intCast(s))..@as(usize, @intCast(s1))];
265 b = b[0..a.len];
266 // Extend the match to be as long as possible.
267 for (a, 0..) |_, i| {
268 if (a[i] != b[i]) {
269 return @as(i32, @intCast(i));
270 }
271 }
272 return @as(i32, @intCast(a.len));
273 }
274
275 // We found a match in the previous block.
276 const tp = @as(i32, @intCast(self.prev_len)) + t;
277 if (tp < 0) {
278 return 0;
279 }
280
281 // Extend the match to be as long as possible.
282 var a = src[@as(usize, @intCast(s))..@as(usize, @intCast(s1))];
283 var b = self.prev[@as(usize, @intCast(tp))..@as(usize, @intCast(self.prev_len))];
284 if (b.len > a.len) {
285 b = b[0..a.len];
286 }
287 a = a[0..b.len];
288 for (b, 0..) |_, i| {
289 if (a[i] != b[i]) {
290 return @as(i32, @intCast(i));
291 }
292 }
293
294 // If we reached our limit, we matched everything we are
295 // allowed to in the previous block and we return.
296 const n = @as(i32, @intCast(b.len));
297 if (@as(u32, @intCast(s + n)) == s1) {
298 return n;
299 }
300
301 // Continue looking for more matches in the current block.
302 a = src[@as(usize, @intCast(s + n))..@as(usize, @intCast(s1))];
303 b = src[0..a.len];
304 for (a, 0..) |_, i| {
305 if (a[i] != b[i]) {
306 return @as(i32, @intCast(i)) + n;
307 }
308 }
309 return @as(i32, @intCast(a.len)) + n;
310 }
311
312 // Reset resets the encoding history.
313 // This ensures that no matches are made to the previous block.
314 pub fn reset(self: *Self) void {
315 self.prev_len = 0;
316 // Bump the offset, so all matches will fail distance check.
317 // Nothing should be >= self.cur in the table.
318 self.cur += max_match_offset;
319
320 // Protect against self.cur wraparound.
321 if (self.cur >= buffer_reset) {
322 self.shiftOffsets();
323 }
324 }
325
326 // shiftOffsets will shift down all match offset.
327 // This is only called in rare situations to prevent integer overflow.
328 //
329 // See https://golang.org/issue/18636 and https://golang.org/issues/34121.
330 fn shiftOffsets(self: *Self) void {
331 if (self.prev_len == 0) {
332 // We have no history; just clear the table.
333 for (self.table, 0..) |_, i| {
334 self.table[i] = TableEntry{ .val = 0, .offset = 0 };
335 }
336 self.cur = max_match_offset + 1;
337 return;
338 }
339
340 // Shift down everything in the table that isn't already too far away.
341 for (self.table, 0..) |_, i| {
342 var v = self.table[i].offset - self.cur + max_match_offset + 1;
343 if (v < 0) {
344 // We want to reset self.cur to max_match_offset + 1, so we need to shift
345 // all table entries down by (self.cur - (max_match_offset + 1)).
346 // Because we ignore matches > max_match_offset, we can cap
347 // any negative offsets at 0.
348 v = 0;
349 }
350 self.table[i].offset = v;
351 }
352 self.cur = max_match_offset + 1;
353 }
354};
355
356test "best speed match 1/3" {
357 if (@import("builtin").os.tag == .wasi) {
358 // https://github.com/ziglang/zig/issues/18885
359 return error.SkipZigTest;
360 }
361 const expectEqual = std.testing.expectEqual;
362
363 {
364 var previous = [_]u8{ 0, 0, 0, 1, 2 };
365 var e = DeflateFast{
366 .prev = &previous,
367 .prev_len = previous.len,
368 .table = undefined,
369 .allocator = undefined,
370 .cur = 0,
371 };
372 var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 };
373 const got: i32 = e.matchLen(3, -3, &current);
374 try expectEqual(@as(i32, 6), got);
375 }
376 {
377 var previous = [_]u8{ 0, 0, 0, 1, 2 };
378 var e = DeflateFast{
379 .prev = &previous,
380 .prev_len = previous.len,
381 .table = undefined,
382 .allocator = undefined,
383 .cur = 0,
384 };
385 var current = [_]u8{ 2, 4, 5, 0, 1, 2, 3, 4, 5 };
386 const got: i32 = e.matchLen(3, -3, &current);
387 try expectEqual(@as(i32, 3), got);
388 }
389 {
390 var previous = [_]u8{ 0, 0, 0, 1, 1 };
391 var e = DeflateFast{
392 .prev = &previous,
393 .prev_len = previous.len,
394 .table = undefined,
395 .allocator = undefined,
396 .cur = 0,
397 };
398 var current = [_]u8{ 3, 4, 5, 0, 1, 2, 3, 4, 5 };
399 const got: i32 = e.matchLen(3, -3, &current);
400 try expectEqual(@as(i32, 2), got);
401 }
402 {
403 var previous = [_]u8{ 0, 0, 0, 1, 2 };
404 var e = DeflateFast{
405 .prev = &previous,
406 .prev_len = previous.len,
407 .table = undefined,
408 .allocator = undefined,
409 .cur = 0,
410 };
411 var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
412 const got: i32 = e.matchLen(0, -1, &current);
413 try expectEqual(@as(i32, 4), got);
414 }
415 {
416 var previous = [_]u8{ 0, 0, 0, 1, 2, 3, 4, 5, 2, 2 };
417 var e = DeflateFast{
418 .prev = &previous,
419 .prev_len = previous.len,
420 .table = undefined,
421 .allocator = undefined,
422 .cur = 0,
423 };
424 var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
425 const got: i32 = e.matchLen(4, -7, &current);
426 try expectEqual(@as(i32, 5), got);
427 }
428 {
429 var previous = [_]u8{ 9, 9, 9, 9, 9 };
430 var e = DeflateFast{
431 .prev = &previous,
432 .prev_len = previous.len,
433 .table = undefined,
434 .allocator = undefined,
435 .cur = 0,
436 };
437 var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
438 const got: i32 = e.matchLen(0, -1, &current);
439 try expectEqual(@as(i32, 0), got);
440 }
441 {
442 var previous = [_]u8{ 9, 9, 9, 9, 9 };
443 var e = DeflateFast{
444 .prev = &previous,
445 .prev_len = previous.len,
446 .table = undefined,
447 .allocator = undefined,
448 .cur = 0,
449 };
450 var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
451 const got: i32 = e.matchLen(1, 0, &current);
452 try expectEqual(@as(i32, 0), got);
453 }
454}
455
456test "best speed match 2/3" {
457 if (@import("builtin").os.tag == .wasi) {
458 // https://github.com/ziglang/zig/issues/18885
459 return error.SkipZigTest;
460 }
461 const expectEqual = std.testing.expectEqual;
462
463 {
464 var previous = [_]u8{};
465 var e = DeflateFast{
466 .prev = &previous,
467 .prev_len = previous.len,
468 .table = undefined,
469 .allocator = undefined,
470 .cur = 0,
471 };
472 var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
473 const got: i32 = e.matchLen(1, -5, &current);
474 try expectEqual(@as(i32, 0), got);
475 }
476 {
477 var previous = [_]u8{};
478 var e = DeflateFast{
479 .prev = &previous,
480 .prev_len = previous.len,
481 .table = undefined,
482 .allocator = undefined,
483 .cur = 0,
484 };
485 var current = [_]u8{ 9, 2, 2, 2, 1, 2, 3, 4, 5 };
486 const got: i32 = e.matchLen(1, -1, &current);
487 try expectEqual(@as(i32, 0), got);
488 }
489 {
490 var previous = [_]u8{};
491 var e = DeflateFast{
492 .prev = &previous,
493 .prev_len = previous.len,
494 .table = undefined,
495 .allocator = undefined,
496 .cur = 0,
497 };
498 var current = [_]u8{ 2, 2, 2, 2, 1, 2, 3, 4, 5 };
499 const got: i32 = e.matchLen(1, 0, &current);
500 try expectEqual(@as(i32, 3), got);
501 }
502 {
503 var previous = [_]u8{ 3, 4, 5 };
504 var e = DeflateFast{
505 .prev = &previous,
506 .prev_len = previous.len,
507 .table = undefined,
508 .allocator = undefined,
509 .cur = 0,
510 };
511 var current = [_]u8{ 3, 4, 5 };
512 const got: i32 = e.matchLen(0, -3, &current);
513 try expectEqual(@as(i32, 3), got);
514 }
515}
516
517test "best speed match 2/2" {
518 const testing = std.testing;
519 const expectEqual = testing.expectEqual;
520
521 const Case = struct {
522 previous: u32,
523 current: u32,
524 s: i32,
525 t: i32,
526 expected: i32,
527 };
528
529 const cases = [_]Case{
530 .{
531 .previous = 1000,
532 .current = 1000,
533 .s = 0,
534 .t = -1000,
535 .expected = max_match_length - 4,
536 },
537 .{
538 .previous = 200,
539 .s = 0,
540 .t = -200,
541 .current = 500,
542 .expected = max_match_length - 4,
543 },
544 .{
545 .previous = 200,
546 .s = 1,
547 .t = 0,
548 .current = 500,
549 .expected = max_match_length - 4,
550 },
551 .{
552 .previous = max_match_length - 4,
553 .s = 0,
554 .t = -(max_match_length - 4),
555 .current = 500,
556 .expected = max_match_length - 4,
557 },
558 .{
559 .previous = 200,
560 .s = 400,
561 .t = -200,
562 .current = 500,
563 .expected = 100,
564 },
565 .{
566 .previous = 10,
567 .s = 400,
568 .t = 200,
569 .current = 500,
570 .expected = 100,
571 },
572 };
573
574 for (cases) |c| {
575 const previous = try testing.allocator.alloc(u8, c.previous);
576 defer testing.allocator.free(previous);
577 @memset(previous, 0);
578
579 const current = try testing.allocator.alloc(u8, c.current);
580 defer testing.allocator.free(current);
581 @memset(current, 0);
582
583 var e = DeflateFast{
584 .prev = previous,
585 .prev_len = @as(u32, @intCast(previous.len)),
586 .table = undefined,
587 .allocator = undefined,
588 .cur = 0,
589 };
590 const got: i32 = e.matchLen(c.s, c.t, current);
591 try expectEqual(@as(i32, c.expected), got);
592 }
593}
594
595test "best speed shift offsets" {
596 const testing = std.testing;
597 const expect = std.testing.expect;
598
599 // Test if shiftoffsets properly preserves matches and resets out-of-range matches
600 // seen in https://github.com/golang/go/issues/4142
601 var enc = deflateFast();
602 try enc.init(testing.allocator);
603 defer enc.deinit();
604
605 // test_data may not generate internal matches.
606 var test_data = [32]u8{
607 0xf5, 0x25, 0xf2, 0x55, 0xf6, 0xc1, 0x1f, 0x0b, 0x10, 0xa1,
608 0xd0, 0x77, 0x56, 0x38, 0xf1, 0x9c, 0x7f, 0x85, 0xc5, 0xbd,
609 0x16, 0x28, 0xd4, 0xf9, 0x03, 0xd4, 0xc0, 0xa1, 0x1e, 0x58,
610 0x5b, 0xc9,
611 };
612
613 var tokens = [_]token.Token{0} ** 32;
614 var tokens_count: u16 = 0;
615
616 // Encode the testdata with clean state.
617 // Second part should pick up matches from the first block.
618 tokens_count = 0;
619 enc.encode(&tokens, &tokens_count, &test_data);
620 const want_first_tokens = tokens_count;
621 tokens_count = 0;
622 enc.encode(&tokens, &tokens_count, &test_data);
623 const want_second_tokens = tokens_count;
624
625 try expect(want_first_tokens > want_second_tokens);
626
627 // Forward the current indicator to before wraparound.
628 enc.cur = buffer_reset - @as(i32, @intCast(test_data.len));
629
630 // Part 1 before wrap, should match clean state.
631 tokens_count = 0;
632 enc.encode(&tokens, &tokens_count, &test_data);
633 var got = tokens_count;
634 try testing.expectEqual(want_first_tokens, got);
635
636 // Verify we are about to wrap.
637 try testing.expectEqual(@as(i32, buffer_reset), enc.cur);
638
639 // Part 2 should match clean state as well even if wrapped.
640 tokens_count = 0;
641 enc.encode(&tokens, &tokens_count, &test_data);
642 got = tokens_count;
643 try testing.expectEqual(want_second_tokens, got);
644
645 // Verify that we wrapped.
646 try expect(enc.cur < buffer_reset);
647
648 // Forward the current buffer, leaving the matches at the bottom.
649 enc.cur = buffer_reset;
650 enc.shiftOffsets();
651
652 // Ensure that no matches were picked up.
653 tokens_count = 0;
654 enc.encode(&tokens, &tokens_count, &test_data);
655 got = tokens_count;
656 try testing.expectEqual(want_first_tokens, got);
657}
658
659test "best speed reset" {
660 // test that encoding is consistent across a warparound of the table offset.
661 // See https://github.com/golang/go/issues/34121
662 const fmt = std.fmt;
663 const testing = std.testing;
664
665 const ArrayList = std.ArrayList;
666
667 const input_size = 65536;
668 const input = try testing.allocator.alloc(u8, input_size);
669 defer testing.allocator.free(input);
670
671 var i: usize = 0;
672 while (i < input_size) : (i += 1) {
673 _ = try fmt.bufPrint(input, "asdfasdfasdfasdf{d}{d}fghfgujyut{d}yutyu\n", .{ i, i, i });
674 }
675 // This is specific to level 1 (best_speed).
676 const level = .best_speed;
677 const offset: usize = 1;
678
679 // We do an encode with a clean buffer to compare.
680 var want = ArrayList(u8).init(testing.allocator);
681 defer want.deinit();
682 var clean_comp = try deflate.compressor(
683 testing.allocator,
684 want.writer(),
685 .{ .level = level },
686 );
687 defer clean_comp.deinit();
688
689 // Write 3 times, close.
690 try clean_comp.writer().writeAll(input);
691 try clean_comp.writer().writeAll(input);
692 try clean_comp.writer().writeAll(input);
693 try clean_comp.close();
694
695 var o = offset;
696 while (o <= 256) : (o *= 2) {
697 var discard = ArrayList(u8).init(testing.allocator);
698 defer discard.deinit();
699
700 var comp = try deflate.compressor(
701 testing.allocator,
702 discard.writer(),
703 .{ .level = level },
704 );
705 defer comp.deinit();
706
707 // Reset until we are right before the wraparound.
708 // Each reset adds max_match_offset to the offset.
709 i = 0;
710 const limit = (buffer_reset - input.len - o - max_match_offset) / max_match_offset;
711 while (i < limit) : (i += 1) {
712 // skip ahead to where we are close to wrap around...
713 comp.reset(discard.writer());
714 }
715 var got = ArrayList(u8).init(testing.allocator);
716 defer got.deinit();
717 comp.reset(got.writer());
718
719 // Write 3 times, close.
720 try comp.writer().writeAll(input);
721 try comp.writer().writeAll(input);
722 try comp.writer().writeAll(input);
723 try comp.close();
724
725 // output must match at wraparound
726 try testing.expectEqualSlices(u8, want.items, got.items);
727 }
728}
lib/std/compress/deflate/deflate_fast_test.zig deleted-160
......@@ -1,160 +0,0 @@
1const std = @import("std");
2const expect = std.testing.expect;
3const io = std.io;
4const mem = std.mem;
5const testing = std.testing;
6
7const ArrayList = std.ArrayList;
8
9const deflate = @import("compressor.zig");
10const inflate = @import("decompressor.zig");
11const deflate_const = @import("deflate_const.zig");
12
13test "best speed" {
14 // Tests that round-tripping through deflate and then inflate recovers the original input.
15 // The Write sizes are near the thresholds in the compressor.encSpeed method (0, 16, 128), as well
16 // as near `deflate_const.max_store_block_size` (65535).
17
18 var abcabc = try testing.allocator.alloc(u8, 131_072);
19 defer testing.allocator.free(abcabc);
20
21 for (abcabc, 0..) |_, i| {
22 abcabc[i] = @as(u8, @intCast(i % 128));
23 }
24
25 var tc_01 = [_]u32{ 65536, 0 };
26 var tc_02 = [_]u32{ 65536, 1 };
27 var tc_03 = [_]u32{ 65536, 1, 256 };
28 var tc_04 = [_]u32{ 65536, 1, 65536 };
29 var tc_05 = [_]u32{ 65536, 14 };
30 var tc_06 = [_]u32{ 65536, 15 };
31 var tc_07 = [_]u32{ 65536, 16 };
32 var tc_08 = [_]u32{ 65536, 16, 256 };
33 var tc_09 = [_]u32{ 65536, 16, 65536 };
34 var tc_10 = [_]u32{ 65536, 127 };
35 var tc_11 = [_]u32{ 65536, 127 };
36 var tc_12 = [_]u32{ 65536, 128 };
37 var tc_13 = [_]u32{ 65536, 128, 256 };
38 var tc_14 = [_]u32{ 65536, 128, 65536 };
39 var tc_15 = [_]u32{ 65536, 129 };
40 var tc_16 = [_]u32{ 65536, 65536, 256 };
41 var tc_17 = [_]u32{ 65536, 65536, 65536 };
42 const test_cases = [_][]u32{
43 &tc_01, &tc_02, &tc_03, &tc_04, &tc_05, &tc_06, &tc_07, &tc_08, &tc_09, &tc_10,
44 &tc_11, &tc_12, &tc_13, &tc_14, &tc_15, &tc_16, &tc_17,
45 };
46
47 for (test_cases) |tc| {
48 const firsts = [_]u32{ 1, 65534, 65535, 65536, 65537, 131072 };
49
50 for (firsts) |first_n| {
51 tc[0] = first_n;
52
53 const to_flush = [_]bool{ false, true };
54 for (to_flush) |flush| {
55 var compressed = ArrayList(u8).init(testing.allocator);
56 defer compressed.deinit();
57
58 var want = ArrayList(u8).init(testing.allocator);
59 defer want.deinit();
60
61 var comp = try deflate.compressor(
62 testing.allocator,
63 compressed.writer(),
64 .{ .level = .best_speed },
65 );
66 defer comp.deinit();
67
68 for (tc) |n| {
69 try want.appendSlice(abcabc[0..n]);
70 try comp.writer().writeAll(abcabc[0..n]);
71 if (flush) {
72 try comp.flush();
73 }
74 }
75
76 try comp.close();
77
78 const decompressed = try testing.allocator.alloc(u8, want.items.len);
79 defer testing.allocator.free(decompressed);
80
81 var fib = io.fixedBufferStream(compressed.items);
82 var decomp = try inflate.decompressor(testing.allocator, fib.reader(), null);
83 defer decomp.deinit();
84
85 const read = try decomp.reader().readAll(decompressed);
86 try decomp.close();
87
88 try testing.expectEqual(want.items.len, read);
89 try testing.expectEqualSlices(u8, want.items, decompressed);
90 }
91 }
92 }
93}
94
95test "best speed max match offset" {
96 const abc = "abcdefgh";
97 const xyz = "stuvwxyz";
98 const input_margin = 16 - 1;
99
100 const match_before = [_]bool{ false, true };
101 for (match_before) |do_match_before| {
102 const extras = [_]u32{
103 0,
104 input_margin - 1,
105 input_margin,
106 input_margin + 1,
107 2 * input_margin,
108 };
109 for (extras) |extra| {
110 var offset_adj: i32 = -5;
111 while (offset_adj <= 5) : (offset_adj += 1) {
112 const offset = deflate_const.max_match_offset + offset_adj;
113
114 // Make src to be a []u8 of the form
115 // fmt("{s}{s}{s}{s}{s}", .{abc, zeros0, xyzMaybe, abc, zeros1})
116 // where:
117 // zeros0 is approximately max_match_offset zeros.
118 // xyzMaybe is either xyz or the empty string.
119 // zeros1 is between 0 and 30 zeros.
120 // The difference between the two abc's will be offset, which
121 // is max_match_offset plus or minus a small adjustment.
122 const src_len: usize = @as(usize, @intCast(offset + @as(i32, abc.len) + @as(i32, @intCast(extra))));
123 var src = try testing.allocator.alloc(u8, src_len);
124 defer testing.allocator.free(src);
125
126 @memcpy(src[0..abc.len], abc);
127 if (!do_match_before) {
128 const src_offset: usize = @as(usize, @intCast(offset - @as(i32, xyz.len)));
129 @memcpy(src[src_offset..][0..xyz.len], xyz);
130 }
131 const src_offset: usize = @as(usize, @intCast(offset));
132 @memcpy(src[src_offset..][0..abc.len], abc);
133
134 var compressed = ArrayList(u8).init(testing.allocator);
135 defer compressed.deinit();
136
137 var comp = try deflate.compressor(
138 testing.allocator,
139 compressed.writer(),
140 .{ .level = .best_speed },
141 );
142 defer comp.deinit();
143 try comp.writer().writeAll(src);
144 _ = try comp.close();
145
146 const decompressed = try testing.allocator.alloc(u8, src.len);
147 defer testing.allocator.free(decompressed);
148
149 var fib = io.fixedBufferStream(compressed.items);
150 var decomp = try inflate.decompressor(testing.allocator, fib.reader(), null);
151 defer decomp.deinit();
152 const read = try decomp.reader().readAll(decompressed);
153 try decomp.close();
154
155 try testing.expectEqual(src.len, read);
156 try testing.expectEqualSlices(u8, src, decompressed);
157 }
158 }
159 }
160}
lib/std/compress/deflate/dict_decoder.zig deleted-423
......@@ -1,423 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const mem = std.mem;
4
5const Allocator = std.mem.Allocator;
6
7// Implements the LZ77 sliding dictionary as used in decompression.
8// LZ77 decompresses data through sequences of two forms of commands:
9//
10// * Literal insertions: Runs of one or more symbols are inserted into the data
11// stream as is. This is accomplished through the writeByte method for a
12// single symbol, or combinations of writeSlice/writeMark for multiple symbols.
13// Any valid stream must start with a literal insertion if no preset dictionary
14// is used.
15//
16// * Backward copies: Runs of one or more symbols are copied from previously
17// emitted data. Backward copies come as the tuple (dist, length) where dist
18// determines how far back in the stream to copy from and length determines how
19// many bytes to copy. Note that it is valid for the length to be greater than
20// the distance. Since LZ77 uses forward copies, that situation is used to
21// perform a form of run-length encoding on repeated runs of symbols.
22// The writeCopy and tryWriteCopy are used to implement this command.
23//
24// For performance reasons, this implementation performs little to no sanity
25// checks about the arguments. As such, the invariants documented for each
26// method call must be respected.
27pub const DictDecoder = struct {
28 const Self = @This();
29
30 allocator: Allocator = undefined,
31
32 hist: []u8 = undefined, // Sliding window history
33
34 // Invariant: 0 <= rd_pos <= wr_pos <= hist.len
35 wr_pos: u32 = 0, // Current output position in buffer
36 rd_pos: u32 = 0, // Have emitted hist[0..rd_pos] already
37 full: bool = false, // Has a full window length been written yet?
38
39 // init initializes DictDecoder to have a sliding window dictionary of the given
40 // size. If a preset dict is provided, it will initialize the dictionary with
41 // the contents of dict.
42 pub fn init(self: *Self, allocator: Allocator, size: u32, dict: ?[]const u8) !void {
43 self.allocator = allocator;
44
45 self.hist = try allocator.alloc(u8, size);
46
47 self.wr_pos = 0;
48
49 if (dict != null) {
50 const src = dict.?[dict.?.len -| self.hist.len..];
51 @memcpy(self.hist[0..src.len], src);
52 self.wr_pos = @as(u32, @intCast(dict.?.len));
53 }
54
55 if (self.wr_pos == self.hist.len) {
56 self.wr_pos = 0;
57 self.full = true;
58 }
59 self.rd_pos = self.wr_pos;
60 }
61
62 pub fn deinit(self: *Self) void {
63 self.allocator.free(self.hist);
64 }
65
66 // Reports the total amount of historical data in the dictionary.
67 pub fn histSize(self: *Self) u32 {
68 if (self.full) {
69 return @as(u32, @intCast(self.hist.len));
70 }
71 return self.wr_pos;
72 }
73
74 // Reports the number of bytes that can be flushed by readFlush.
75 pub fn availRead(self: *Self) u32 {
76 return self.wr_pos - self.rd_pos;
77 }
78
79 // Reports the available amount of output buffer space.
80 pub fn availWrite(self: *Self) u32 {
81 return @as(u32, @intCast(self.hist.len - self.wr_pos));
82 }
83
84 // Returns a slice of the available buffer to write data to.
85 //
86 // This invariant will be kept: s.len <= availWrite()
87 pub fn writeSlice(self: *Self) []u8 {
88 return self.hist[self.wr_pos..];
89 }
90
91 // Advances the writer pointer by `count`.
92 //
93 // This invariant must be kept: 0 <= count <= availWrite()
94 pub fn writeMark(self: *Self, count: u32) void {
95 assert(0 <= count and count <= self.availWrite());
96 self.wr_pos += count;
97 }
98
99 // Writes a single byte to the dictionary.
100 //
101 // This invariant must be kept: 0 < availWrite()
102 pub fn writeByte(self: *Self, byte: u8) void {
103 self.hist[self.wr_pos] = byte;
104 self.wr_pos += 1;
105 }
106
107 /// TODO: eliminate this function because the callsites should care about whether
108 /// or not their arguments alias and then they should directly call `@memcpy` or
109 /// `mem.copyForwards`.
110 fn copy(dst: []u8, src: []const u8) u32 {
111 if (src.len > dst.len) {
112 mem.copyForwards(u8, dst, src[0..dst.len]);
113 return @as(u32, @intCast(dst.len));
114 }
115 mem.copyForwards(u8, dst[0..src.len], src);
116 return @as(u32, @intCast(src.len));
117 }
118
119 // Copies a string at a given (dist, length) to the output.
120 // This returns the number of bytes copied and may be less than the requested
121 // length if the available space in the output buffer is too small.
122 //
123 // This invariant must be kept: 0 < dist <= histSize()
124 pub fn writeCopy(self: *Self, dist: u32, length: u32) u32 {
125 assert(0 < dist and dist <= self.histSize());
126 const dst_base = self.wr_pos;
127 var dst_pos = dst_base;
128 var src_pos: i32 = @as(i32, @intCast(dst_pos)) - @as(i32, @intCast(dist));
129 var end_pos = dst_pos + length;
130 if (end_pos > self.hist.len) {
131 end_pos = @as(u32, @intCast(self.hist.len));
132 }
133
134 // Copy non-overlapping section after destination position.
135 //
136 // This section is non-overlapping in that the copy length for this section
137 // is always less than or equal to the backwards distance. This can occur
138 // if a distance refers to data that wraps-around in the buffer.
139 // Thus, a backwards copy is performed here; that is, the exact bytes in
140 // the source prior to the copy is placed in the destination.
141 if (src_pos < 0) {
142 src_pos += @as(i32, @intCast(self.hist.len));
143 dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@as(usize, @intCast(src_pos))..]);
144 src_pos = 0;
145 }
146
147 // Copy possibly overlapping section before destination position.
148 //
149 // This section can overlap if the copy length for this section is larger
150 // than the backwards distance. This is allowed by LZ77 so that repeated
151 // strings can be succinctly represented using (dist, length) pairs.
152 // Thus, a forwards copy is performed here; that is, the bytes copied is
153 // possibly dependent on the resulting bytes in the destination as the copy
154 // progresses along. This is functionally equivalent to the following:
155 //
156 // var i = 0;
157 // while(i < end_pos - dst_pos) : (i+=1) {
158 // self.hist[dst_pos+i] = self.hist[src_pos+i];
159 // }
160 // dst_pos = end_pos;
161 //
162 while (dst_pos < end_pos) {
163 dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[@as(usize, @intCast(src_pos))..dst_pos]);
164 }
165
166 self.wr_pos = dst_pos;
167 return dst_pos - dst_base;
168 }
169
170 // Tries to copy a string at a given (distance, length) to the
171 // output. This specialized version is optimized for short distances.
172 //
173 // This method is designed to be inlined for performance reasons.
174 //
175 // This invariant must be kept: 0 < dist <= histSize()
176 pub fn tryWriteCopy(self: *Self, dist: u32, length: u32) u32 {
177 var dst_pos = self.wr_pos;
178 const end_pos = dst_pos + length;
179 if (dst_pos < dist or end_pos > self.hist.len) {
180 return 0;
181 }
182 const dst_base = dst_pos;
183 const src_pos = dst_pos - dist;
184
185 // Copy possibly overlapping section before destination position.
186 while (dst_pos < end_pos) {
187 dst_pos += copy(self.hist[dst_pos..end_pos], self.hist[src_pos..dst_pos]);
188 }
189
190 self.wr_pos = dst_pos;
191 return dst_pos - dst_base;
192 }
193
194 // Returns a slice of the historical buffer that is ready to be
195 // emitted to the user. The data returned by readFlush must be fully consumed
196 // before calling any other DictDecoder methods.
197 pub fn readFlush(self: *Self) []u8 {
198 const to_read = self.hist[self.rd_pos..self.wr_pos];
199 self.rd_pos = self.wr_pos;
200 if (self.wr_pos == self.hist.len) {
201 self.wr_pos = 0;
202 self.rd_pos = 0;
203 self.full = true;
204 }
205 return to_read;
206 }
207};
208
209// tests
210
211test "dictionary decoder" {
212 const ArrayList = std.ArrayList;
213 const testing = std.testing;
214
215 const abc = "ABC\n";
216 const fox = "The quick brown fox jumped over the lazy dog!\n";
217 const poem: []const u8 =
218 \\The Road Not Taken
219 \\Robert Frost
220 \\
221 \\Two roads diverged in a yellow wood,
222 \\And sorry I could not travel both
223 \\And be one traveler, long I stood
224 \\And looked down one as far as I could
225 \\To where it bent in the undergrowth;
226 \\
227 \\Then took the other, as just as fair,
228 \\And having perhaps the better claim,
229 \\Because it was grassy and wanted wear;
230 \\Though as for that the passing there
231 \\Had worn them really about the same,
232 \\
233 \\And both that morning equally lay
234 \\In leaves no step had trodden black.
235 \\Oh, I kept the first for another day!
236 \\Yet knowing how way leads on to way,
237 \\I doubted if I should ever come back.
238 \\
239 \\I shall be telling this with a sigh
240 \\Somewhere ages and ages hence:
241 \\Two roads diverged in a wood, and I-
242 \\I took the one less traveled by,
243 \\And that has made all the difference.
244 \\
245 ;
246
247 const uppercase: []const u8 =
248 \\THE ROAD NOT TAKEN
249 \\ROBERT FROST
250 \\
251 \\TWO ROADS DIVERGED IN A YELLOW WOOD,
252 \\AND SORRY I COULD NOT TRAVEL BOTH
253 \\AND BE ONE TRAVELER, LONG I STOOD
254 \\AND LOOKED DOWN ONE AS FAR AS I COULD
255 \\TO WHERE IT BENT IN THE UNDERGROWTH;
256 \\
257 \\THEN TOOK THE OTHER, AS JUST AS FAIR,
258 \\AND HAVING PERHAPS THE BETTER CLAIM,
259 \\BECAUSE IT WAS GRASSY AND WANTED WEAR;
260 \\THOUGH AS FOR THAT THE PASSING THERE
261 \\HAD WORN THEM REALLY ABOUT THE SAME,
262 \\
263 \\AND BOTH THAT MORNING EQUALLY LAY
264 \\IN LEAVES NO STEP HAD TRODDEN BLACK.
265 \\OH, I KEPT THE FIRST FOR ANOTHER DAY!
266 \\YET KNOWING HOW WAY LEADS ON TO WAY,
267 \\I DOUBTED IF I SHOULD EVER COME BACK.
268 \\
269 \\I SHALL BE TELLING THIS WITH A SIGH
270 \\SOMEWHERE AGES AND AGES HENCE:
271 \\TWO ROADS DIVERGED IN A WOOD, AND I-
272 \\I TOOK THE ONE LESS TRAVELED BY,
273 \\AND THAT HAS MADE ALL THE DIFFERENCE.
274 \\
275 ;
276
277 const PoemRefs = struct {
278 dist: u32, // Backward distance (0 if this is an insertion)
279 length: u32, // Length of copy or insertion
280 };
281
282 const poem_refs = [_]PoemRefs{
283 .{ .dist = 0, .length = 38 }, .{ .dist = 33, .length = 3 }, .{ .dist = 0, .length = 48 },
284 .{ .dist = 79, .length = 3 }, .{ .dist = 0, .length = 11 }, .{ .dist = 34, .length = 5 },
285 .{ .dist = 0, .length = 6 }, .{ .dist = 23, .length = 7 }, .{ .dist = 0, .length = 8 },
286 .{ .dist = 50, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 69, .length = 3 },
287 .{ .dist = 34, .length = 5 }, .{ .dist = 0, .length = 4 }, .{ .dist = 97, .length = 3 },
288 .{ .dist = 0, .length = 4 }, .{ .dist = 43, .length = 5 }, .{ .dist = 0, .length = 6 },
289 .{ .dist = 7, .length = 4 }, .{ .dist = 88, .length = 7 }, .{ .dist = 0, .length = 12 },
290 .{ .dist = 80, .length = 3 }, .{ .dist = 0, .length = 2 }, .{ .dist = 141, .length = 4 },
291 .{ .dist = 0, .length = 1 }, .{ .dist = 196, .length = 3 }, .{ .dist = 0, .length = 3 },
292 .{ .dist = 157, .length = 3 }, .{ .dist = 0, .length = 6 }, .{ .dist = 181, .length = 3 },
293 .{ .dist = 0, .length = 2 }, .{ .dist = 23, .length = 3 }, .{ .dist = 77, .length = 3 },
294 .{ .dist = 28, .length = 5 }, .{ .dist = 128, .length = 3 }, .{ .dist = 110, .length = 4 },
295 .{ .dist = 70, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 85, .length = 6 },
296 .{ .dist = 0, .length = 2 }, .{ .dist = 182, .length = 6 }, .{ .dist = 0, .length = 4 },
297 .{ .dist = 133, .length = 3 }, .{ .dist = 0, .length = 7 }, .{ .dist = 47, .length = 5 },
298 .{ .dist = 0, .length = 20 }, .{ .dist = 112, .length = 5 }, .{ .dist = 0, .length = 1 },
299 .{ .dist = 58, .length = 3 }, .{ .dist = 0, .length = 8 }, .{ .dist = 59, .length = 3 },
300 .{ .dist = 0, .length = 4 }, .{ .dist = 173, .length = 3 }, .{ .dist = 0, .length = 5 },
301 .{ .dist = 114, .length = 3 }, .{ .dist = 0, .length = 4 }, .{ .dist = 92, .length = 5 },
302 .{ .dist = 0, .length = 2 }, .{ .dist = 71, .length = 3 }, .{ .dist = 0, .length = 2 },
303 .{ .dist = 76, .length = 5 }, .{ .dist = 0, .length = 1 }, .{ .dist = 46, .length = 3 },
304 .{ .dist = 96, .length = 4 }, .{ .dist = 130, .length = 4 }, .{ .dist = 0, .length = 3 },
305 .{ .dist = 360, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 178, .length = 5 },
306 .{ .dist = 0, .length = 7 }, .{ .dist = 75, .length = 3 }, .{ .dist = 0, .length = 3 },
307 .{ .dist = 45, .length = 6 }, .{ .dist = 0, .length = 6 }, .{ .dist = 299, .length = 6 },
308 .{ .dist = 180, .length = 3 }, .{ .dist = 70, .length = 6 }, .{ .dist = 0, .length = 1 },
309 .{ .dist = 48, .length = 3 }, .{ .dist = 66, .length = 4 }, .{ .dist = 0, .length = 3 },
310 .{ .dist = 47, .length = 5 }, .{ .dist = 0, .length = 9 }, .{ .dist = 325, .length = 3 },
311 .{ .dist = 0, .length = 1 }, .{ .dist = 359, .length = 3 }, .{ .dist = 318, .length = 3 },
312 .{ .dist = 0, .length = 2 }, .{ .dist = 199, .length = 3 }, .{ .dist = 0, .length = 1 },
313 .{ .dist = 344, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 248, .length = 3 },
314 .{ .dist = 0, .length = 10 }, .{ .dist = 310, .length = 3 }, .{ .dist = 0, .length = 3 },
315 .{ .dist = 93, .length = 6 }, .{ .dist = 0, .length = 3 }, .{ .dist = 252, .length = 3 },
316 .{ .dist = 157, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 273, .length = 5 },
317 .{ .dist = 0, .length = 14 }, .{ .dist = 99, .length = 4 }, .{ .dist = 0, .length = 1 },
318 .{ .dist = 464, .length = 4 }, .{ .dist = 0, .length = 2 }, .{ .dist = 92, .length = 4 },
319 .{ .dist = 495, .length = 3 }, .{ .dist = 0, .length = 1 }, .{ .dist = 322, .length = 4 },
320 .{ .dist = 16, .length = 4 }, .{ .dist = 0, .length = 3 }, .{ .dist = 402, .length = 3 },
321 .{ .dist = 0, .length = 2 }, .{ .dist = 237, .length = 4 }, .{ .dist = 0, .length = 2 },
322 .{ .dist = 432, .length = 4 }, .{ .dist = 0, .length = 1 }, .{ .dist = 483, .length = 5 },
323 .{ .dist = 0, .length = 2 }, .{ .dist = 294, .length = 4 }, .{ .dist = 0, .length = 2 },
324 .{ .dist = 306, .length = 3 }, .{ .dist = 113, .length = 5 }, .{ .dist = 0, .length = 1 },
325 .{ .dist = 26, .length = 4 }, .{ .dist = 164, .length = 3 }, .{ .dist = 488, .length = 4 },
326 .{ .dist = 0, .length = 1 }, .{ .dist = 542, .length = 3 }, .{ .dist = 248, .length = 6 },
327 .{ .dist = 0, .length = 5 }, .{ .dist = 205, .length = 3 }, .{ .dist = 0, .length = 8 },
328 .{ .dist = 48, .length = 3 }, .{ .dist = 449, .length = 6 }, .{ .dist = 0, .length = 2 },
329 .{ .dist = 192, .length = 3 }, .{ .dist = 328, .length = 4 }, .{ .dist = 9, .length = 5 },
330 .{ .dist = 433, .length = 3 }, .{ .dist = 0, .length = 3 }, .{ .dist = 622, .length = 25 },
331 .{ .dist = 615, .length = 5 }, .{ .dist = 46, .length = 5 }, .{ .dist = 0, .length = 2 },
332 .{ .dist = 104, .length = 3 }, .{ .dist = 475, .length = 10 }, .{ .dist = 549, .length = 3 },
333 .{ .dist = 0, .length = 4 }, .{ .dist = 597, .length = 8 }, .{ .dist = 314, .length = 3 },
334 .{ .dist = 0, .length = 1 }, .{ .dist = 473, .length = 6 }, .{ .dist = 317, .length = 5 },
335 .{ .dist = 0, .length = 1 }, .{ .dist = 400, .length = 3 }, .{ .dist = 0, .length = 3 },
336 .{ .dist = 109, .length = 3 }, .{ .dist = 151, .length = 3 }, .{ .dist = 48, .length = 4 },
337 .{ .dist = 0, .length = 4 }, .{ .dist = 125, .length = 3 }, .{ .dist = 108, .length = 3 },
338 .{ .dist = 0, .length = 2 },
339 };
340
341 var got_list = ArrayList(u8).init(testing.allocator);
342 defer got_list.deinit();
343 var got = got_list.writer();
344
345 var want_list = ArrayList(u8).init(testing.allocator);
346 defer want_list.deinit();
347 var want = want_list.writer();
348
349 var dd = DictDecoder{};
350 try dd.init(testing.allocator, 1 << 11, null);
351 defer dd.deinit();
352
353 const util = struct {
354 fn writeCopy(dst_dd: *DictDecoder, dst: anytype, dist: u32, length: u32) !void {
355 var len = length;
356 while (len > 0) {
357 var n = dst_dd.tryWriteCopy(dist, len);
358 if (n == 0) {
359 n = dst_dd.writeCopy(dist, len);
360 }
361
362 len -= n;
363 if (dst_dd.availWrite() == 0) {
364 _ = try dst.write(dst_dd.readFlush());
365 }
366 }
367 }
368 fn writeString(dst_dd: *DictDecoder, dst: anytype, str: []const u8) !void {
369 var string = str;
370 while (string.len > 0) {
371 const cnt = DictDecoder.copy(dst_dd.writeSlice(), string);
372 dst_dd.writeMark(cnt);
373 string = string[cnt..];
374 if (dst_dd.availWrite() == 0) {
375 _ = try dst.write(dst_dd.readFlush());
376 }
377 }
378 }
379 };
380
381 try util.writeString(&dd, got, ".");
382 _ = try want.write(".");
383
384 var str = poem;
385 for (poem_refs, 0..) |ref, i| {
386 _ = i;
387 if (ref.dist == 0) {
388 try util.writeString(&dd, got, str[0..ref.length]);
389 } else {
390 try util.writeCopy(&dd, got, ref.dist, ref.length);
391 }
392 str = str[ref.length..];
393 }
394 _ = try want.write(poem);
395
396 try util.writeCopy(&dd, got, dd.histSize(), 33);
397 _ = try want.write(want_list.items[0..33]);
398
399 try util.writeString(&dd, got, abc);
400 try util.writeCopy(&dd, got, abc.len, 59 * abc.len);
401 _ = try want.write(abc ** 60);
402
403 try util.writeString(&dd, got, fox);
404 try util.writeCopy(&dd, got, fox.len, 9 * fox.len);
405 _ = try want.write(fox ** 10);
406
407 try util.writeString(&dd, got, ".");
408 try util.writeCopy(&dd, got, 1, 9);
409 _ = try want.write("." ** 10);
410
411 try util.writeString(&dd, got, uppercase);
412 try util.writeCopy(&dd, got, uppercase.len, 7 * uppercase.len);
413 var i: u8 = 0;
414 while (i < 8) : (i += 1) {
415 _ = try want.write(uppercase);
416 }
417
418 try util.writeCopy(&dd, got, dd.histSize(), 10);
419 _ = try want.write(want_list.items[want_list.items.len - dd.histSize() ..][0..10]);
420
421 _ = try got.write(dd.readFlush());
422 try testing.expectEqualSlices(u8, want_list.items, got_list.items);
423}
lib/std/compress/deflate/huffman_bit_writer.zig deleted-1686
......@@ -1,1686 +0,0 @@
1const std = @import("std");
2const io = std.io;
3
4const Allocator = std.mem.Allocator;
5
6const deflate_const = @import("deflate_const.zig");
7const hm_code = @import("huffman_code.zig");
8const token = @import("token.zig");
9
10// The first length code.
11const length_codes_start = 257;
12
13// The number of codegen codes.
14const codegen_code_count = 19;
15const bad_code = 255;
16
17// buffer_flush_size indicates the buffer size
18// after which bytes are flushed to the writer.
19// Should preferably be a multiple of 6, since
20// we accumulate 6 bytes between writes to the buffer.
21const buffer_flush_size = 240;
22
23// buffer_size is the actual output byte buffer size.
24// It must have additional headroom for a flush
25// which can contain up to 8 bytes.
26const buffer_size = buffer_flush_size + 8;
27
28// The number of extra bits needed by length code X - LENGTH_CODES_START.
29var length_extra_bits = [_]u8{
30 0, 0, 0, // 257
31 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, // 260
32 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, // 270
33 4, 5, 5, 5, 5, 0, // 280
34};
35
36// The length indicated by length code X - LENGTH_CODES_START.
37var length_base = [_]u32{
38 0, 1, 2, 3, 4, 5, 6, 7, 8, 10,
39 12, 14, 16, 20, 24, 28, 32, 40, 48, 56,
40 64, 80, 96, 112, 128, 160, 192, 224, 255,
41};
42
43// offset code word extra bits.
44var offset_extra_bits = [_]i8{
45 0, 0, 0, 0, 1, 1, 2, 2, 3, 3,
46 4, 4, 5, 5, 6, 6, 7, 7, 8, 8,
47 9, 9, 10, 10, 11, 11, 12, 12, 13, 13,
48};
49
50var offset_base = [_]u32{
51 0x000000, 0x000001, 0x000002, 0x000003, 0x000004,
52 0x000006, 0x000008, 0x00000c, 0x000010, 0x000018,
53 0x000020, 0x000030, 0x000040, 0x000060, 0x000080,
54 0x0000c0, 0x000100, 0x000180, 0x000200, 0x000300,
55 0x000400, 0x000600, 0x000800, 0x000c00, 0x001000,
56 0x001800, 0x002000, 0x003000, 0x004000, 0x006000,
57};
58
59// The odd order in which the codegen code sizes are written.
60var codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
61
62pub fn HuffmanBitWriter(comptime WriterType: type) type {
63 return struct {
64 const Self = @This();
65 pub const Error = WriterType.Error;
66
67 // writer is the underlying writer.
68 // Do not use it directly; use the write method, which ensures
69 // that Write errors are sticky.
70 inner_writer: WriterType,
71 bytes_written: usize,
72
73 // Data waiting to be written is bytes[0 .. nbytes]
74 // and then the low nbits of bits. Data is always written
75 // sequentially into the bytes array.
76 bits: u64,
77 nbits: u32, // number of bits
78 bytes: [buffer_size]u8,
79 codegen_freq: [codegen_code_count]u16,
80 nbytes: u32, // number of bytes
81 literal_freq: []u16,
82 offset_freq: []u16,
83 codegen: []u8,
84 literal_encoding: hm_code.HuffmanEncoder,
85 offset_encoding: hm_code.HuffmanEncoder,
86 codegen_encoding: hm_code.HuffmanEncoder,
87 err: bool = false,
88 fixed_literal_encoding: hm_code.HuffmanEncoder,
89 fixed_offset_encoding: hm_code.HuffmanEncoder,
90 allocator: Allocator,
91 huff_offset: hm_code.HuffmanEncoder,
92
93 pub fn reset(self: *Self, new_writer: WriterType) void {
94 self.inner_writer = new_writer;
95 self.bytes_written = 0;
96 self.bits = 0;
97 self.nbits = 0;
98 self.nbytes = 0;
99 self.err = false;
100 }
101
102 pub fn flush(self: *Self) Error!void {
103 if (self.err) {
104 self.nbits = 0;
105 return;
106 }
107 var n = self.nbytes;
108 while (self.nbits != 0) {
109 self.bytes[n] = @as(u8, @truncate(self.bits));
110 self.bits >>= 8;
111 if (self.nbits > 8) { // Avoid underflow
112 self.nbits -= 8;
113 } else {
114 self.nbits = 0;
115 }
116 n += 1;
117 }
118 self.bits = 0;
119 try self.write(self.bytes[0..n]);
120 self.nbytes = 0;
121 }
122
123 fn write(self: *Self, b: []const u8) Error!void {
124 if (self.err) {
125 return;
126 }
127 try self.inner_writer.writeAll(b);
128 self.bytes_written += b.len;
129 }
130
131 fn writeBits(self: *Self, b: u32, nb: u32) Error!void {
132 if (self.err) {
133 return;
134 }
135 self.bits |= @as(u64, @intCast(b)) << @as(u6, @intCast(self.nbits));
136 self.nbits += nb;
137 if (self.nbits >= 48) {
138 const bits = self.bits;
139 self.bits >>= 48;
140 self.nbits -= 48;
141 var n = self.nbytes;
142 var bytes = self.bytes[n..][0..6];
143 bytes[0] = @as(u8, @truncate(bits));
144 bytes[1] = @as(u8, @truncate(bits >> 8));
145 bytes[2] = @as(u8, @truncate(bits >> 16));
146 bytes[3] = @as(u8, @truncate(bits >> 24));
147 bytes[4] = @as(u8, @truncate(bits >> 32));
148 bytes[5] = @as(u8, @truncate(bits >> 40));
149 n += 6;
150 if (n >= buffer_flush_size) {
151 try self.write(self.bytes[0..n]);
152 n = 0;
153 }
154 self.nbytes = n;
155 }
156 }
157
158 pub fn writeBytes(self: *Self, bytes: []const u8) Error!void {
159 if (self.err) {
160 return;
161 }
162 var n = self.nbytes;
163 if (self.nbits & 7 != 0) {
164 self.err = true; // unfinished bits
165 return;
166 }
167 while (self.nbits != 0) {
168 self.bytes[n] = @as(u8, @truncate(self.bits));
169 self.bits >>= 8;
170 self.nbits -= 8;
171 n += 1;
172 }
173 if (n != 0) {
174 try self.write(self.bytes[0..n]);
175 }
176 self.nbytes = 0;
177 try self.write(bytes);
178 }
179
180 // RFC 1951 3.2.7 specifies a special run-length encoding for specifying
181 // the literal and offset lengths arrays (which are concatenated into a single
182 // array). This method generates that run-length encoding.
183 //
184 // The result is written into the codegen array, and the frequencies
185 // of each code is written into the codegen_freq array.
186 // Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
187 // information. Code bad_code is an end marker
188 //
189 // num_literals: The number of literals in literal_encoding
190 // num_offsets: The number of offsets in offset_encoding
191 // lit_enc: The literal encoder to use
192 // off_enc: The offset encoder to use
193 fn generateCodegen(
194 self: *Self,
195 num_literals: u32,
196 num_offsets: u32,
197 lit_enc: *hm_code.HuffmanEncoder,
198 off_enc: *hm_code.HuffmanEncoder,
199 ) void {
200 for (self.codegen_freq, 0..) |_, i| {
201 self.codegen_freq[i] = 0;
202 }
203
204 // Note that we are using codegen both as a temporary variable for holding
205 // a copy of the frequencies, and as the place where we put the result.
206 // This is fine because the output is always shorter than the input used
207 // so far.
208 var codegen = self.codegen; // cache
209 // Copy the concatenated code sizes to codegen. Put a marker at the end.
210 var cgnl = codegen[0..num_literals];
211 for (cgnl, 0..) |_, i| {
212 cgnl[i] = @as(u8, @intCast(lit_enc.codes[i].len));
213 }
214
215 cgnl = codegen[num_literals .. num_literals + num_offsets];
216 for (cgnl, 0..) |_, i| {
217 cgnl[i] = @as(u8, @intCast(off_enc.codes[i].len));
218 }
219 codegen[num_literals + num_offsets] = bad_code;
220
221 var size = codegen[0];
222 var count: i32 = 1;
223 var out_index: u32 = 0;
224 var in_index: u32 = 1;
225 while (size != bad_code) : (in_index += 1) {
226 // INVARIANT: We have seen "count" copies of size that have not yet
227 // had output generated for them.
228 const next_size = codegen[in_index];
229 if (next_size == size) {
230 count += 1;
231 continue;
232 }
233 // We need to generate codegen indicating "count" of size.
234 if (size != 0) {
235 codegen[out_index] = size;
236 out_index += 1;
237 self.codegen_freq[size] += 1;
238 count -= 1;
239 while (count >= 3) {
240 var n: i32 = 6;
241 if (n > count) {
242 n = count;
243 }
244 codegen[out_index] = 16;
245 out_index += 1;
246 codegen[out_index] = @as(u8, @intCast(n - 3));
247 out_index += 1;
248 self.codegen_freq[16] += 1;
249 count -= n;
250 }
251 } else {
252 while (count >= 11) {
253 var n: i32 = 138;
254 if (n > count) {
255 n = count;
256 }
257 codegen[out_index] = 18;
258 out_index += 1;
259 codegen[out_index] = @as(u8, @intCast(n - 11));
260 out_index += 1;
261 self.codegen_freq[18] += 1;
262 count -= n;
263 }
264 if (count >= 3) {
265 // 3 <= count <= 10
266 codegen[out_index] = 17;
267 out_index += 1;
268 codegen[out_index] = @as(u8, @intCast(count - 3));
269 out_index += 1;
270 self.codegen_freq[17] += 1;
271 count = 0;
272 }
273 }
274 count -= 1;
275 while (count >= 0) : (count -= 1) {
276 codegen[out_index] = size;
277 out_index += 1;
278 self.codegen_freq[size] += 1;
279 }
280 // Set up invariant for next time through the loop.
281 size = next_size;
282 count = 1;
283 }
284 // Marker indicating the end of the codegen.
285 codegen[out_index] = bad_code;
286 }
287
288 // dynamicSize returns the size of dynamically encoded data in bits.
289 fn dynamicSize(
290 self: *Self,
291 lit_enc: *hm_code.HuffmanEncoder, // literal encoder
292 off_enc: *hm_code.HuffmanEncoder, // offset encoder
293 extra_bits: u32,
294 ) DynamicSize {
295 var num_codegens = self.codegen_freq.len;
296 while (num_codegens > 4 and self.codegen_freq[codegen_order[num_codegens - 1]] == 0) {
297 num_codegens -= 1;
298 }
299 const header = 3 + 5 + 5 + 4 + (3 * num_codegens) +
300 self.codegen_encoding.bitLength(self.codegen_freq[0..]) +
301 self.codegen_freq[16] * 2 +
302 self.codegen_freq[17] * 3 +
303 self.codegen_freq[18] * 7;
304 const size = header +
305 lit_enc.bitLength(self.literal_freq) +
306 off_enc.bitLength(self.offset_freq) +
307 extra_bits;
308
309 return DynamicSize{
310 .size = @as(u32, @intCast(size)),
311 .num_codegens = @as(u32, @intCast(num_codegens)),
312 };
313 }
314
315 // fixedSize returns the size of dynamically encoded data in bits.
316 fn fixedSize(self: *Self, extra_bits: u32) u32 {
317 return 3 +
318 self.fixed_literal_encoding.bitLength(self.literal_freq) +
319 self.fixed_offset_encoding.bitLength(self.offset_freq) +
320 extra_bits;
321 }
322
323 // storedSizeFits calculates the stored size, including header.
324 // The function returns the size in bits and whether the block
325 // fits inside a single block.
326 fn storedSizeFits(in: ?[]const u8) StoredSize {
327 if (in == null) {
328 return .{ .size = 0, .storable = false };
329 }
330 if (in.?.len <= deflate_const.max_store_block_size) {
331 return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
332 }
333 return .{ .size = 0, .storable = false };
334 }
335
336 fn writeCode(self: *Self, c: hm_code.HuffCode) Error!void {
337 if (self.err) {
338 return;
339 }
340 self.bits |= @as(u64, @intCast(c.code)) << @as(u6, @intCast(self.nbits));
341 self.nbits += @as(u32, @intCast(c.len));
342 if (self.nbits >= 48) {
343 const bits = self.bits;
344 self.bits >>= 48;
345 self.nbits -= 48;
346 var n = self.nbytes;
347 var bytes = self.bytes[n..][0..6];
348 bytes[0] = @as(u8, @truncate(bits));
349 bytes[1] = @as(u8, @truncate(bits >> 8));
350 bytes[2] = @as(u8, @truncate(bits >> 16));
351 bytes[3] = @as(u8, @truncate(bits >> 24));
352 bytes[4] = @as(u8, @truncate(bits >> 32));
353 bytes[5] = @as(u8, @truncate(bits >> 40));
354 n += 6;
355 if (n >= buffer_flush_size) {
356 try self.write(self.bytes[0..n]);
357 n = 0;
358 }
359 self.nbytes = n;
360 }
361 }
362
363 // Write the header of a dynamic Huffman block to the output stream.
364 //
365 // num_literals: The number of literals specified in codegen
366 // num_offsets: The number of offsets specified in codegen
367 // num_codegens: The number of codegens used in codegen
368 // is_eof: Is it the end-of-file? (end of stream)
369 fn writeDynamicHeader(
370 self: *Self,
371 num_literals: u32,
372 num_offsets: u32,
373 num_codegens: u32,
374 is_eof: bool,
375 ) Error!void {
376 if (self.err) {
377 return;
378 }
379 var first_bits: u32 = 4;
380 if (is_eof) {
381 first_bits = 5;
382 }
383 try self.writeBits(first_bits, 3);
384 try self.writeBits(@as(u32, @intCast(num_literals - 257)), 5);
385 try self.writeBits(@as(u32, @intCast(num_offsets - 1)), 5);
386 try self.writeBits(@as(u32, @intCast(num_codegens - 4)), 4);
387
388 var i: u32 = 0;
389 while (i < num_codegens) : (i += 1) {
390 const value = @as(u32, @intCast(self.codegen_encoding.codes[codegen_order[i]].len));
391 try self.writeBits(@as(u32, @intCast(value)), 3);
392 }
393
394 i = 0;
395 while (true) {
396 const code_word: u32 = @as(u32, @intCast(self.codegen[i]));
397 i += 1;
398 if (code_word == bad_code) {
399 break;
400 }
401 try self.writeCode(self.codegen_encoding.codes[@as(u32, @intCast(code_word))]);
402
403 switch (code_word) {
404 16 => {
405 try self.writeBits(@as(u32, @intCast(self.codegen[i])), 2);
406 i += 1;
407 },
408 17 => {
409 try self.writeBits(@as(u32, @intCast(self.codegen[i])), 3);
410 i += 1;
411 },
412 18 => {
413 try self.writeBits(@as(u32, @intCast(self.codegen[i])), 7);
414 i += 1;
415 },
416 else => {},
417 }
418 }
419 }
420
421 pub fn writeStoredHeader(self: *Self, length: usize, is_eof: bool) Error!void {
422 if (self.err) {
423 return;
424 }
425 var flag: u32 = 0;
426 if (is_eof) {
427 flag = 1;
428 }
429 try self.writeBits(flag, 3);
430 try self.flush();
431 try self.writeBits(@as(u32, @intCast(length)), 16);
432 try self.writeBits(@as(u32, @intCast(~@as(u16, @intCast(length)))), 16);
433 }
434
435 fn writeFixedHeader(self: *Self, is_eof: bool) Error!void {
436 if (self.err) {
437 return;
438 }
439 // Indicate that we are a fixed Huffman block
440 var value: u32 = 2;
441 if (is_eof) {
442 value = 3;
443 }
444 try self.writeBits(value, 3);
445 }
446
447 // Write a block of tokens with the smallest encoding.
448 // The original input can be supplied, and if the huffman encoded data
449 // is larger than the original bytes, the data will be written as a
450 // stored block.
451 // If the input is null, the tokens will always be Huffman encoded.
452 pub fn writeBlock(
453 self: *Self,
454 tokens: []const token.Token,
455 eof: bool,
456 input: ?[]const u8,
457 ) Error!void {
458 if (self.err) {
459 return;
460 }
461
462 const lit_and_off = self.indexTokens(tokens);
463 const num_literals = lit_and_off.num_literals;
464 const num_offsets = lit_and_off.num_offsets;
465
466 var extra_bits: u32 = 0;
467 const ret = storedSizeFits(input);
468 const stored_size = ret.size;
469 const storable = ret.storable;
470
471 if (storable) {
472 // We only bother calculating the costs of the extra bits required by
473 // the length of offset fields (which will be the same for both fixed
474 // and dynamic encoding), if we need to compare those two encodings
475 // against stored encoding.
476 var length_code: u32 = length_codes_start + 8;
477 while (length_code < num_literals) : (length_code += 1) {
478 // First eight length codes have extra size = 0.
479 extra_bits += @as(u32, @intCast(self.literal_freq[length_code])) *
480 @as(u32, @intCast(length_extra_bits[length_code - length_codes_start]));
481 }
482 var offset_code: u32 = 4;
483 while (offset_code < num_offsets) : (offset_code += 1) {
484 // First four offset codes have extra size = 0.
485 extra_bits += @as(u32, @intCast(self.offset_freq[offset_code])) *
486 @as(u32, @intCast(offset_extra_bits[offset_code]));
487 }
488 }
489
490 // Figure out smallest code.
491 // Fixed Huffman baseline.
492 var literal_encoding = &self.fixed_literal_encoding;
493 var offset_encoding = &self.fixed_offset_encoding;
494 var size = self.fixedSize(extra_bits);
495
496 // Dynamic Huffman?
497 var num_codegens: u32 = 0;
498
499 // Generate codegen and codegenFrequencies, which indicates how to encode
500 // the literal_encoding and the offset_encoding.
501 self.generateCodegen(
502 num_literals,
503 num_offsets,
504 &self.literal_encoding,
505 &self.offset_encoding,
506 );
507 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
508 const dynamic_size = self.dynamicSize(
509 &self.literal_encoding,
510 &self.offset_encoding,
511 extra_bits,
512 );
513 const dyn_size = dynamic_size.size;
514 num_codegens = dynamic_size.num_codegens;
515
516 if (dyn_size < size) {
517 size = dyn_size;
518 literal_encoding = &self.literal_encoding;
519 offset_encoding = &self.offset_encoding;
520 }
521
522 // Stored bytes?
523 if (storable and stored_size < size) {
524 try self.writeStoredHeader(input.?.len, eof);
525 try self.writeBytes(input.?);
526 return;
527 }
528
529 // Huffman.
530 if (@intFromPtr(literal_encoding) == @intFromPtr(&self.fixed_literal_encoding)) {
531 try self.writeFixedHeader(eof);
532 } else {
533 try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
534 }
535
536 // Write the tokens.
537 try self.writeTokens(tokens, literal_encoding.codes, offset_encoding.codes);
538 }
539
540 // writeBlockDynamic encodes a block using a dynamic Huffman table.
541 // This should be used if the symbols used have a disproportionate
542 // histogram distribution.
543 // If input is supplied and the compression savings are below 1/16th of the
544 // input size the block is stored.
545 pub fn writeBlockDynamic(
546 self: *Self,
547 tokens: []const token.Token,
548 eof: bool,
549 input: ?[]const u8,
550 ) Error!void {
551 if (self.err) {
552 return;
553 }
554
555 const total_tokens = self.indexTokens(tokens);
556 const num_literals = total_tokens.num_literals;
557 const num_offsets = total_tokens.num_offsets;
558
559 // Generate codegen and codegenFrequencies, which indicates how to encode
560 // the literal_encoding and the offset_encoding.
561 self.generateCodegen(
562 num_literals,
563 num_offsets,
564 &self.literal_encoding,
565 &self.offset_encoding,
566 );
567 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
568 const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.offset_encoding, 0);
569 const size = dynamic_size.size;
570 const num_codegens = dynamic_size.num_codegens;
571
572 // Store bytes, if we don't get a reasonable improvement.
573
574 const stored_size = storedSizeFits(input);
575 const ssize = stored_size.size;
576 const storable = stored_size.storable;
577 if (storable and ssize < (size + (size >> 4))) {
578 try self.writeStoredHeader(input.?.len, eof);
579 try self.writeBytes(input.?);
580 return;
581 }
582
583 // Write Huffman table.
584 try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
585
586 // Write the tokens.
587 try self.writeTokens(tokens, self.literal_encoding.codes, self.offset_encoding.codes);
588 }
589
590 const TotalIndexedTokens = struct {
591 num_literals: u32,
592 num_offsets: u32,
593 };
594
595 // Indexes a slice of tokens followed by an end_block_marker, and updates
596 // literal_freq and offset_freq, and generates literal_encoding
597 // and offset_encoding.
598 // The number of literal and offset tokens is returned.
599 fn indexTokens(self: *Self, tokens: []const token.Token) TotalIndexedTokens {
600 var num_literals: u32 = 0;
601 var num_offsets: u32 = 0;
602
603 for (self.literal_freq, 0..) |_, i| {
604 self.literal_freq[i] = 0;
605 }
606 for (self.offset_freq, 0..) |_, i| {
607 self.offset_freq[i] = 0;
608 }
609
610 for (tokens) |t| {
611 if (t < token.match_type) {
612 self.literal_freq[token.literal(t)] += 1;
613 continue;
614 }
615 const length = token.length(t);
616 const offset = token.offset(t);
617 self.literal_freq[length_codes_start + token.lengthCode(length)] += 1;
618 self.offset_freq[token.offsetCode(offset)] += 1;
619 }
620 // add end_block_marker token at the end
621 self.literal_freq[token.literal(deflate_const.end_block_marker)] += 1;
622
623 // get the number of literals
624 num_literals = @as(u32, @intCast(self.literal_freq.len));
625 while (self.literal_freq[num_literals - 1] == 0) {
626 num_literals -= 1;
627 }
628 // get the number of offsets
629 num_offsets = @as(u32, @intCast(self.offset_freq.len));
630 while (num_offsets > 0 and self.offset_freq[num_offsets - 1] == 0) {
631 num_offsets -= 1;
632 }
633 if (num_offsets == 0) {
634 // We haven't found a single match. If we want to go with the dynamic encoding,
635 // we should count at least one offset to be sure that the offset huffman tree could be encoded.
636 self.offset_freq[0] = 1;
637 num_offsets = 1;
638 }
639 self.literal_encoding.generate(self.literal_freq, 15);
640 self.offset_encoding.generate(self.offset_freq, 15);
641 return TotalIndexedTokens{
642 .num_literals = num_literals,
643 .num_offsets = num_offsets,
644 };
645 }
646
647 // Writes a slice of tokens to the output followed by and end_block_marker.
648 // codes for literal and offset encoding must be supplied.
649 fn writeTokens(
650 self: *Self,
651 tokens: []const token.Token,
652 le_codes: []hm_code.HuffCode,
653 oe_codes: []hm_code.HuffCode,
654 ) Error!void {
655 if (self.err) {
656 return;
657 }
658 for (tokens) |t| {
659 if (t < token.match_type) {
660 try self.writeCode(le_codes[token.literal(t)]);
661 continue;
662 }
663 // Write the length
664 const length = token.length(t);
665 const length_code = token.lengthCode(length);
666 try self.writeCode(le_codes[length_code + length_codes_start]);
667 const extra_length_bits = @as(u32, @intCast(length_extra_bits[length_code]));
668 if (extra_length_bits > 0) {
669 const extra_length = @as(u32, @intCast(length - length_base[length_code]));
670 try self.writeBits(extra_length, extra_length_bits);
671 }
672 // Write the offset
673 const offset = token.offset(t);
674 const offset_code = token.offsetCode(offset);
675 try self.writeCode(oe_codes[offset_code]);
676 const extra_offset_bits = @as(u32, @intCast(offset_extra_bits[offset_code]));
677 if (extra_offset_bits > 0) {
678 const extra_offset = @as(u32, @intCast(offset - offset_base[offset_code]));
679 try self.writeBits(extra_offset, extra_offset_bits);
680 }
681 }
682 // add end_block_marker at the end
683 try self.writeCode(le_codes[token.literal(deflate_const.end_block_marker)]);
684 }
685
686 // Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
687 // if the results only gains very little from compression.
688 pub fn writeBlockHuff(self: *Self, eof: bool, input: []const u8) Error!void {
689 if (self.err) {
690 return;
691 }
692
693 // Clear histogram
694 for (self.literal_freq, 0..) |_, i| {
695 self.literal_freq[i] = 0;
696 }
697
698 // Add everything as literals
699 histogram(input, &self.literal_freq);
700
701 self.literal_freq[deflate_const.end_block_marker] = 1;
702
703 const num_literals = deflate_const.end_block_marker + 1;
704 self.offset_freq[0] = 1;
705 const num_offsets = 1;
706
707 self.literal_encoding.generate(self.literal_freq, 15);
708
709 // Figure out smallest code.
710 // Always use dynamic Huffman or Store
711 var num_codegens: u32 = 0;
712
713 // Generate codegen and codegenFrequencies, which indicates how to encode
714 // the literal_encoding and the offset_encoding.
715 self.generateCodegen(
716 num_literals,
717 num_offsets,
718 &self.literal_encoding,
719 &self.huff_offset,
720 );
721 self.codegen_encoding.generate(self.codegen_freq[0..], 7);
722 const dynamic_size = self.dynamicSize(&self.literal_encoding, &self.huff_offset, 0);
723 const size = dynamic_size.size;
724 num_codegens = dynamic_size.num_codegens;
725
726 // Store bytes, if we don't get a reasonable improvement.
727
728 const stored_size_ret = storedSizeFits(input);
729 const ssize = stored_size_ret.size;
730 const storable = stored_size_ret.storable;
731
732 if (storable and ssize < (size + (size >> 4))) {
733 try self.writeStoredHeader(input.len, eof);
734 try self.writeBytes(input);
735 return;
736 }
737
738 // Huffman.
739 try self.writeDynamicHeader(num_literals, num_offsets, num_codegens, eof);
740 const encoding = self.literal_encoding.codes[0..257];
741 var n = self.nbytes;
742 for (input) |t| {
743 // Bitwriting inlined, ~30% speedup
744 const c = encoding[t];
745 self.bits |= @as(u64, @intCast(c.code)) << @as(u6, @intCast(self.nbits));
746 self.nbits += @as(u32, @intCast(c.len));
747 if (self.nbits < 48) {
748 continue;
749 }
750 // Store 6 bytes
751 const bits = self.bits;
752 self.bits >>= 48;
753 self.nbits -= 48;
754 var bytes = self.bytes[n..][0..6];
755 bytes[0] = @as(u8, @truncate(bits));
756 bytes[1] = @as(u8, @truncate(bits >> 8));
757 bytes[2] = @as(u8, @truncate(bits >> 16));
758 bytes[3] = @as(u8, @truncate(bits >> 24));
759 bytes[4] = @as(u8, @truncate(bits >> 32));
760 bytes[5] = @as(u8, @truncate(bits >> 40));
761 n += 6;
762 if (n < buffer_flush_size) {
763 continue;
764 }
765 try self.write(self.bytes[0..n]);
766 if (self.err) {
767 return; // Return early in the event of write failures
768 }
769 n = 0;
770 }
771 self.nbytes = n;
772 try self.writeCode(encoding[deflate_const.end_block_marker]);
773 }
774
775 pub fn deinit(self: *Self) void {
776 self.allocator.free(self.literal_freq);
777 self.allocator.free(self.offset_freq);
778 self.allocator.free(self.codegen);
779 self.literal_encoding.deinit();
780 self.codegen_encoding.deinit();
781 self.offset_encoding.deinit();
782 self.fixed_literal_encoding.deinit();
783 self.fixed_offset_encoding.deinit();
784 self.huff_offset.deinit();
785 }
786 };
787}
788
789const DynamicSize = struct {
790 size: u32,
791 num_codegens: u32,
792};
793
794const StoredSize = struct {
795 size: u32,
796 storable: bool,
797};
798
799pub fn huffmanBitWriter(allocator: Allocator, writer: anytype) !HuffmanBitWriter(@TypeOf(writer)) {
800 var offset_freq = [1]u16{0} ** deflate_const.offset_code_count;
801 offset_freq[0] = 1;
802 // huff_offset is a static offset encoder used for huffman only encoding.
803 // It can be reused since we will not be encoding offset values.
804 var huff_offset = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count);
805 huff_offset.generate(offset_freq[0..], 15);
806
807 return HuffmanBitWriter(@TypeOf(writer)){
808 .inner_writer = writer,
809 .bytes_written = 0,
810 .bits = 0,
811 .nbits = 0,
812 .nbytes = 0,
813 .bytes = [1]u8{0} ** buffer_size,
814 .codegen_freq = [1]u16{0} ** codegen_code_count,
815 .literal_freq = try allocator.alloc(u16, deflate_const.max_num_lit),
816 .offset_freq = try allocator.alloc(u16, deflate_const.offset_code_count),
817 .codegen = try allocator.alloc(u8, deflate_const.max_num_lit + deflate_const.offset_code_count + 1),
818 .literal_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.max_num_lit),
819 .codegen_encoding = try hm_code.newHuffmanEncoder(allocator, codegen_code_count),
820 .offset_encoding = try hm_code.newHuffmanEncoder(allocator, deflate_const.offset_code_count),
821 .allocator = allocator,
822 .fixed_literal_encoding = try hm_code.generateFixedLiteralEncoding(allocator),
823 .fixed_offset_encoding = try hm_code.generateFixedOffsetEncoding(allocator),
824 .huff_offset = huff_offset,
825 };
826}
827
828// histogram accumulates a histogram of b in h.
829//
830// h.len must be >= 256, and h's elements must be all zeroes.
831fn histogram(b: []const u8, h: *[]u16) void {
832 var lh = h.*[0..256];
833 for (b) |t| {
834 lh[t] += 1;
835 }
836}
837
838// tests
839const expect = std.testing.expect;
840const fmt = std.fmt;
841const math = std.math;
842const mem = std.mem;
843const testing = std.testing;
844
845const ArrayList = std.ArrayList;
846
847test "writeBlockHuff" {
848 // Tests huffman encoding against reference files to detect possible regressions.
849 // If encoding/bit allocation changes you can regenerate these files
850
851 try testBlockHuff(
852 "huffman-null-max.input",
853 "huffman-null-max.golden",
854 );
855 try testBlockHuff(
856 "huffman-pi.input",
857 "huffman-pi.golden",
858 );
859 try testBlockHuff(
860 "huffman-rand-1k.input",
861 "huffman-rand-1k.golden",
862 );
863 try testBlockHuff(
864 "huffman-rand-limit.input",
865 "huffman-rand-limit.golden",
866 );
867 try testBlockHuff(
868 "huffman-rand-max.input",
869 "huffman-rand-max.golden",
870 );
871 try testBlockHuff(
872 "huffman-shifts.input",
873 "huffman-shifts.golden",
874 );
875 try testBlockHuff(
876 "huffman-text.input",
877 "huffman-text.golden",
878 );
879 try testBlockHuff(
880 "huffman-text-shift.input",
881 "huffman-text-shift.golden",
882 );
883 try testBlockHuff(
884 "huffman-zero.input",
885 "huffman-zero.golden",
886 );
887}
888
889fn testBlockHuff(comptime in_name: []const u8, comptime want_name: []const u8) !void {
890 const in: []const u8 = @embedFile("testdata/" ++ in_name);
891 const want: []const u8 = @embedFile("testdata/" ++ want_name);
892
893 var buf = ArrayList(u8).init(testing.allocator);
894 defer buf.deinit();
895 var bw = try huffmanBitWriter(testing.allocator, buf.writer());
896 defer bw.deinit();
897 try bw.writeBlockHuff(false, in);
898 try bw.flush();
899
900 try std.testing.expectEqualSlices(u8, want, buf.items);
901
902 // Test if the writer produces the same output after reset.
903 var buf_after_reset = ArrayList(u8).init(testing.allocator);
904 defer buf_after_reset.deinit();
905
906 bw.reset(buf_after_reset.writer());
907
908 try bw.writeBlockHuff(false, in);
909 try bw.flush();
910
911 try std.testing.expectEqualSlices(u8, buf.items, buf_after_reset.items);
912 try std.testing.expectEqualSlices(u8, want, buf_after_reset.items);
913
914 try testWriterEOF(.write_huffman_block, &[0]token.Token{}, in);
915}
916
917const HuffTest = struct {
918 tokens: []const token.Token,
919 input: []const u8 = "", // File name of input data matching the tokens.
920 want: []const u8 = "", // File name of data with the expected output with input available.
921 want_no_input: []const u8 = "", // File name of the expected output when no input is available.
922};
923
924const ml = 0x7fc00000; // Maximum length token. Used to reduce the size of writeBlockTests
925
926const writeBlockTests = &[_]HuffTest{
927 HuffTest{
928 .input = "huffman-null-max.input",
929 .want = "huffman-null-max.{s}.expect",
930 .want_no_input = "huffman-null-max.{s}.expect-noinput",
931 .tokens = &[_]token.Token{
932 0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
933 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
934 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
935 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
936 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
937 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
938 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
939 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
940 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
941 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
942 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
943 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
944 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, 0x0, 0x0,
945 },
946 },
947 HuffTest{
948 .input = "huffman-pi.input",
949 .want = "huffman-pi.{s}.expect",
950 .want_no_input = "huffman-pi.{s}.expect-noinput",
951 .tokens = &[_]token.Token{
952 0x33, 0x2e, 0x31, 0x34, 0x31, 0x35, 0x39,
953 0x32, 0x36, 0x35, 0x33, 0x35, 0x38, 0x39,
954 0x37, 0x39, 0x33, 0x32, 0x33, 0x38, 0x34,
955 0x36, 0x32, 0x36, 0x34, 0x33, 0x33, 0x38,
956 0x33, 0x32, 0x37, 0x39, 0x35, 0x30, 0x32,
957 0x38, 0x38, 0x34, 0x31, 0x39, 0x37, 0x31,
958 0x36, 0x39, 0x33, 0x39, 0x39, 0x33, 0x37,
959 0x35, 0x31, 0x30, 0x35, 0x38, 0x32, 0x30,
960 0x39, 0x37, 0x34, 0x39, 0x34, 0x34, 0x35,
961 0x39, 0x32, 0x33, 0x30, 0x37, 0x38, 0x31,
962 0x36, 0x34, 0x30, 0x36, 0x32, 0x38, 0x36,
963 0x32, 0x30, 0x38, 0x39, 0x39, 0x38, 0x36,
964 0x32, 0x38, 0x30, 0x33, 0x34, 0x38, 0x32,
965 0x35, 0x33, 0x34, 0x32, 0x31, 0x31, 0x37,
966 0x30, 0x36, 0x37, 0x39, 0x38, 0x32, 0x31,
967 0x34, 0x38, 0x30, 0x38, 0x36, 0x35, 0x31,
968 0x33, 0x32, 0x38, 0x32, 0x33, 0x30, 0x36,
969 0x36, 0x34, 0x37, 0x30, 0x39, 0x33, 0x38,
970 0x34, 0x34, 0x36, 0x30, 0x39, 0x35, 0x35,
971 0x30, 0x35, 0x38, 0x32, 0x32, 0x33, 0x31,
972 0x37, 0x32, 0x35, 0x33, 0x35, 0x39, 0x34,
973 0x30, 0x38, 0x31, 0x32, 0x38, 0x34, 0x38,
974 0x31, 0x31, 0x31, 0x37, 0x34, 0x4040007e, 0x34,
975 0x31, 0x30, 0x32, 0x37, 0x30, 0x31, 0x39,
976 0x33, 0x38, 0x35, 0x32, 0x31, 0x31, 0x30,
977 0x35, 0x35, 0x35, 0x39, 0x36, 0x34, 0x34,
978 0x36, 0x32, 0x32, 0x39, 0x34, 0x38, 0x39,
979 0x35, 0x34, 0x39, 0x33, 0x30, 0x33, 0x38,
980 0x31, 0x40400012, 0x32, 0x38, 0x38, 0x31, 0x30,
981 0x39, 0x37, 0x35, 0x36, 0x36, 0x35, 0x39,
982 0x33, 0x33, 0x34, 0x34, 0x36, 0x40400047, 0x37,
983 0x35, 0x36, 0x34, 0x38, 0x32, 0x33, 0x33,
984 0x37, 0x38, 0x36, 0x37, 0x38, 0x33, 0x31,
985 0x36, 0x35, 0x32, 0x37, 0x31, 0x32, 0x30,
986 0x31, 0x39, 0x30, 0x39, 0x31, 0x34, 0x4040001a,
987 0x35, 0x36, 0x36, 0x39, 0x32, 0x33, 0x34,
988 0x36, 0x404000b2, 0x36, 0x31, 0x30, 0x34, 0x35,
989 0x34, 0x33, 0x32, 0x36, 0x40400032, 0x31, 0x33,
990 0x33, 0x39, 0x33, 0x36, 0x30, 0x37, 0x32,
991 0x36, 0x30, 0x32, 0x34, 0x39, 0x31, 0x34,
992 0x31, 0x32, 0x37, 0x33, 0x37, 0x32, 0x34,
993 0x35, 0x38, 0x37, 0x30, 0x30, 0x36, 0x36,
994 0x30, 0x36, 0x33, 0x31, 0x35, 0x35, 0x38,
995 0x38, 0x31, 0x37, 0x34, 0x38, 0x38, 0x31,
996 0x35, 0x32, 0x30, 0x39, 0x32, 0x30, 0x39,
997 0x36, 0x32, 0x38, 0x32, 0x39, 0x32, 0x35,
998 0x34, 0x30, 0x39, 0x31, 0x37, 0x31, 0x35,
999 0x33, 0x36, 0x34, 0x33, 0x36, 0x37, 0x38,
1000 0x39, 0x32, 0x35, 0x39, 0x30, 0x33, 0x36,
1001 0x30, 0x30, 0x31, 0x31, 0x33, 0x33, 0x30,
1002 0x35, 0x33, 0x30, 0x35, 0x34, 0x38, 0x38,
1003 0x32, 0x30, 0x34, 0x36, 0x36, 0x35, 0x32,
1004 0x31, 0x33, 0x38, 0x34, 0x31, 0x34, 0x36,
1005 0x39, 0x35, 0x31, 0x39, 0x34, 0x31, 0x35,
1006 0x31, 0x31, 0x36, 0x30, 0x39, 0x34, 0x33,
1007 0x33, 0x30, 0x35, 0x37, 0x32, 0x37, 0x30,
1008 0x33, 0x36, 0x35, 0x37, 0x35, 0x39, 0x35,
1009 0x39, 0x31, 0x39, 0x35, 0x33, 0x30, 0x39,
1010 0x32, 0x31, 0x38, 0x36, 0x31, 0x31, 0x37,
1011 0x404000e9, 0x33, 0x32, 0x40400009, 0x39, 0x33, 0x31,
1012 0x30, 0x35, 0x31, 0x31, 0x38, 0x35, 0x34,
1013 0x38, 0x30, 0x37, 0x4040010e, 0x33, 0x37, 0x39,
1014 0x39, 0x36, 0x32, 0x37, 0x34, 0x39, 0x35,
1015 0x36, 0x37, 0x33, 0x35, 0x31, 0x38, 0x38,
1016 0x35, 0x37, 0x35, 0x32, 0x37, 0x32, 0x34,
1017 0x38, 0x39, 0x31, 0x32, 0x32, 0x37, 0x39,
1018 0x33, 0x38, 0x31, 0x38, 0x33, 0x30, 0x31,
1019 0x31, 0x39, 0x34, 0x39, 0x31, 0x32, 0x39,
1020 0x38, 0x33, 0x33, 0x36, 0x37, 0x33, 0x33,
1021 0x36, 0x32, 0x34, 0x34, 0x30, 0x36, 0x35,
1022 0x36, 0x36, 0x34, 0x33, 0x30, 0x38, 0x36,
1023 0x30, 0x32, 0x31, 0x33, 0x39, 0x34, 0x39,
1024 0x34, 0x36, 0x33, 0x39, 0x35, 0x32, 0x32,
1025 0x34, 0x37, 0x33, 0x37, 0x31, 0x39, 0x30,
1026 0x37, 0x30, 0x32, 0x31, 0x37, 0x39, 0x38,
1027 0x40800099, 0x37, 0x30, 0x32, 0x37, 0x37, 0x30,
1028 0x35, 0x33, 0x39, 0x32, 0x31, 0x37, 0x31,
1029 0x37, 0x36, 0x32, 0x39, 0x33, 0x31, 0x37,
1030 0x36, 0x37, 0x35, 0x40800232, 0x37, 0x34, 0x38,
1031 0x31, 0x40400006, 0x36, 0x36, 0x39, 0x34, 0x30,
1032 0x404001e7, 0x30, 0x30, 0x30, 0x35, 0x36, 0x38,
1033 0x31, 0x32, 0x37, 0x31, 0x34, 0x35, 0x32,
1034 0x36, 0x33, 0x35, 0x36, 0x30, 0x38, 0x32,
1035 0x37, 0x37, 0x38, 0x35, 0x37, 0x37, 0x31,
1036 0x33, 0x34, 0x32, 0x37, 0x35, 0x37, 0x37,
1037 0x38, 0x39, 0x36, 0x40400129, 0x33, 0x36, 0x33,
1038 0x37, 0x31, 0x37, 0x38, 0x37, 0x32, 0x31,
1039 0x34, 0x36, 0x38, 0x34, 0x34, 0x30, 0x39,
1040 0x30, 0x31, 0x32, 0x32, 0x34, 0x39, 0x35,
1041 0x33, 0x34, 0x33, 0x30, 0x31, 0x34, 0x36,
1042 0x35, 0x34, 0x39, 0x35, 0x38, 0x35, 0x33,
1043 0x37, 0x31, 0x30, 0x35, 0x30, 0x37, 0x39,
1044 0x404000ca, 0x36, 0x40400153, 0x38, 0x39, 0x32, 0x33,
1045 0x35, 0x34, 0x404001c9, 0x39, 0x35, 0x36, 0x31,
1046 0x31, 0x32, 0x31, 0x32, 0x39, 0x30, 0x32,
1047 0x31, 0x39, 0x36, 0x30, 0x38, 0x36, 0x34,
1048 0x30, 0x33, 0x34, 0x34, 0x31, 0x38, 0x31,
1049 0x35, 0x39, 0x38, 0x31, 0x33, 0x36, 0x32,
1050 0x39, 0x37, 0x37, 0x34, 0x40400074, 0x30, 0x39,
1051 0x39, 0x36, 0x30, 0x35, 0x31, 0x38, 0x37,
1052 0x30, 0x37, 0x32, 0x31, 0x31, 0x33, 0x34,
1053 0x39, 0x40800000, 0x38, 0x33, 0x37, 0x32, 0x39,
1054 0x37, 0x38, 0x30, 0x34, 0x39, 0x39, 0x404002da,
1055 0x39, 0x37, 0x33, 0x31, 0x37, 0x33, 0x32,
1056 0x38, 0x4040018a, 0x36, 0x33, 0x31, 0x38, 0x35,
1057 0x40400301, 0x404002e8, 0x34, 0x35, 0x35, 0x33, 0x34,
1058 0x36, 0x39, 0x30, 0x38, 0x33, 0x30, 0x32,
1059 0x36, 0x34, 0x32, 0x35, 0x32, 0x32, 0x33,
1060 0x30, 0x404002e3, 0x40400267, 0x38, 0x35, 0x30, 0x33,
1061 0x35, 0x32, 0x36, 0x31, 0x39, 0x33, 0x31,
1062 0x31, 0x40400212, 0x31, 0x30, 0x31, 0x30, 0x30,
1063 0x30, 0x33, 0x31, 0x33, 0x37, 0x38, 0x33,
1064 0x38, 0x37, 0x35, 0x32, 0x38, 0x38, 0x36,
1065 0x35, 0x38, 0x37, 0x35, 0x33, 0x33, 0x32,
1066 0x30, 0x38, 0x33, 0x38, 0x31, 0x34, 0x32,
1067 0x30, 0x36, 0x40400140, 0x4040012b, 0x31, 0x34, 0x37,
1068 0x33, 0x30, 0x33, 0x35, 0x39, 0x4080032e, 0x39,
1069 0x30, 0x34, 0x32, 0x38, 0x37, 0x35, 0x35,
1070 0x34, 0x36, 0x38, 0x37, 0x33, 0x31, 0x31,
1071 0x35, 0x39, 0x35, 0x40400355, 0x33, 0x38, 0x38,
1072 0x32, 0x33, 0x35, 0x33, 0x37, 0x38, 0x37,
1073 0x35, 0x4080037f, 0x39, 0x4040013a, 0x31, 0x40400148, 0x38,
1074 0x30, 0x35, 0x33, 0x4040018a, 0x32, 0x32, 0x36,
1075 0x38, 0x30, 0x36, 0x36, 0x31, 0x33, 0x30,
1076 0x30, 0x31, 0x39, 0x32, 0x37, 0x38, 0x37,
1077 0x36, 0x36, 0x31, 0x31, 0x31, 0x39, 0x35,
1078 0x39, 0x40400237, 0x36, 0x40800124, 0x38, 0x39, 0x33,
1079 0x38, 0x30, 0x39, 0x35, 0x32, 0x35, 0x37,
1080 0x32, 0x30, 0x31, 0x30, 0x36, 0x35, 0x34,
1081 0x38, 0x35, 0x38, 0x36, 0x33, 0x32, 0x37,
1082 0x4040009a, 0x39, 0x33, 0x36, 0x31, 0x35, 0x33,
1083 0x40400220, 0x4080015c, 0x32, 0x33, 0x30, 0x33, 0x30,
1084 0x31, 0x39, 0x35, 0x32, 0x30, 0x33, 0x35,
1085 0x33, 0x30, 0x31, 0x38, 0x35, 0x32, 0x40400171,
1086 0x40400075, 0x33, 0x36, 0x32, 0x32, 0x35, 0x39,
1087 0x39, 0x34, 0x31, 0x33, 0x40400254, 0x34, 0x39,
1088 0x37, 0x32, 0x31, 0x37, 0x404000de, 0x33, 0x34,
1089 0x37, 0x39, 0x31, 0x33, 0x31, 0x35, 0x31,
1090 0x35, 0x35, 0x37, 0x34, 0x38, 0x35, 0x37,
1091 0x32, 0x34, 0x32, 0x34, 0x35, 0x34, 0x31,
1092 0x35, 0x30, 0x36, 0x39, 0x4040013f, 0x38, 0x32,
1093 0x39, 0x35, 0x33, 0x33, 0x31, 0x31, 0x36,
1094 0x38, 0x36, 0x31, 0x37, 0x32, 0x37, 0x38,
1095 0x40400337, 0x39, 0x30, 0x37, 0x35, 0x30, 0x39,
1096 0x4040010d, 0x37, 0x35, 0x34, 0x36, 0x33, 0x37,
1097 0x34, 0x36, 0x34, 0x39, 0x33, 0x39, 0x33,
1098 0x31, 0x39, 0x32, 0x35, 0x35, 0x30, 0x36,
1099 0x30, 0x34, 0x30, 0x30, 0x39, 0x4040026b, 0x31,
1100 0x36, 0x37, 0x31, 0x31, 0x33, 0x39, 0x30,
1101 0x30, 0x39, 0x38, 0x40400335, 0x34, 0x30, 0x31,
1102 0x32, 0x38, 0x35, 0x38, 0x33, 0x36, 0x31,
1103 0x36, 0x30, 0x33, 0x35, 0x36, 0x33, 0x37,
1104 0x30, 0x37, 0x36, 0x36, 0x30, 0x31, 0x30,
1105 0x34, 0x40400172, 0x38, 0x31, 0x39, 0x34, 0x32,
1106 0x39, 0x4080041e, 0x404000ef, 0x4040028b, 0x37, 0x38, 0x33,
1107 0x37, 0x34, 0x404004a8, 0x38, 0x32, 0x35, 0x35,
1108 0x33, 0x37, 0x40800209, 0x32, 0x36, 0x38, 0x4040002e,
1109 0x34, 0x30, 0x34, 0x37, 0x404001d1, 0x34, 0x404004b5,
1110 0x4040038d, 0x38, 0x34, 0x404003a8, 0x36, 0x40c0031f, 0x33,
1111 0x33, 0x31, 0x33, 0x36, 0x37, 0x37, 0x30,
1112 0x32, 0x38, 0x39, 0x38, 0x39, 0x31, 0x35,
1113 0x32, 0x40400062, 0x35, 0x32, 0x31, 0x36, 0x32,
1114 0x30, 0x35, 0x36, 0x39, 0x36, 0x40400411, 0x30,
1115 0x35, 0x38, 0x40400477, 0x35, 0x40400498, 0x35, 0x31,
1116 0x31, 0x40400209, 0x38, 0x32, 0x34, 0x33, 0x30,
1117 0x30, 0x33, 0x35, 0x35, 0x38, 0x37, 0x36,
1118 0x34, 0x30, 0x32, 0x34, 0x37, 0x34, 0x39,
1119 0x36, 0x34, 0x37, 0x33, 0x32, 0x36, 0x33,
1120 0x4040043e, 0x39, 0x39, 0x32, 0x4040044b, 0x34, 0x32,
1121 0x36, 0x39, 0x40c002c5, 0x37, 0x404001d6, 0x34, 0x4040053d,
1122 0x4040041d, 0x39, 0x33, 0x34, 0x31, 0x37, 0x404001ad,
1123 0x31, 0x32, 0x4040002a, 0x34, 0x4040019e, 0x31, 0x35,
1124 0x30, 0x33, 0x30, 0x32, 0x38, 0x36, 0x31,
1125 0x38, 0x32, 0x39, 0x37, 0x34, 0x35, 0x35,
1126 0x35, 0x37, 0x30, 0x36, 0x37, 0x34, 0x40400135,
1127 0x35, 0x30, 0x35, 0x34, 0x39, 0x34, 0x35,
1128 0x38, 0x404001c5, 0x39, 0x40400051, 0x35, 0x36, 0x404001ec,
1129 0x37, 0x32, 0x31, 0x30, 0x37, 0x39, 0x40400159,
1130 0x33, 0x30, 0x4040010a, 0x33, 0x32, 0x31, 0x31,
1131 0x36, 0x35, 0x33, 0x34, 0x34, 0x39, 0x38,
1132 0x37, 0x32, 0x30, 0x32, 0x37, 0x4040011b, 0x30,
1133 0x32, 0x33, 0x36, 0x34, 0x4040022e, 0x35, 0x34,
1134 0x39, 0x39, 0x31, 0x31, 0x39, 0x38, 0x40400418,
1135 0x34, 0x4040011b, 0x35, 0x33, 0x35, 0x36, 0x36,
1136 0x33, 0x36, 0x39, 0x40400450, 0x32, 0x36, 0x35,
1137 0x404002e4, 0x37, 0x38, 0x36, 0x32, 0x35, 0x35,
1138 0x31, 0x404003da, 0x31, 0x37, 0x35, 0x37, 0x34,
1139 0x36, 0x37, 0x32, 0x38, 0x39, 0x30, 0x39,
1140 0x37, 0x37, 0x37, 0x37, 0x40800453, 0x30, 0x30,
1141 0x30, 0x404005fd, 0x37, 0x30, 0x404004df, 0x36, 0x404003e9,
1142 0x34, 0x39, 0x31, 0x4040041e, 0x40400297, 0x32, 0x31,
1143 0x34, 0x37, 0x37, 0x32, 0x33, 0x35, 0x30,
1144 0x31, 0x34, 0x31, 0x34, 0x40400643, 0x33, 0x35,
1145 0x36, 0x404004af, 0x31, 0x36, 0x31, 0x33, 0x36,
1146 0x31, 0x31, 0x35, 0x37, 0x33, 0x35, 0x32,
1147 0x35, 0x40400504, 0x33, 0x34, 0x4040005b, 0x31, 0x38,
1148 0x4040047b, 0x38, 0x34, 0x404005e7, 0x33, 0x33, 0x32,
1149 0x33, 0x39, 0x30, 0x37, 0x33, 0x39, 0x34,
1150 0x31, 0x34, 0x33, 0x33, 0x33, 0x34, 0x35,
1151 0x34, 0x37, 0x37, 0x36, 0x32, 0x34, 0x40400242,
1152 0x32, 0x35, 0x31, 0x38, 0x39, 0x38, 0x33,
1153 0x35, 0x36, 0x39, 0x34, 0x38, 0x35, 0x35,
1154 0x36, 0x32, 0x30, 0x39, 0x39, 0x32, 0x31,
1155 0x39, 0x32, 0x32, 0x32, 0x31, 0x38, 0x34,
1156 0x32, 0x37, 0x4040023e, 0x32, 0x404000ba, 0x36, 0x38,
1157 0x38, 0x37, 0x36, 0x37, 0x31, 0x37, 0x39,
1158 0x30, 0x40400055, 0x30, 0x40800106, 0x36, 0x36, 0x404003e7,
1159 0x38, 0x38, 0x36, 0x32, 0x37, 0x32, 0x404006dc,
1160 0x31, 0x37, 0x38, 0x36, 0x30, 0x38, 0x35,
1161 0x37, 0x40400073, 0x33, 0x408002fc, 0x37, 0x39, 0x37,
1162 0x36, 0x36, 0x38, 0x31, 0x404002bd, 0x30, 0x30,
1163 0x39, 0x35, 0x33, 0x38, 0x38, 0x40400638, 0x33,
1164 0x404006a5, 0x30, 0x36, 0x38, 0x30, 0x30, 0x36,
1165 0x34, 0x32, 0x32, 0x35, 0x31, 0x32, 0x35,
1166 0x32, 0x4040057b, 0x37, 0x33, 0x39, 0x32, 0x40400297,
1167 0x40400474, 0x34, 0x408006b3, 0x38, 0x36, 0x32, 0x36,
1168 0x39, 0x34, 0x35, 0x404001e5, 0x34, 0x31, 0x39,
1169 0x36, 0x35, 0x32, 0x38, 0x35, 0x30, 0x40400099,
1170 0x4040039c, 0x31, 0x38, 0x36, 0x33, 0x404001be, 0x34,
1171 0x40800154, 0x32, 0x30, 0x33, 0x39, 0x4040058b, 0x34,
1172 0x35, 0x404002bc, 0x32, 0x33, 0x37, 0x4040042c, 0x36,
1173 0x40400510, 0x35, 0x36, 0x40400638, 0x37, 0x31, 0x39,
1174 0x31, 0x37, 0x32, 0x38, 0x40400171, 0x37, 0x36,
1175 0x34, 0x36, 0x35, 0x37, 0x35, 0x37, 0x33,
1176 0x39, 0x40400101, 0x33, 0x38, 0x39, 0x40400748, 0x38,
1177 0x33, 0x32, 0x36, 0x34, 0x35, 0x39, 0x39,
1178 0x35, 0x38, 0x404006a7, 0x30, 0x34, 0x37, 0x38,
1179 0x404001de, 0x40400328, 0x39, 0x4040002d, 0x36, 0x34, 0x30,
1180 0x37, 0x38, 0x39, 0x35, 0x31, 0x4040008e, 0x36,
1181 0x38, 0x33, 0x4040012f, 0x32, 0x35, 0x39, 0x35,
1182 0x37, 0x30, 0x40400468, 0x38, 0x32, 0x32, 0x404002c8,
1183 0x32, 0x4040061b, 0x34, 0x30, 0x37, 0x37, 0x32,
1184 0x36, 0x37, 0x31, 0x39, 0x34, 0x37, 0x38,
1185 0x40400319, 0x38, 0x32, 0x36, 0x30, 0x31, 0x34,
1186 0x37, 0x36, 0x39, 0x39, 0x30, 0x39, 0x404004e8,
1187 0x30, 0x31, 0x33, 0x36, 0x33, 0x39, 0x34,
1188 0x34, 0x33, 0x4040027f, 0x33, 0x30, 0x40400105, 0x32,
1189 0x30, 0x33, 0x34, 0x39, 0x36, 0x32, 0x35,
1190 0x32, 0x34, 0x35, 0x31, 0x37, 0x404003b5, 0x39,
1191 0x36, 0x35, 0x31, 0x34, 0x33, 0x31, 0x34,
1192 0x32, 0x39, 0x38, 0x30, 0x39, 0x31, 0x39,
1193 0x30, 0x36, 0x35, 0x39, 0x32, 0x40400282, 0x37,
1194 0x32, 0x32, 0x31, 0x36, 0x39, 0x36, 0x34,
1195 0x36, 0x40400419, 0x4040007a, 0x35, 0x4040050e, 0x34, 0x40800565,
1196 0x38, 0x40400559, 0x39, 0x37, 0x4040057b, 0x35, 0x34,
1197 0x4040049d, 0x4040023e, 0x37, 0x4040065a, 0x38, 0x34, 0x36,
1198 0x38, 0x31, 0x33, 0x4040008c, 0x36, 0x38, 0x33,
1199 0x38, 0x36, 0x38, 0x39, 0x34, 0x32, 0x37,
1200 0x37, 0x34, 0x31, 0x35, 0x35, 0x39, 0x39,
1201 0x31, 0x38, 0x35, 0x4040005a, 0x32, 0x34, 0x35,
1202 0x39, 0x35, 0x33, 0x39, 0x35, 0x39, 0x34,
1203 0x33, 0x31, 0x404005b7, 0x37, 0x40400012, 0x36, 0x38,
1204 0x30, 0x38, 0x34, 0x35, 0x404002e7, 0x37, 0x33,
1205 0x4040081e, 0x39, 0x35, 0x38, 0x34, 0x38, 0x36,
1206 0x35, 0x33, 0x38, 0x404006e8, 0x36, 0x32, 0x404000f2,
1207 0x36, 0x30, 0x39, 0x404004b6, 0x36, 0x30, 0x38,
1208 0x30, 0x35, 0x31, 0x32, 0x34, 0x33, 0x38,
1209 0x38, 0x34, 0x4040013a, 0x4040000b, 0x34, 0x31, 0x33,
1210 0x4040030f, 0x37, 0x36, 0x32, 0x37, 0x38, 0x40400341,
1211 0x37, 0x31, 0x35, 0x4040059b, 0x33, 0x35, 0x39,
1212 0x39, 0x37, 0x37, 0x30, 0x30, 0x31, 0x32,
1213 0x39, 0x40400472, 0x38, 0x39, 0x34, 0x34, 0x31,
1214 0x40400277, 0x36, 0x38, 0x35, 0x35, 0x4040005f, 0x34,
1215 0x30, 0x36, 0x33, 0x404008e6, 0x32, 0x30, 0x37,
1216 0x32, 0x32, 0x40400158, 0x40800203, 0x34, 0x38, 0x31,
1217 0x35, 0x38, 0x40400205, 0x404001fe, 0x4040027a, 0x40400298, 0x33,
1218 0x39, 0x34, 0x35, 0x32, 0x32, 0x36, 0x37,
1219 0x40c00496, 0x38, 0x4040058a, 0x32, 0x31, 0x404002ea, 0x32,
1220 0x40400387, 0x35, 0x34, 0x36, 0x36, 0x36, 0x4040051b,
1221 0x32, 0x33, 0x39, 0x38, 0x36, 0x34, 0x35,
1222 0x36, 0x404004c4, 0x31, 0x36, 0x33, 0x35, 0x40800253,
1223 0x40400811, 0x37, 0x404008ad, 0x39, 0x38, 0x4040045e, 0x39,
1224 0x33, 0x36, 0x33, 0x34, 0x4040075b, 0x37, 0x34,
1225 0x33, 0x32, 0x34, 0x4040047b, 0x31, 0x35, 0x30,
1226 0x37, 0x36, 0x404004bb, 0x37, 0x39, 0x34, 0x35,
1227 0x31, 0x30, 0x39, 0x4040003e, 0x30, 0x39, 0x34,
1228 0x30, 0x404006a6, 0x38, 0x38, 0x37, 0x39, 0x37,
1229 0x31, 0x30, 0x38, 0x39, 0x33, 0x404008f0, 0x36,
1230 0x39, 0x31, 0x33, 0x36, 0x38, 0x36, 0x37,
1231 0x32, 0x4040025b, 0x404001fe, 0x35, 0x4040053f, 0x40400468, 0x40400801,
1232 0x31, 0x37, 0x39, 0x32, 0x38, 0x36, 0x38,
1233 0x404008cc, 0x38, 0x37, 0x34, 0x37, 0x4080079e, 0x38,
1234 0x32, 0x34, 0x4040097a, 0x38, 0x4040025b, 0x37, 0x31,
1235 0x34, 0x39, 0x30, 0x39, 0x36, 0x37, 0x35,
1236 0x39, 0x38, 0x404006ef, 0x33, 0x36, 0x35, 0x40400134,
1237 0x38, 0x31, 0x4040005c, 0x40400745, 0x40400936, 0x36, 0x38,
1238 0x32, 0x39, 0x4040057e, 0x38, 0x37, 0x32, 0x32,
1239 0x36, 0x35, 0x38, 0x38, 0x30, 0x40400611, 0x35,
1240 0x40400249, 0x34, 0x32, 0x37, 0x30, 0x34, 0x37,
1241 0x37, 0x35, 0x35, 0x4040081e, 0x33, 0x37, 0x39,
1242 0x36, 0x34, 0x31, 0x34, 0x35, 0x31, 0x35,
1243 0x32, 0x404005fd, 0x32, 0x33, 0x34, 0x33, 0x36,
1244 0x34, 0x35, 0x34, 0x404005de, 0x34, 0x34, 0x34,
1245 0x37, 0x39, 0x35, 0x4040003c, 0x40400523, 0x408008e6, 0x34,
1246 0x31, 0x4040052a, 0x33, 0x40400304, 0x35, 0x32, 0x33,
1247 0x31, 0x40800841, 0x31, 0x36, 0x36, 0x31, 0x404008b2,
1248 0x35, 0x39, 0x36, 0x39, 0x35, 0x33, 0x36,
1249 0x32, 0x33, 0x31, 0x34, 0x404005ff, 0x32, 0x34,
1250 0x38, 0x34, 0x39, 0x33, 0x37, 0x31, 0x38,
1251 0x37, 0x31, 0x31, 0x30, 0x31, 0x34, 0x35,
1252 0x37, 0x36, 0x35, 0x34, 0x40400761, 0x30, 0x32,
1253 0x37, 0x39, 0x39, 0x33, 0x34, 0x34, 0x30,
1254 0x33, 0x37, 0x34, 0x32, 0x30, 0x30, 0x37,
1255 0x4040093f, 0x37, 0x38, 0x35, 0x33, 0x39, 0x30,
1256 0x36, 0x32, 0x31, 0x39, 0x40800299, 0x40400345, 0x38,
1257 0x34, 0x37, 0x408003d2, 0x38, 0x33, 0x33, 0x32,
1258 0x31, 0x34, 0x34, 0x35, 0x37, 0x31, 0x40400284,
1259 0x40400776, 0x34, 0x33, 0x35, 0x30, 0x40400928, 0x40400468,
1260 0x35, 0x33, 0x31, 0x39, 0x31, 0x30, 0x34,
1261 0x38, 0x34, 0x38, 0x31, 0x30, 0x30, 0x35,
1262 0x33, 0x37, 0x30, 0x36, 0x404008bc, 0x4080059d, 0x40800781,
1263 0x31, 0x40400559, 0x37, 0x4040031b, 0x35, 0x404007ec, 0x4040040c,
1264 0x36, 0x33, 0x408007dc, 0x34, 0x40400971, 0x4080034e, 0x408003f5,
1265 0x38, 0x4080052d, 0x40800887, 0x39, 0x40400187, 0x39, 0x31,
1266 0x404008ce, 0x38, 0x31, 0x34, 0x36, 0x37, 0x35,
1267 0x31, 0x4040062b, 0x31, 0x32, 0x33, 0x39, 0x40c001a9,
1268 0x39, 0x30, 0x37, 0x31, 0x38, 0x36, 0x34,
1269 0x39, 0x34, 0x32, 0x33, 0x31, 0x39, 0x36,
1270 0x31, 0x35, 0x36, 0x404001ec, 0x404006bc, 0x39, 0x35,
1271 0x40400926, 0x40400469, 0x4040011b, 0x36, 0x30, 0x33, 0x38,
1272 0x40400a25, 0x4040016f, 0x40400384, 0x36, 0x32, 0x4040045a, 0x35,
1273 0x4040084c, 0x36, 0x33, 0x38, 0x39, 0x33, 0x37,
1274 0x37, 0x38, 0x37, 0x404008c5, 0x404000f8, 0x39, 0x37,
1275 0x39, 0x32, 0x30, 0x37, 0x37, 0x33, 0x404005d7,
1276 0x32, 0x31, 0x38, 0x32, 0x35, 0x36, 0x404007df,
1277 0x36, 0x36, 0x404006d6, 0x34, 0x32, 0x4080067e, 0x36,
1278 0x404006e6, 0x34, 0x34, 0x40400024, 0x35, 0x34, 0x39,
1279 0x32, 0x30, 0x32, 0x36, 0x30, 0x35, 0x40400ab3,
1280 0x408003e4, 0x32, 0x30, 0x31, 0x34, 0x39, 0x404004d2,
1281 0x38, 0x35, 0x30, 0x37, 0x33, 0x40400599, 0x36,
1282 0x36, 0x36, 0x30, 0x40400194, 0x32, 0x34, 0x33,
1283 0x34, 0x30, 0x40400087, 0x30, 0x4040076b, 0x38, 0x36,
1284 0x33, 0x40400956, 0x404007e4, 0x4040042b, 0x40400174, 0x35, 0x37,
1285 0x39, 0x36, 0x32, 0x36, 0x38, 0x35, 0x36,
1286 0x40400140, 0x35, 0x30, 0x38, 0x40400523, 0x35, 0x38,
1287 0x37, 0x39, 0x36, 0x39, 0x39, 0x40400711, 0x35,
1288 0x37, 0x34, 0x40400a18, 0x38, 0x34, 0x30, 0x404008b3,
1289 0x31, 0x34, 0x35, 0x39, 0x31, 0x4040078c, 0x37,
1290 0x30, 0x40400234, 0x30, 0x31, 0x40400be7, 0x31, 0x32,
1291 0x40400c74, 0x30, 0x404003c3, 0x33, 0x39, 0x40400b2a, 0x40400112,
1292 0x37, 0x31, 0x35, 0x404003b0, 0x34, 0x32, 0x30,
1293 0x40800bf2, 0x39, 0x40400bc2, 0x30, 0x37, 0x40400341, 0x40400795,
1294 0x40400aaf, 0x40400c62, 0x32, 0x31, 0x40400960, 0x32, 0x35,
1295 0x31, 0x4040057b, 0x40400944, 0x39, 0x32, 0x404001b2, 0x38,
1296 0x32, 0x36, 0x40400b66, 0x32, 0x40400278, 0x33, 0x32,
1297 0x31, 0x35, 0x37, 0x39, 0x31, 0x39, 0x38,
1298 0x34, 0x31, 0x34, 0x4080087b, 0x39, 0x31, 0x36,
1299 0x34, 0x408006e8, 0x39, 0x40800b58, 0x404008db, 0x37, 0x32,
1300 0x32, 0x40400321, 0x35, 0x404008a4, 0x40400141, 0x39, 0x31,
1301 0x30, 0x404000bc, 0x40400c5b, 0x35, 0x32, 0x38, 0x30,
1302 0x31, 0x37, 0x40400231, 0x37, 0x31, 0x32, 0x40400914,
1303 0x38, 0x33, 0x32, 0x40400373, 0x31, 0x40400589, 0x30,
1304 0x39, 0x33, 0x35, 0x33, 0x39, 0x36, 0x35,
1305 0x37, 0x4040064b, 0x31, 0x30, 0x38, 0x33, 0x40400069,
1306 0x35, 0x31, 0x4040077a, 0x40400d5a, 0x31, 0x34, 0x34,
1307 0x34, 0x32, 0x31, 0x30, 0x30, 0x40400202, 0x30,
1308 0x33, 0x4040019c, 0x31, 0x31, 0x30, 0x33, 0x40400c81,
1309 0x40400009, 0x40400026, 0x40c00602, 0x35, 0x31, 0x36, 0x404005d9,
1310 0x40800883, 0x4040092a, 0x35, 0x40800c42, 0x38, 0x35, 0x31,
1311 0x37, 0x31, 0x34, 0x33, 0x37, 0x40400605, 0x4040006d,
1312 0x31, 0x35, 0x35, 0x36, 0x35, 0x30, 0x38,
1313 0x38, 0x404003b9, 0x39, 0x38, 0x39, 0x38, 0x35,
1314 0x39, 0x39, 0x38, 0x32, 0x33, 0x38, 0x404001cf,
1315 0x404009ba, 0x33, 0x4040016c, 0x4040043e, 0x404009c3, 0x38, 0x40800e05,
1316 0x33, 0x32, 0x40400107, 0x35, 0x40400305, 0x33, 0x404001ca,
1317 0x39, 0x4040041b, 0x39, 0x38, 0x4040087d, 0x34, 0x40400cb8,
1318 0x37, 0x4040064b, 0x30, 0x37, 0x404000e5, 0x34, 0x38,
1319 0x31, 0x34, 0x31, 0x40400539, 0x38, 0x35, 0x39,
1320 0x34, 0x36, 0x31, 0x40400bc9, 0x38, 0x30,
1321 },
1322 },
1323 HuffTest{
1324 .input = "huffman-rand-1k.input",
1325 .want = "huffman-rand-1k.{s}.expect",
1326 .want_no_input = "huffman-rand-1k.{s}.expect-noinput",
1327 .tokens = &[_]token.Token{
1328 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xd, 0x85, 0x94, 0x25, 0x80, 0xaf, 0xc2, 0xfe, 0x8d,
1329 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde, 0x6, 0xea, 0x7d, 0x18, 0x8b,
1330 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73, 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4,
1331 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15, 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde,
1332 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9, 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14,
1333 0x13, 0x6f, 0x23, 0xa9, 0xd, 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c,
1334 0x99, 0xdf, 0xfd, 0x70, 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5,
1335 0xd4, 0x80, 0x59, 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20,
1336 0x1b, 0x46, 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4,
1337 0x7b, 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28,
1338 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f, 0xa0,
1339 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4, 0x75, 0xda,
1340 0xea, 0x9b, 0xa, 0x64, 0xb, 0xe0, 0x23, 0x29, 0xbd, 0xf7, 0xe7, 0x83, 0x3c, 0xfb,
1341 0xdf, 0xb3, 0xae, 0x4f, 0xa4, 0x47, 0x55, 0x99, 0xde, 0x2f, 0x96, 0x6e, 0x1c, 0x43,
1342 0x4c, 0x87, 0xe2, 0x7c, 0xd9, 0x5f, 0x4c, 0x7c, 0xe8, 0x90, 0x3, 0xdb, 0x30, 0x95,
1343 0xd6, 0x22, 0xc, 0x47, 0xb8, 0x4d, 0x6b, 0xbd, 0x24, 0x11, 0xab, 0x2c, 0xd7, 0xbe,
1344 0x6e, 0x7a, 0xd6, 0x8, 0xa3, 0x98, 0xd8, 0xdd, 0x15, 0x6a, 0xfa, 0x93, 0x30, 0x1,
1345 0x25, 0x1d, 0xa2, 0x74, 0x86, 0x4b, 0x6a, 0x95, 0xe8, 0xe1, 0x4e, 0xe, 0x76, 0xb9,
1346 0x49, 0xa9, 0x5f, 0xa0, 0xa6, 0x63, 0x3c, 0x7e, 0x7e, 0x20, 0x13, 0x4f, 0xbb, 0x66,
1347 0x92, 0xb8, 0x2e, 0xa4, 0xfa, 0x48, 0xcb, 0xae, 0xb9, 0x3c, 0xaf, 0xd3, 0x1f, 0xe1,
1348 0xd5, 0x8d, 0x42, 0x6d, 0xf0, 0xfc, 0x8c, 0xc, 0x0, 0xde, 0x40, 0xab, 0x8b, 0x47,
1349 0x97, 0x4e, 0xa8, 0xcf, 0x8e, 0xdb, 0xa6, 0x8b, 0x20, 0x9, 0x84, 0x7a, 0x66, 0xe5,
1350 0x98, 0x29, 0x2, 0x95, 0xe6, 0x38, 0x32, 0x60, 0x3, 0xe3, 0x9a, 0x1e, 0x54, 0xe8,
1351 0x63, 0x80, 0x48, 0x9c, 0xe7, 0x63, 0x33, 0x6e, 0xa0, 0x65, 0x83, 0xfa, 0xc6, 0xba,
1352 0x7a, 0x43, 0x71, 0x5, 0xf5, 0x68, 0x69, 0x85, 0x9c, 0xba, 0x45, 0xcd, 0x6b, 0xb,
1353 0x19, 0xd1, 0xbb, 0x7f, 0x70, 0x85, 0x92, 0xd1, 0xb4, 0x64, 0x82, 0xb1, 0xe4, 0x62,
1354 0xc5, 0x3c, 0x46, 0x1f, 0x92, 0x31, 0x1c, 0x4e, 0x41, 0x77, 0xf7, 0xe7, 0x87, 0xa2,
1355 0xf, 0x6e, 0xe8, 0x92, 0x3, 0x6b, 0xa, 0xe7, 0xa9, 0x3b, 0x11, 0xda, 0x66, 0x8a,
1356 0x29, 0xda, 0x79, 0xe1, 0x64, 0x8d, 0xe3, 0x54, 0xd4, 0xf5, 0xef, 0x64, 0x87, 0x3b,
1357 0xf4, 0xc2, 0xf4, 0x71, 0x13, 0xa9, 0xe9, 0xe0, 0xa2, 0x6, 0x14, 0xab, 0x5d, 0xa7,
1358 0x96, 0x0, 0xd6, 0xc3, 0xcc, 0x57, 0xed, 0x39, 0x6a, 0x25, 0xcd, 0x76, 0xea, 0xba,
1359 0x3a, 0xf2, 0xa1, 0x95, 0x5d, 0xe5, 0x71, 0xcf, 0x9c, 0x62, 0x9e, 0x6a, 0xfa, 0xd5,
1360 0x31, 0xd1, 0xa8, 0x66, 0x30, 0x33, 0xaa, 0x51, 0x17, 0x13, 0x82, 0x99, 0xc8, 0x14,
1361 0x60, 0x9f, 0x4d, 0x32, 0x6d, 0xda, 0x19, 0x26, 0x21, 0xdc, 0x7e, 0x2e, 0x25, 0x67,
1362 0x72, 0xca, 0xf, 0x92, 0xcd, 0xf6, 0xd6, 0xcb, 0x97, 0x8a, 0x33, 0x58, 0x73, 0x70,
1363 0x91, 0x1d, 0xbf, 0x28, 0x23, 0xa3, 0xc, 0xf1, 0x83, 0xc3, 0xc8, 0x56, 0x77, 0x68,
1364 0xe3, 0x82, 0xba, 0xb9, 0x57, 0x56, 0x57, 0x9c, 0xc3, 0xd6, 0x14, 0x5, 0x3c, 0xb1,
1365 0xaf, 0x93, 0xc8, 0x8a, 0x57, 0x7f, 0x53, 0xfa, 0x2f, 0xaa, 0x6e, 0x66, 0x83, 0xfa,
1366 0x33, 0xd1, 0x21, 0xab, 0x1b, 0x71, 0xb4, 0x7c, 0xda, 0xfd, 0xfb, 0x7f, 0x20, 0xab,
1367 0x5e, 0xd5, 0xca, 0xfd, 0xdd, 0xe0, 0xee, 0xda, 0xba, 0xa8, 0x27, 0x99, 0x97, 0x69,
1368 0xc1, 0x3c, 0x82, 0x8c, 0xa, 0x5c, 0x2d, 0x5b, 0x88, 0x3e, 0x34, 0x35, 0x86, 0x37,
1369 0x46, 0x79, 0xe1, 0xaa, 0x19, 0xfb, 0xaa, 0xde, 0x15, 0x9, 0xd, 0x1a, 0x57, 0xff,
1370 0xb5, 0xf, 0xf3, 0x2b, 0x5a, 0x6a, 0x4d, 0x19, 0x77, 0x71, 0x45, 0xdf, 0x4f, 0xb3,
1371 0xec, 0xf1, 0xeb, 0x18, 0x53, 0x3e, 0x3b, 0x47, 0x8, 0x9a, 0x73, 0xa0, 0x5c, 0x8c,
1372 0x5f, 0xeb, 0xf, 0x3a, 0xc2, 0x43, 0x67, 0xb4, 0x66, 0x67, 0x80, 0x58, 0xe, 0xc1,
1373 0xec, 0x40, 0xd4, 0x22, 0x94, 0xca, 0xf9, 0xe8, 0x92, 0xe4, 0x69, 0x38, 0xbe, 0x67,
1374 0x64, 0xca, 0x50, 0xc7, 0x6, 0x67, 0x42, 0x6e, 0xa3, 0xf0, 0xb7, 0x6c, 0xf2, 0xe8,
1375 0x5f, 0xb1, 0xaf, 0xe7, 0xdb, 0xbb, 0x77, 0xb5, 0xf8, 0xcb, 0x8, 0xc4, 0x75, 0x7e,
1376 0xc0, 0xf9, 0x1c, 0x7f, 0x3c, 0x89, 0x2f, 0xd2, 0x58, 0x3a, 0xe2, 0xf8, 0x91, 0xb6,
1377 0x7b, 0x24, 0x27, 0xe9, 0xae, 0x84, 0x8b, 0xde, 0x74, 0xac, 0xfd, 0xd9, 0xb7, 0x69,
1378 0x2a, 0xec, 0x32, 0x6f, 0xf0, 0x92, 0x84, 0xf1, 0x40, 0xc, 0x8a, 0xbc, 0x39, 0x6e,
1379 0x2e, 0x73, 0xd4, 0x6e, 0x8a, 0x74, 0x2a, 0xdc, 0x60, 0x1f, 0xa3, 0x7, 0xde, 0x75,
1380 0x8b, 0x74, 0xc8, 0xfe, 0x63, 0x75, 0xf6, 0x3d, 0x63, 0xac, 0x33, 0x89, 0xc3, 0xf0,
1381 0xf8, 0x2d, 0x6b, 0xb4, 0x9e, 0x74, 0x8b, 0x5c, 0x33, 0xb4, 0xca, 0xa8, 0xe4, 0x99,
1382 0xb6, 0x90, 0xa1, 0xef, 0xf, 0xd3, 0x61, 0xb2, 0xc6, 0x1a, 0x94, 0x7c, 0x44, 0x55,
1383 0xf4, 0x45, 0xff, 0x9e, 0xa5, 0x5a, 0xc6, 0xa0, 0xe8, 0x2a, 0xc1, 0x8d, 0x6f, 0x34,
1384 0x11, 0xb9, 0xbe, 0x4e, 0xd9, 0x87, 0x97, 0x73, 0xcf, 0x3d, 0x23, 0xae, 0xd5, 0x1a,
1385 0x5e, 0xae, 0x5d, 0x6a, 0x3, 0xf9, 0x22, 0xd, 0x10, 0xd9, 0x47, 0x69, 0x15, 0x3f,
1386 0xee, 0x52, 0xa3, 0x8, 0xd2, 0x3c, 0x51, 0xf4, 0xf8, 0x9d, 0xe4, 0x98, 0x89, 0xc8,
1387 0x67, 0x39, 0xd5, 0x5e, 0x35, 0x78, 0x27, 0xe8, 0x3c, 0x80, 0xae, 0x79, 0x71, 0xd2,
1388 0x93, 0xf4, 0xaa, 0x51, 0x12, 0x1c, 0x4b, 0x1b, 0xe5, 0x6e, 0x15, 0x6f, 0xe4, 0xbb,
1389 0x51, 0x9b, 0x45, 0x9f, 0xf9, 0xc4, 0x8c, 0x2a, 0xfb, 0x1a, 0xdf, 0x55, 0xd3, 0x48,
1390 0x93, 0x27, 0x1, 0x26, 0xc2, 0x6b, 0x55, 0x6d, 0xa2, 0xfb, 0x84, 0x8b, 0xc9, 0x9e,
1391 0x28, 0xc2, 0xef, 0x1a, 0x24, 0xec, 0x9b, 0xae, 0xbd, 0x60, 0xe9, 0x15, 0x35, 0xee,
1392 0x42, 0xa4, 0x33, 0x5b, 0xfa, 0xf, 0xb6, 0xf7, 0x1, 0xa6, 0x2, 0x4c, 0xca, 0x90,
1393 0x58, 0x3a, 0x96, 0x41, 0xe7, 0xcb, 0x9, 0x8c, 0xdb, 0x85, 0x4d, 0xa8, 0x89, 0xf3,
1394 0xb5, 0x8e, 0xfd, 0x75, 0x5b, 0x4f, 0xed, 0xde, 0x3f, 0xeb, 0x38, 0xa3, 0xbe, 0xb0,
1395 0x73, 0xfc, 0xb8, 0x54, 0xf7, 0x4c, 0x30, 0x67, 0x2e, 0x38, 0xa2, 0x54, 0x18, 0xba,
1396 0x8, 0xbf, 0xf2, 0x39, 0xd5, 0xfe, 0xa5, 0x41, 0xc6, 0x66, 0x66, 0xba, 0x81, 0xef,
1397 0x67, 0xe4, 0xe6, 0x3c, 0xc, 0xca, 0xa4, 0xa, 0x79, 0xb3, 0x57, 0x8b, 0x8a, 0x75,
1398 0x98, 0x18, 0x42, 0x2f, 0x29, 0xa3, 0x82, 0xef, 0x9f, 0x86, 0x6, 0x23, 0xe1, 0x75,
1399 0xfa, 0x8, 0xb1, 0xde, 0x17, 0x4a,
1400 },
1401 },
1402 HuffTest{
1403 .input = "huffman-rand-limit.input",
1404 .want = "huffman-rand-limit.{s}.expect",
1405 .want_no_input = "huffman-rand-limit.{s}.expect-noinput",
1406 .tokens = &[_]token.Token{
1407 0x61, 0x51c00000, 0xa, 0xf8, 0x8b, 0x96, 0x76, 0x48, 0xa, 0x85, 0x94, 0x25, 0x80,
1408 0xaf, 0xc2, 0xfe, 0x8d, 0xe8, 0x20, 0xeb, 0x17, 0x86, 0xc9, 0xb7, 0xc5, 0xde,
1409 0x6, 0xea, 0x7d, 0x18, 0x8b, 0xe7, 0x3e, 0x7, 0xda, 0xdf, 0xff, 0x6c, 0x73,
1410 0xde, 0xcc, 0xe7, 0x6d, 0x8d, 0x4, 0x19, 0x49, 0x7f, 0x47, 0x1f, 0x48, 0x15,
1411 0xb0, 0xe8, 0x9e, 0xf2, 0x31, 0x59, 0xde, 0x34, 0xb4, 0x5b, 0xe5, 0xe0, 0x9,
1412 0x11, 0x30, 0xc2, 0x88, 0x5b, 0x7c, 0x5d, 0x14, 0x13, 0x6f, 0x23, 0xa9, 0xa,
1413 0xbc, 0x2d, 0x23, 0xbe, 0xd9, 0xed, 0x75, 0x4, 0x6c, 0x99, 0xdf, 0xfd, 0x70,
1414 0x66, 0xe6, 0xee, 0xd9, 0xb1, 0x9e, 0x6e, 0x83, 0x59, 0xd5, 0xd4, 0x80, 0x59,
1415 0x98, 0x77, 0x89, 0x43, 0x38, 0xc9, 0xaf, 0x30, 0x32, 0x9a, 0x20, 0x1b, 0x46,
1416 0x3d, 0x67, 0x6e, 0xd7, 0x72, 0x9e, 0x4e, 0x21, 0x4f, 0xc6, 0xe0, 0xd4, 0x7b,
1417 0x4, 0x8d, 0xa5, 0x3, 0xf6, 0x5, 0x9b, 0x6b, 0xdc, 0x2a, 0x93, 0x77, 0x28,
1418 0xfd, 0xb4, 0x62, 0xda, 0x20, 0xe7, 0x1f, 0xab, 0x6b, 0x51, 0x43, 0x39, 0x2f,
1419 0xa0, 0x92, 0x1, 0x6c, 0x75, 0x3e, 0xf4, 0x35, 0xfd, 0x43, 0x2e, 0xf7, 0xa4,
1420 0x75, 0xda, 0xea, 0x9b, 0xa,
1421 },
1422 },
1423 HuffTest{
1424 .input = "huffman-shifts.input",
1425 .want = "huffman-shifts.{s}.expect",
1426 .want_no_input = "huffman-shifts.{s}.expect-noinput",
1427 .tokens = &[_]token.Token{
1428 0x31, 0x30, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1429 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1430 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x52400001, 0xd, 0xa, 0x32,
1431 0x33, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7fc00001,
1432 0x7fc00001, 0x7fc00001, 0x7fc00001, 0x7f400001,
1433 },
1434 },
1435 HuffTest{
1436 .input = "huffman-text-shift.input",
1437 .want = "huffman-text-shift.{s}.expect",
1438 .want_no_input = "huffman-text-shift.{s}.expect-noinput",
1439 .tokens = &[_]token.Token{
1440 0x2f, 0x2f, 0x43, 0x6f, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68,
1441 0x74, 0x32, 0x30, 0x30, 0x39, 0x54, 0x68, 0x47, 0x6f, 0x41,
1442 0x75, 0x74, 0x68, 0x6f, 0x72, 0x2e, 0x41, 0x6c, 0x6c, 0x40800016,
1443 0x72, 0x72, 0x76, 0x64, 0x2e, 0xd, 0xa, 0x2f, 0x2f, 0x55,
1444 0x6f, 0x66, 0x74, 0x68, 0x69, 0x6f, 0x75, 0x72, 0x63, 0x63,
1445 0x6f, 0x64, 0x69, 0x67, 0x6f, 0x76, 0x72, 0x6e, 0x64, 0x62,
1446 0x79, 0x42, 0x53, 0x44, 0x2d, 0x74, 0x79, 0x6c, 0x40400020, 0x6c,
1447 0x69, 0x63, 0x6e, 0x74, 0x68, 0x74, 0x63, 0x6e, 0x62, 0x66,
1448 0x6f, 0x75, 0x6e, 0x64, 0x69, 0x6e, 0x74, 0x68, 0x4c, 0x49,
1449 0x43, 0x45, 0x4e, 0x53, 0x45, 0x66, 0x69, 0x6c, 0x2e, 0xd,
1450 0xa, 0xd, 0xa, 0x70, 0x63, 0x6b, 0x67, 0x6d, 0x69, 0x6e,
1451 0x4040000a, 0x69, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x22, 0x6f, 0x22,
1452 0x4040000c, 0x66, 0x75, 0x6e, 0x63, 0x6d, 0x69, 0x6e, 0x28, 0x29,
1453 0x7b, 0xd, 0xa, 0x9, 0x76, 0x72, 0x62, 0x3d, 0x6d, 0x6b,
1454 0x28, 0x5b, 0x5d, 0x62, 0x79, 0x74, 0x2c, 0x36, 0x35, 0x35,
1455 0x33, 0x35, 0x29, 0xd, 0xa, 0x9, 0x66, 0x2c, 0x5f, 0x3a,
1456 0x3d, 0x6f, 0x2e, 0x43, 0x72, 0x74, 0x28, 0x22, 0x68, 0x75,
1457 0x66, 0x66, 0x6d, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c, 0x2d,
1458 0x6d, 0x78, 0x2e, 0x69, 0x6e, 0x22, 0x40800021, 0x2e, 0x57, 0x72,
1459 0x69, 0x74, 0x28, 0x62, 0x29, 0xd, 0xa, 0x7d, 0xd, 0xa,
1460 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4a,
1461 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, 0x51, 0x52, 0x53, 0x54,
1462 0x55, 0x56, 0x58, 0x78, 0x79, 0x7a, 0x21, 0x22, 0x23, 0xc2,
1463 0xa4, 0x25, 0x26, 0x2f, 0x3f, 0x22,
1464 },
1465 },
1466 HuffTest{
1467 .input = "huffman-text.input",
1468 .want = "huffman-text.{s}.expect",
1469 .want_no_input = "huffman-text.{s}.expect-noinput",
1470 .tokens = &[_]token.Token{
1471 0x2f, 0x2f, 0x20, 0x7a, 0x69, 0x67, 0x20, 0x76,
1472 0x30, 0x2e, 0x31, 0x30, 0x2e, 0x30, 0x0a, 0x2f,
1473 0x2f, 0x20, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65,
1474 0x20, 0x61, 0x20, 0x66, 0x69, 0x6c, 0x65, 0x40400004,
1475 0x6c, 0x65, 0x64, 0x20, 0x77, 0x69, 0x74, 0x68,
1476 0x20, 0x30, 0x78, 0x30, 0x30, 0x0a, 0x63, 0x6f,
1477 0x6e, 0x73, 0x74, 0x20, 0x73, 0x74, 0x64, 0x20,
1478 0x3d, 0x20, 0x40, 0x69, 0x6d, 0x70, 0x6f, 0x72,
1479 0x74, 0x28, 0x22, 0x73, 0x74, 0x64, 0x22, 0x29,
1480 0x3b, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x20, 0x66,
1481 0x6e, 0x20, 0x6d, 0x61, 0x69, 0x6e, 0x28, 0x29,
1482 0x20, 0x21, 0x76, 0x6f, 0x69, 0x64, 0x20, 0x7b,
1483 0x0a, 0x20, 0x20, 0x20, 0x20, 0x76, 0x61, 0x72,
1484 0x20, 0x62, 0x20, 0x3d, 0x20, 0x5b, 0x31, 0x5d,
1485 0x75, 0x38, 0x7b, 0x30, 0x7d, 0x20, 0x2a, 0x2a,
1486 0x20, 0x36, 0x35, 0x35, 0x33, 0x35, 0x3b, 0x4080001e,
1487 0x40c00055, 0x66, 0x20, 0x3d, 0x20, 0x74, 0x72, 0x79,
1488 0x4040005d, 0x2e, 0x66, 0x73, 0x2e, 0x63, 0x77, 0x64,
1489 0x28, 0x29, 0x2e, 0x40c0008f, 0x46, 0x69, 0x6c, 0x65,
1490 0x28, 0x4080002a, 0x40400000, 0x22, 0x68, 0x75, 0x66, 0x66,
1491 0x6d, 0x61, 0x6e, 0x2d, 0x6e, 0x75, 0x6c, 0x6c,
1492 0x2d, 0x6d, 0x61, 0x78, 0x2e, 0x69, 0x6e, 0x22,
1493 0x2c, 0x4180001e, 0x2e, 0x7b, 0x20, 0x2e, 0x72, 0x65,
1494 0x61, 0x64, 0x4080004e, 0x75, 0x65, 0x20, 0x7d, 0x40c0001a,
1495 0x29, 0x40c0006b, 0x64, 0x65, 0x66, 0x65, 0x72, 0x20,
1496 0x66, 0x2e, 0x63, 0x6c, 0x6f, 0x73, 0x65, 0x28,
1497 0x404000b6, 0x40400015, 0x5f, 0x4100007b, 0x66, 0x2e, 0x77, 0x72,
1498 0x69, 0x74, 0x65, 0x41, 0x6c, 0x6c, 0x28, 0x62,
1499 0x5b, 0x30, 0x2e, 0x2e, 0x5d, 0x29, 0x3b, 0x0a,
1500 0x7d, 0x0a,
1501 },
1502 },
1503 HuffTest{
1504 .input = "huffman-zero.input",
1505 .want = "huffman-zero.{s}.expect",
1506 .want_no_input = "huffman-zero.{s}.expect-noinput",
1507 .tokens = &[_]token.Token{ 0x30, ml, 0x4b800000 },
1508 },
1509 HuffTest{
1510 .input = "",
1511 .want = "",
1512 .want_no_input = "null-long-match.{s}.expect-noinput",
1513 .tokens = &[_]token.Token{
1514 0x0, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1515 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1516 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1517 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1518 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1519 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1520 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1521 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1522 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1523 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1524 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1525 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1526 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1527 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1528 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1529 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1530 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1531 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1532 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1533 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1534 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1535 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1536 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1537 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1538 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1539 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1540 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1541 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1542 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1543 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1544 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1545 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1546 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1547 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1548 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1549 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1550 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1551 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
1552 ml, ml, ml, 0x41400000,
1553 },
1554 },
1555};
1556
1557const TestType = enum {
1558 write_block,
1559 write_dyn_block, // write dynamic block
1560 write_huffman_block,
1561
1562 fn to_s(self: TestType) []const u8 {
1563 return switch (self) {
1564 .write_block => "wb",
1565 .write_dyn_block => "dyn",
1566 .write_huffman_block => "huff",
1567 };
1568 }
1569};
1570
1571test "writeBlock" {
1572 // tests if the writeBlock encoding has changed.
1573
1574 const ttype: TestType = .write_block;
1575 try testBlock(writeBlockTests[0], ttype);
1576 try testBlock(writeBlockTests[1], ttype);
1577 try testBlock(writeBlockTests[2], ttype);
1578 try testBlock(writeBlockTests[3], ttype);
1579 try testBlock(writeBlockTests[4], ttype);
1580 try testBlock(writeBlockTests[5], ttype);
1581 try testBlock(writeBlockTests[6], ttype);
1582 try testBlock(writeBlockTests[7], ttype);
1583 try testBlock(writeBlockTests[8], ttype);
1584}
1585
1586test "writeBlockDynamic" {
1587 // tests if the writeBlockDynamic encoding has changed.
1588
1589 const ttype: TestType = .write_dyn_block;
1590 try testBlock(writeBlockTests[0], ttype);
1591 try testBlock(writeBlockTests[1], ttype);
1592 try testBlock(writeBlockTests[2], ttype);
1593 try testBlock(writeBlockTests[3], ttype);
1594 try testBlock(writeBlockTests[4], ttype);
1595 try testBlock(writeBlockTests[5], ttype);
1596 try testBlock(writeBlockTests[6], ttype);
1597 try testBlock(writeBlockTests[7], ttype);
1598 try testBlock(writeBlockTests[8], ttype);
1599}
1600
1601// testBlock tests a block against its references,
1602// or regenerate the references, if "-update" flag is set.
1603fn testBlock(comptime ht: HuffTest, comptime ttype: TestType) !void {
1604 if (ht.input.len != 0 and ht.want.len != 0) {
1605 const want_name = comptime fmt.comptimePrint(ht.want, .{ttype.to_s()});
1606 const input = @embedFile("testdata/" ++ ht.input);
1607 const want = @embedFile("testdata/" ++ want_name);
1608
1609 var buf = ArrayList(u8).init(testing.allocator);
1610 var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1611 try writeToType(ttype, &bw, ht.tokens, input);
1612
1613 var got = buf.items;
1614 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
1615
1616 // Test if the writer produces the same output after reset.
1617 buf.deinit();
1618 buf = ArrayList(u8).init(testing.allocator);
1619 defer buf.deinit();
1620
1621 bw.reset(buf.writer());
1622 defer bw.deinit();
1623
1624 try writeToType(ttype, &bw, ht.tokens, input);
1625 try bw.flush();
1626 got = buf.items;
1627 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
1628 try testWriterEOF(.write_block, ht.tokens, input);
1629 }
1630
1631 const want_name_no_input = comptime fmt.comptimePrint(ht.want_no_input, .{ttype.to_s()});
1632 const want_ni = @embedFile("testdata/" ++ want_name_no_input);
1633
1634 var buf = ArrayList(u8).init(testing.allocator);
1635 var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1636
1637 try writeToType(ttype, &bw, ht.tokens, null);
1638
1639 var got = buf.items;
1640 try testing.expectEqualSlices(u8, want_ni, got); // expect writeBlock to yield expected result
1641 try expect(got[0] & 1 != 1); // expect no EOF
1642
1643 // Test if the writer produces the same output after reset.
1644 buf.deinit();
1645 buf = ArrayList(u8).init(testing.allocator);
1646 defer buf.deinit();
1647
1648 bw.reset(buf.writer());
1649 defer bw.deinit();
1650
1651 try writeToType(ttype, &bw, ht.tokens, null);
1652 try bw.flush();
1653 got = buf.items;
1654
1655 try testing.expectEqualSlices(u8, want_ni, got); // expect writeBlock to yield expected result
1656 try testWriterEOF(.write_block, ht.tokens, &[0]u8{});
1657}
1658
1659fn writeToType(ttype: TestType, bw: anytype, tok: []const token.Token, input: ?[]const u8) !void {
1660 switch (ttype) {
1661 .write_block => try bw.writeBlock(tok, false, input),
1662 .write_dyn_block => try bw.writeBlockDynamic(tok, false, input),
1663 else => unreachable,
1664 }
1665 try bw.flush();
1666}
1667
1668// Tests if the written block contains an EOF marker.
1669fn testWriterEOF(ttype: TestType, ht_tokens: []const token.Token, input: []const u8) !void {
1670 var buf = ArrayList(u8).init(testing.allocator);
1671 defer buf.deinit();
1672 var bw = try huffmanBitWriter(testing.allocator, buf.writer());
1673 defer bw.deinit();
1674
1675 switch (ttype) {
1676 .write_block => try bw.writeBlock(ht_tokens, true, input),
1677 .write_dyn_block => try bw.writeBlockDynamic(ht_tokens, true, input),
1678 .write_huffman_block => try bw.writeBlockHuff(true, input),
1679 }
1680
1681 try bw.flush();
1682
1683 const b = buf.items;
1684 try expect(b.len > 0);
1685 try expect(b[0] & 1 == 1);
1686}
lib/std/compress/deflate/huffman_code.zig deleted-432
......@@ -1,432 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const math = std.math;
4const mem = std.mem;
5const sort = std.sort;
6const testing = std.testing;
7
8const Allocator = std.mem.Allocator;
9
10const bu = @import("bits_utils.zig");
11const deflate_const = @import("deflate_const.zig");
12
13const max_bits_limit = 16;
14
15const LiteralNode = struct {
16 literal: u16,
17 freq: u16,
18};
19
20// Describes the state of the constructed tree for a given depth.
21const LevelInfo = struct {
22 // Our level. for better printing
23 level: u32,
24
25 // The frequency of the last node at this level
26 last_freq: u32,
27
28 // The frequency of the next character to add to this level
29 next_char_freq: u32,
30
31 // The frequency of the next pair (from level below) to add to this level.
32 // Only valid if the "needed" value of the next lower level is 0.
33 next_pair_freq: u32,
34
35 // The number of chains remaining to generate for this level before moving
36 // up to the next level
37 needed: u32,
38};
39
40// hcode is a huffman code with a bit code and bit length.
41pub const HuffCode = struct {
42 code: u16 = 0,
43 len: u16 = 0,
44
45 // set sets the code and length of an hcode.
46 fn set(self: *HuffCode, code: u16, length: u16) void {
47 self.len = length;
48 self.code = code;
49 }
50};
51
52pub const HuffmanEncoder = struct {
53 codes: []HuffCode,
54 freq_cache: []LiteralNode = undefined,
55 bit_count: [17]u32 = undefined,
56 lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
57 lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
58 allocator: Allocator,
59
60 pub fn deinit(self: *HuffmanEncoder) void {
61 self.allocator.free(self.codes);
62 self.allocator.free(self.freq_cache);
63 }
64
65 // Update this Huffman Code object to be the minimum code for the specified frequency count.
66 //
67 // freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
68 // max_bits The maximum number of bits to use for any literal.
69 pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void {
70 var list = self.freq_cache[0 .. freq.len + 1];
71 // Number of non-zero literals
72 var count: u32 = 0;
73 // Set list to be the set of all non-zero literals and their frequencies
74 for (freq, 0..) |f, i| {
75 if (f != 0) {
76 list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
77 count += 1;
78 } else {
79 list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
80 self.codes[i].len = 0;
81 }
82 }
83 list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
84
85 list = list[0..count];
86 if (count <= 2) {
87 // Handle the small cases here, because they are awkward for the general case code. With
88 // two or fewer literals, everything has bit length 1.
89 for (list, 0..) |node, i| {
90 // "list" is in order of increasing literal value.
91 self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
92 }
93 return;
94 }
95 self.lfs = list;
96 mem.sort(LiteralNode, self.lfs, {}, byFreq);
97
98 // Get the number of literals for each bit count
99 const bit_count = self.bitCounts(list, max_bits);
100 // And do the assignment
101 self.assignEncodingAndSize(bit_count, list);
102 }
103
104 pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 {
105 var total: u32 = 0;
106 for (freq, 0..) |f, i| {
107 if (f != 0) {
108 total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
109 }
110 }
111 return total;
112 }
113
114 // Return the number of literals assigned to each bit size in the Huffman encoding
115 //
116 // This method is only called when list.len >= 3
117 // The cases of 0, 1, and 2 literals are handled by special case code.
118 //
119 // list: An array of the literals with non-zero frequencies
120 // and their associated frequencies. The array is in order of increasing
121 // frequency, and has as its last element a special element with frequency
122 // std.math.maxInt(i32)
123 //
124 // max_bits: The maximum number of bits that should be used to encode any literal.
125 // Must be less than 16.
126 //
127 // Returns an integer array in which array[i] indicates the number of literals
128 // that should be encoded in i bits.
129 fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 {
130 var max_bits = max_bits_to_use;
131 const n = list.len;
132
133 assert(max_bits < max_bits_limit);
134
135 // The tree can't have greater depth than n - 1, no matter what. This
136 // saves a little bit of work in some small cases
137 max_bits = @min(max_bits, n - 1);
138
139 // Create information about each of the levels.
140 // A bogus "Level 0" whose sole purpose is so that
141 // level1.prev.needed == 0. This makes level1.next_pair_freq
142 // be a legitimate value that never gets chosen.
143 var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
144 // leaf_counts[i] counts the number of literals at the left
145 // of ancestors of the rightmost node at level i.
146 // leaf_counts[i][j] is the number of literals at the left
147 // of the level j ancestor.
148 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
149
150 {
151 var level = @as(u32, 1);
152 while (level <= max_bits) : (level += 1) {
153 // For every level, the first two items are the first two characters.
154 // We initialize the levels as if we had already figured this out.
155 levels[level] = LevelInfo{
156 .level = level,
157 .last_freq = list[1].freq,
158 .next_char_freq = list[2].freq,
159 .next_pair_freq = list[0].freq + list[1].freq,
160 .needed = 0,
161 };
162 leaf_counts[level][level] = 2;
163 if (level == 1) {
164 levels[level].next_pair_freq = math.maxInt(i32);
165 }
166 }
167 }
168
169 // We need a total of 2*n - 2 items at top level and have already generated 2.
170 levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
171
172 {
173 var level = max_bits;
174 while (true) {
175 var l = &levels[level];
176 if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
177 // We've run out of both leafs and pairs.
178 // End all calculations for this level.
179 // To make sure we never come back to this level or any lower level,
180 // set next_pair_freq impossibly large.
181 l.needed = 0;
182 levels[level + 1].next_pair_freq = math.maxInt(i32);
183 level += 1;
184 continue;
185 }
186
187 const prev_freq = l.last_freq;
188 if (l.next_char_freq < l.next_pair_freq) {
189 // The next item on this row is a leaf node.
190 const next = leaf_counts[level][level] + 1;
191 l.last_freq = l.next_char_freq;
192 // Lower leaf_counts are the same of the previous node.
193 leaf_counts[level][level] = next;
194 if (next >= list.len) {
195 l.next_char_freq = maxNode().freq;
196 } else {
197 l.next_char_freq = list[next].freq;
198 }
199 } else {
200 // The next item on this row is a pair from the previous row.
201 // next_pair_freq isn't valid until we generate two
202 // more values in the level below
203 l.last_freq = l.next_pair_freq;
204 // Take leaf counts from the lower level, except counts[level] remains the same.
205 @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
206 levels[l.level - 1].needed = 2;
207 }
208
209 l.needed -= 1;
210 if (l.needed == 0) {
211 // We've done everything we need to do for this level.
212 // Continue calculating one level up. Fill in next_pair_freq
213 // of that level with the sum of the two nodes we've just calculated on
214 // this level.
215 if (l.level == max_bits) {
216 // All done!
217 break;
218 }
219 levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
220 level += 1;
221 } else {
222 // If we stole from below, move down temporarily to replenish it.
223 while (levels[level - 1].needed > 0) {
224 level -= 1;
225 if (level == 0) {
226 break;
227 }
228 }
229 }
230 }
231 }
232
233 // Somethings is wrong if at the end, the top level is null or hasn't used
234 // all of the leaves.
235 assert(leaf_counts[max_bits][max_bits] == n);
236
237 var bit_count = self.bit_count[0 .. max_bits + 1];
238 var bits: u32 = 1;
239 const counts = &leaf_counts[max_bits];
240 {
241 var level = max_bits;
242 while (level > 0) : (level -= 1) {
243 // counts[level] gives the number of literals requiring at least "bits"
244 // bits to encode.
245 bit_count[bits] = counts[level] - counts[level - 1];
246 bits += 1;
247 if (level == 0) {
248 break;
249 }
250 }
251 }
252 return bit_count;
253 }
254
255 // Look at the leaves and assign them a bit count and an encoding as specified
256 // in RFC 1951 3.2.2
257 fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void {
258 var code = @as(u16, 0);
259 var list = list_arg;
260
261 for (bit_count, 0..) |bits, n| {
262 code <<= 1;
263 if (n == 0 or bits == 0) {
264 continue;
265 }
266 // The literals list[list.len-bits] .. list[list.len-bits]
267 // are encoded using "bits" bits, and get the values
268 // code, code + 1, .... The code values are
269 // assigned in literal order (not frequency order).
270 const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
271
272 self.lns = chunk;
273 mem.sort(LiteralNode, self.lns, {}, byLiteral);
274
275 for (chunk) |node| {
276 self.codes[node.literal] = HuffCode{
277 .code = bu.bitReverse(u16, code, @as(u5, @intCast(n))),
278 .len = @as(u16, @intCast(n)),
279 };
280 code += 1;
281 }
282 list = list[0 .. list.len - @as(u32, @intCast(bits))];
283 }
284 }
285};
286
287fn maxNode() LiteralNode {
288 return LiteralNode{
289 .literal = math.maxInt(u16),
290 .freq = math.maxInt(u16),
291 };
292}
293
294pub fn newHuffmanEncoder(allocator: Allocator, size: u32) !HuffmanEncoder {
295 return HuffmanEncoder{
296 .codes = try allocator.alloc(HuffCode, size),
297 // Allocate a reusable buffer with the longest possible frequency table.
298 // (deflate_const.max_num_frequencies).
299 .freq_cache = try allocator.alloc(LiteralNode, deflate_const.max_num_frequencies + 1),
300 .allocator = allocator,
301 };
302}
303
304// Generates a HuffmanCode corresponding to the fixed literal table
305pub fn generateFixedLiteralEncoding(allocator: Allocator) !HuffmanEncoder {
306 const h = try newHuffmanEncoder(allocator, deflate_const.max_num_frequencies);
307 var codes = h.codes;
308 var ch: u16 = 0;
309
310 while (ch < deflate_const.max_num_frequencies) : (ch += 1) {
311 var bits: u16 = undefined;
312 var size: u16 = undefined;
313 switch (ch) {
314 0...143 => {
315 // size 8, 000110000 .. 10111111
316 bits = ch + 48;
317 size = 8;
318 },
319 144...255 => {
320 // size 9, 110010000 .. 111111111
321 bits = ch + 400 - 144;
322 size = 9;
323 },
324 256...279 => {
325 // size 7, 0000000 .. 0010111
326 bits = ch - 256;
327 size = 7;
328 },
329 else => {
330 // size 8, 11000000 .. 11000111
331 bits = ch + 192 - 280;
332 size = 8;
333 },
334 }
335 codes[ch] = HuffCode{ .code = bu.bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
336 }
337 return h;
338}
339
340pub fn generateFixedOffsetEncoding(allocator: Allocator) !HuffmanEncoder {
341 const h = try newHuffmanEncoder(allocator, 30);
342 var codes = h.codes;
343 for (codes, 0..) |_, ch| {
344 codes[ch] = HuffCode{ .code = bu.bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
345 }
346 return h;
347}
348
349fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
350 _ = context;
351 return a.literal < b.literal;
352}
353
354fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
355 _ = context;
356 if (a.freq == b.freq) {
357 return a.literal < b.literal;
358 }
359 return a.freq < b.freq;
360}
361
362test "generate a Huffman code from an array of frequencies" {
363 var freqs: [19]u16 = [_]u16{
364 8, // 0
365 1, // 1
366 1, // 2
367 2, // 3
368 5, // 4
369 10, // 5
370 9, // 6
371 1, // 7
372 0, // 8
373 0, // 9
374 0, // 10
375 0, // 11
376 0, // 12
377 0, // 13
378 0, // 14
379 0, // 15
380 1, // 16
381 3, // 17
382 5, // 18
383 };
384
385 var enc = try newHuffmanEncoder(testing.allocator, freqs.len);
386 defer enc.deinit();
387 enc.generate(freqs[0..], 7);
388
389 try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
390
391 try testing.expectEqual(@as(usize, 3), enc.codes[0].len);
392 try testing.expectEqual(@as(usize, 6), enc.codes[1].len);
393 try testing.expectEqual(@as(usize, 6), enc.codes[2].len);
394 try testing.expectEqual(@as(usize, 5), enc.codes[3].len);
395 try testing.expectEqual(@as(usize, 3), enc.codes[4].len);
396 try testing.expectEqual(@as(usize, 2), enc.codes[5].len);
397 try testing.expectEqual(@as(usize, 2), enc.codes[6].len);
398 try testing.expectEqual(@as(usize, 6), enc.codes[7].len);
399 try testing.expectEqual(@as(usize, 0), enc.codes[8].len);
400 try testing.expectEqual(@as(usize, 0), enc.codes[9].len);
401 try testing.expectEqual(@as(usize, 0), enc.codes[10].len);
402 try testing.expectEqual(@as(usize, 0), enc.codes[11].len);
403 try testing.expectEqual(@as(usize, 0), enc.codes[12].len);
404 try testing.expectEqual(@as(usize, 0), enc.codes[13].len);
405 try testing.expectEqual(@as(usize, 0), enc.codes[14].len);
406 try testing.expectEqual(@as(usize, 0), enc.codes[15].len);
407 try testing.expectEqual(@as(usize, 6), enc.codes[16].len);
408 try testing.expectEqual(@as(usize, 5), enc.codes[17].len);
409 try testing.expectEqual(@as(usize, 3), enc.codes[18].len);
410
411 try testing.expectEqual(@as(u16, 0x0), enc.codes[5].code);
412 try testing.expectEqual(@as(u16, 0x2), enc.codes[6].code);
413 try testing.expectEqual(@as(u16, 0x1), enc.codes[0].code);
414 try testing.expectEqual(@as(u16, 0x5), enc.codes[4].code);
415 try testing.expectEqual(@as(u16, 0x3), enc.codes[18].code);
416 try testing.expectEqual(@as(u16, 0x7), enc.codes[3].code);
417 try testing.expectEqual(@as(u16, 0x17), enc.codes[17].code);
418 try testing.expectEqual(@as(u16, 0x0f), enc.codes[1].code);
419 try testing.expectEqual(@as(u16, 0x2f), enc.codes[2].code);
420 try testing.expectEqual(@as(u16, 0x1f), enc.codes[7].code);
421 try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
422}
423
424test "generate a Huffman code for the fixed literal table specific to Deflate" {
425 var enc = try generateFixedLiteralEncoding(testing.allocator);
426 defer enc.deinit();
427}
428
429test "generate a Huffman code for the 30 possible relative offsets (LZ77 distances) of Deflate" {
430 var enc = try generateFixedOffsetEncoding(testing.allocator);
431 defer enc.deinit();
432}
lib/std/compress/deflate/testdata/compress-e.txt deleted-1
......@@ -1 +0,0 @@
12.7182818284590452353602874713526624977572470936999595749669676277240766303535475945713821785251664274274663919320030599218174135966290435729003342952605956307381323286279434907632338298807531952510190115738341879307021540891499348841675092447614606680822648001684774118537423454424371075390777449920695517027618386062613313845830007520449338265602976067371132007093287091274437470472306969772093101416928368190255151086574637721112523897844250569536967707854499699679468644549059879316368892300987931277361782154249992295763514822082698951936680331825288693984964651058209392398294887933203625094431173012381970684161403970198376793206832823764648042953118023287825098194558153017567173613320698112509961818815930416903515988885193458072738667385894228792284998920868058257492796104841984443634632449684875602336248270419786232090021609902353043699418491463140934317381436405462531520961836908887070167683964243781405927145635490613031072085103837505101157477041718986106873969655212671546889570350354021234078498193343210681701210056278802351930332247450158539047304199577770935036604169973297250886876966403555707162268447162560798826517871341951246652010305921236677194325278675398558944896970964097545918569563802363701621120477427228364896134225164450781824423529486363721417402388934412479635743702637552944483379980161254922785092577825620926226483262779333865664816277251640191059004916449982893150566047258027786318641551956532442586982946959308019152987211725563475463964479101459040905862984967912874068705048958586717479854667757573205681288459205413340539220001137863009455606881667400169842055804033637953764520304024322566135278369511778838638744396625322498506549958862342818997077332761717839280349465014345588970719425863987727547109629537415211151368350627526023264847287039207643100595841166120545297030236472549296669381151373227536450988890313602057248176585118063036442812314965507047510254465011727211555194866850800368532281831521960037356252794495158284188294787610852639813955990067376482922443752871846245780361929819713991475644882626039033814418232625150974827987779964373089970388867782271383605772978824125611907176639465070633045279546618550966661856647097113444740160704626215680717481877844371436988218559670959102596862002353718588748569652200050311734392073211390803293634479727355955277349071783793421637012050054513263835440001863239914907054797780566978533580489669062951194324730995876552368128590413832411607226029983305353708761389396391779574540161372236187893652605381558415871869255386061647798340254351284396129460352913325942794904337299085731580290958631382683291477116396337092400316894586360606458459251269946557248391865642097526850823075442545993769170419777800853627309417101634349076964237222943523661255725088147792231519747780605696725380171807763603462459278778465850656050780844211529697521890874019660906651803516501792504619501366585436632712549639908549144200014574760819302212066024330096412704894390397177195180699086998606636583232278709376502260149291011517177635944602023249300280401867723910288097866605651183260043688508817157238669842242201024950551881694803221002515426494639812873677658927688163598312477886520141174110913601164995076629077943646005851941998560162647907615321038727557126992518275687989302761761146162549356495903798045838182323368612016243736569846703785853305275833337939907521660692380533698879565137285593883499894707416181550125397064648171946708348197214488898790676503795903669672494992545279033729636162658976039498576741397359441023744329709355477982629614591442936451428617158587339746791897571211956187385783644758448423555581050025611492391518893099463428413936080383091662818811503715284967059741625628236092168075150177725387402564253470879089137291722828611515915683725241630772254406337875931059826760944203261924285317018781772960235413060672136046000389661093647095141417185777014180606443636815464440053316087783143174440811949422975599314011888683314832802706553833004693290115744147563139997221703804617092894579096271662260740718749975359212756084414737823303270330168237193648002173285734935947564334129943024850235732214597843282641421684878721673367010615094243456984401873312810107945127223737886126058165668053714396127888732527373890392890506865324138062796025930387727697783792868409325365880733988457218746021005311483351323850047827169376218004904795597959290591655470505777514308175112698985188408718564026035305583737832422924185625644255022672155980274012617971928047139600689163828665277009752767069777036439260224372841840883251848770472638440379530166905465937461619323840363893131364327137688841026811219891275223056256756254701725086349765367288605966752740868627407912856576996313789753034660616669804218267724560530660773899624218340859882071864682623215080288286359746839654358856685503773131296587975810501214916207656769950659715344763470320853215603674828608378656803073062657633469774295634643716709397193060876963495328846833613038829431040800296873869117066666146800015121143442256023874474325250769387077775193299942137277211258843608715834835626961661980572526612206797540621062080649882918454395301529982092503005498257043390553570168653120526495614857249257386206917403695213533732531666345466588597286659451136441370331393672118569553952108458407244323835586063106806964924851232632699514603596037297253198368423363904632136710116192821711150282801604488058802382031981493096369596735832742024988245684941273860566491352526706046234450549227581151709314921879592718001940968866986837037302200475314338181092708030017205935530520700706072233999463990571311587099635777359027196285061146514837526209565346713290025994397663114545902685898979115837093419370441155121920117164880566945938131183843765620627846310490346293950029458341164824114969758326011800731699437393506966295712410273239138741754923071862454543222039552735295240245903805744502892246886285336542213815722131163288112052146489805180092024719391710555390113943316681515828843687606961102505171007392762385553386272553538830960671644662370922646809671254061869502143176211668140097595281493907222601112681153108387317617323235263605838173151034595736538223534992935822836851007810884634349983518404451704270189381994243410090575376257767571118090088164183319201962623416288166521374717325477727783488774366518828752156685719506371936565390389449366421764003121527870222366463635755503565576948886549500270853923617105502131147413744106134445544192101336172996285694899193369184729478580729156088510396781959429833186480756083679551496636448965592948187851784038773326247051945050419847742014183947731202815886845707290544057510601285258056594703046836344592652552137008068752009593453607316226118728173928074623094685367823106097921599360019946237993434210687813497346959246469752506246958616909178573976595199392993995567542714654910456860702099012606818704984178079173924071945996323060254707901774527513186809982284730860766536866855516467702911336827563107223346726113705490795365834538637196235856312618387156774118738527722922594743373785695538456246801013905727871016512966636764451872465653730402443684140814488732957847348490003019477888020460324660842875351848364959195082888323206522128104190448047247949291342284951970022601310430062410717971502793433263407995960531446053230488528972917659876016667811937932372453857209607582277178483361613582612896226118129455927462767137794487586753657544861407611931125958512655759734573015333642630767985443385761715333462325270572005303988289499034259566232975782488735029259166825894456894655992658454762694528780516501720674785417887982276806536650641910973434528878338621726156269582654478205672987756426325321594294418039943217000090542650763095588465895171709147607437136893319469090981904501290307099566226620303182649365733698419555776963787624918852865686607600566025605445711337286840205574416030837052312242587223438854123179481388550075689381124935386318635287083799845692619981794523364087429591180747453419551420351726184200845509170845682368200897739455842679214273477560879644279202708312150156406341341617166448069815483764491573900121217041547872591998943825364950514771379399147205219529079396137621107238494290616357604596231253506068537651423115349665683715116604220796394466621163255157729070978473156278277598788136491951257483328793771571459091064841642678309949723674420175862269402159407924480541255360431317992696739157542419296607312393763542139230617876753958711436104089409966089471418340698362993675362621545247298464213752891079884381306095552622720837518629837066787224430195793793786072107254277289071732854874374355781966511716618330881129120245204048682200072344035025448202834254187884653602591506445271657700044521097735585897622655484941621714989532383421600114062950718490427789258552743035221396835679018076406042138307308774460170842688272261177180842664333651780002171903449234264266292261456004337383868335555343453004264818473989215627086095650629340405264943244261445665921291225648893569655009154306426134252668472594914314239398845432486327461842846655985332312210466259890141712103446084271616619001257195870793217569698544013397622096749454185407118446433946990162698351607848924514058940946395267807354579700307051163682519487701189764002827648414160587206184185297189154019688253289309149665345753571427318482016384644832499037886069008072709327673127581966563941148961716832980455139729506687604740915420428429993541025829113502241690769431668574242522509026939034814856451303069925199590436384028429267412573422447765584177886171737265462085498294498946787350929581652632072258992368768457017823038096567883112289305809140572610865884845873101658151167533327674887014829167419701512559782572707406431808601428149024146780472327597684269633935773542930186739439716388611764209004068663398856841681003872389214483176070116684503887212364367043314091155733280182977988736590916659612402021778558854876176161989370794380056663364884365089144805571039765214696027662583599051987042300179465536788567430285974600143785483237068701190078499404930918919181649327259774030074879681484882342932023012128032327460392219687528340516906974194257614673978110715464186273369091584973185011183960482533518748438923177292613543024932562896371361977285456622924461644497284597867711574125670307871885109336344480149675240618536569532074170533486782754827815415561966911055101472799040386897220465550833170782394808785990501947563108984124144672821865459971596639015641941751820935932616316888380132758752601460507676098392625726411120135288591317848299475682472564885533357279772205543568126302535748216585414000805314820697137262149755576051890481622376790414926742600071045922695314835188137463887104273544767623577933993970632396604969145303273887874557905934937772320142954803345000695256980935282887783710670585567749481373858630385762823040694005665340584887527005308832459182183494318049834199639981458773435863115940570443683515285383609442955964360676090221741896883548131643997437764158365242234642619597390455450680695232850751868719449064767791886720306418630751053512149851051207313846648717547518382979990189317751550639981016466414592102406838294603208535554058147159273220677567669213664081505900806952540610628536408293276621931939933861623836069111767785448236129326858199965239275488427435414402884536455595124735546139403154952097397051896240157976832639450633230452192645049651735466775699295718989690470902730288544945416699791992948038254980285946029052763145580316514066229171223429375806143993484914362107993576737317948964252488813720435579287511385856973381976083524423240466778020948399639946684833774706725483618848273000648319163826022110555221246733323184463005504481849916996622087746140216157021029603318588727333298779352570182393861244026868339555870607758169954398469568540671174444932479519572159419645863736126915526457574786985964242176592896862383506370433939811671397544736228625506803682664135541448048997721373174119199970017293907303350869020922519124447393278376156321810842898207706974138707053266117683698647741787180202729412982310888796831880854367327806879771659111654224453806625861711729498038248879986504061563975629936962809358189761491017145343556659542757064194408833816841111166200759787244137082333917886114708228657531078536674695018462140736493917366254937783014074302668422150335117736471853872324040421037907750266020114814935482228916663640782450166815341213505278578539332606110249802273093636740213515386431693015267460536064351732154701091440650878823636764236831187390937464232609021646365627553976834019482932795750624399645272578624400375983422050808935129023122475970644105678361870877172333555465482598906861201410107222465904008553798235253885171623518256518482203125214950700378300411216212126052726059944320443056274522916128891766814160639131235975350390320077529587392412476451850809163911459296071156344204347133544720981178461451077872399140606290228276664309264900592249810291068759434533858330391178747575977065953570979640012224092199031158229259667913153991561438070129260780197022589662923368154312499412259460023399472228171056603931877226800493833148980338548909468685130789292064242819174795866199944411196208730498064385006852620258432842085582338566936649849720817046135376163584015342840674118587581546514598270228676671855309311923340191286170613364873183197560812569460089402953094429119590295968563923037689976327462283900735457144596414108229285922239332836210192822937243590283003884445701383771632056518351970100115722010956997890484964453434612129224964732356126321951155701565824427661599326463155806672053127596948538057364208384918887095176052287817339462747644656858900936266123311152910816041524100214195937349786431661556732702792109593543055579732660554677963552005378304619540636971842916168582734122217145885870814274090248185446421774876925093328785670674677381226752831653559245204578070541352576903253522738963847495646255940378924925007624386893776475310102323746733771474581625530698032499033676455430305274561512961214585944432150749051491453950981001388737926379964873728396416897555132275962011838248650746985492038097691932606437608743209385602815642849756549307909733854185583515789409814007691892389063090542534883896831762904120212949167195811935791203162514344096503132835216728021372415947344095498316138322505486708172221475138425166790445416617303200820330902895488808516797258495813407132180533988828139346049850532340472595097214331492586604248511405819579711564191458842833000525684776874305916390494306871343118796189637475503362820939949343690321031976898112055595369465424704173323895394046035325396758354395350516720261647961347790912327995264929045151148307923369382166010702872651938143844844532639517394110131152502750465749343063766541866128915264446926222884366299462732467958736383501937142786471398054038215513463223702071533134887083174146591492406359493020921122052610312390682941345696785958518393491382340884274312419099152870804332809132993078936867127413922890033069995875921815297612482409116951587789964090352577345938248232053055567238095022266790439614231852991989181065554412477204508510210071522352342792531266930108270633942321762570076323139159349709946933241013908779161651226804414809765618979735043151396066913258379033748620836695475083280318786707751177525663963479259219733577949555498655214193398170268639987388347010255262052312317215254062571636771270010760912281528326508984359568975961038372157726831170734552250194121701541318793651818502020877326906133592182000762327269503283827391243828198170871168108951187896746707073377869592565542713340052326706040004348843432902760360498027862160749469654989210474443927871934536701798673920803845633723311983855862638008516345597194441994344624761123844617615736242015935078520825600604101556889899501732554337298073561699861101908472096600708320280569917042590103876928658336557728758684250492690370934262028022399861803400211320742198642917383679176232826444645756330336556777374808644109969141827774253417010988435853189339175934511574023847292909015468559163792696196841000676598399744972047287881831200233383298030567865480871476464512824264478216644266616732096012564794514827125671326697067367144617795643752391742928503987022583734069852309190464967260243411270345611114149835783901793499713790913696706497637127248466613279908254305449295528594932793818341607827091326680865655921102733746700132583428715240835661522165574998431236278287106649401564670141943713823863454729606978693335973109537126499416282656463708490580151538205338326511289504938566468752921135932220265681856418260827538790002407915892646028490894922299966167437731347776134150965262448332709343898412056926145108857812249139616912534202918139898683901335795857624435194008943955180554746554000051766240202825944828833811886381749594284892013520090951007864941868256009273977667585642598378587497776669563350170748579027248701370264203283965756348010818356182372177082236423186591595883669487322411726504487268392328453010991677518376831599821263237123854357312681202445175401852132663740538802901249728180895021553100673598184430429105288459323064725590442355960551978839325930339572934663055160430923785677229293537208416693134575284011873746854691620648991164726909428982971065606801805807843600461866223562874591385185904416250663222249561448724413813849763797102676020845531824111963927941069619465426480006761727618115630063644321116224837379105623611358836334550102286170517890440570419577859833348463317921904494652923021469259756566389965893747728751393377105569802455757436190501772466214587592374418657530064998056688376964229825501195065837843125232135309371235243969149662310110328243570065781487677299160941153954063362752423712935549926713485031578238899567545287915578420483105749330060197958207739558522807307048950936235550769837881926357141779338750216344391014187576711938914416277109602859415809719913429313295145924373636456473035037374538503489286113141638094752301745088784885645741275003353303416138096560043105860548355773946625033230034341587814634602169235079216111013148948281895391028916816328709309713184139815427678818067628650978085718262117003140003377301581536334149093237034703637513354537634521050370995452942055232078817449370937677056009306353645510913481627378204985657055608784211964039972344556458607689515569686899384896439195225232309703301037277227710870564912966121061494072782442033414057441446459968236966118878411656290355117839944070961772567164919790168195234523807446299877664824873753313018142763910519234685081979001796519907050490865237442841652776611425351538665162781316090964802801234493372427866930894827913465443931965254154829494577875758599482099181824522449312077768250830768282335001597040419199560509705364696473142448453825888112602753909548852639708652339052941829691802357120545328231809270356491743371932080628731303589640570873779967845174740515317401384878082881006046388936711640477755985481263907504747295012609419990373721246201677030517790352952793168766305099837441859803498821239340919805055103821539827677291373138006715339240126954586376422065097810852907639079727841301764553247527073788764069366420012194745702358295481365781809867944020220280822637957006755393575808086318932075864444206644691649334467698180811716568665213389686173592450920801465312529777966137198695916451869432324246404401672381978020728394418264502183131483366019384891972317817154372192103946638473715630226701801343515930442853848941825678870721238520597263859224934763623122188113706307506918260109689069251417142514218153491532129077723748506635489170892850760234351768218355008829647410655814882049239533702270536705630750317499788187009989251020178015601042277836283644323729779929935160925884515772055232896978333126427671291093993103773425910592303277652667641874842441076564447767097790392324958416348527735171981064673837142742974468992320406932506062834468937543016787815320616009057693404906146176607094380110915443261929000745209895959201159412324102274845482605404361871836330268992858623582145643879695210235266673372434423091577183277565800211928270391042391966426911155333594569685782817020325495552528875464466074620294766116004435551604735044292127916358748473501590215522120388281168021413865865168464569964810015633741255098479730138656275460161279246359783661480163871602794405482710196290774543628092612567507181773641749763254436773503632580004042919906963117397787875081560227368824967077635559869284901628768699628053790181848148810833946900016380791075960745504688912686792812391148880036720729730801354431325347713094186717178607522981373539126772812593958220524289991371690685650421575056729991274177149279608831502358697816190894908487717722503860872618384947939757440664912760518878124233683125467278331513186758915668300679210215947336858591201395360301678110413444411030903388761520488296909104689167671555373346622545575975202624771242796225983278405833585897671474205724047439720232895903726148688388003174146490203843590358527993123871042845981608996101945691646983837718267264685264869172948414153004604004299585035164101899027529366867431834955447458124140190754681607770977920579383895378192128847409929537040546962226547278807248685508046571043123854873351653070570784584243335550958221912862797205455466267099131902370311779690892786623112661337671178512943059323281605826535623848164192144732543731002062738466812351691016359252588256806438946389880872735284406462208149513862275239938938734905082625472417781702582044129853760499827899020083498387362992498125742354568439023012261733665820546785671147973065077035475620567428300187473019197310881157516777005071432012726354601912460800451608108641835539669946936947322271670748972850464195392966434725254724357659192969949061670189061433616907056148280980363243454128229968275980226694045642181328624517549652147221620839824594576613342710564957193564431561774500828376935700995419541839029151033187933907614207467028867968594985439789457300768939890070073924697461812855764662265412913204052279071212820653775058280040897163467163709024906774736309136904002615646432159560910851092445162454420141442641660181385990017417408244245378610158433361777292580611159192008414091888191208858207627011483671760749046980914443057262211104583300789331698191603917150622792986282709446275915009683226345073725451366858172483498470080840163868209726371345205439802277866337293290829914010645589761697455978409211409167684020269370229231743334499986901841510888993165125090001163719114994852024821586396216294981753094623047604832399379391002142532996476235163569009445086058091202459904612118623318278614464727795523218635916551883057930657703331498510068357135624341881884405780028844018129031378653794869614630467726914552953690154167025838032477842272417994513653582260971652588356712133519546838335349801503269359798167463231847628306340588324731228951257944267639877946713121042763380872695738609314631539148548792514028885025189788076023838995615684850391995855029256054176767663145354058496296796781349420116003325874431438746248313850214980401681940795687219268462617287403480967931949965604299190281810597603263251746405016454606266765529010639868703668263299050577706266397868453584384057673298268163448646707439990917504018892319267557518354054956017732907127219134577524905771512773358423314008356080926962298894163047287780054743798498545562870729968407382937218623831766524716090967192007237658894226186550487552614557855898773008703234726418384831040394818743616224455286163287628541175946460497027724490799275146445792982549802258601001772437840167723166802004162547244179415547810554178036773553354467030326469619447560812831933095679685582771932031205941616693902049665352189672822671972640029493307384717544753761937017882976382487233361813499414541694736549254840633793674361541081593464960431603544354737728802361047743115330785159902977771499610274627769759612488879448609863349422852847651310277926279743981957617505591300993377368240510902583759345170015340522266144077237050890044496613295859536020556034009492820943862994618834790932894161098856594954213114335608810239423706087108026465913203560121875933791639666437282836752328391688865373751335794859860107569374889645657187292540448508624449947816273842517229343960137212406286783636675845331904743954740664015260871940915743955282773904303868772728262065663129387459875317749973799293043294371763801856280061141619563942414312254397099163565102848315765427037906837175764870230052388197498746636856292655058222887713221781440489538099681072143012394693530931524054081215705402274414521876541901428386744260011889041724570537470755550581632831687247110220353727166112304857340460879272501694701067831178927095527253222125224361673343366384756590949728221809418684074238351567868893421148203905824224324264643630201441787982022116248471657468291146315407563770222740135841109076078464780070182766336227978104546331131294044833570134869585165267459515187680033395522410548181767867772152798270250117195816577603549732923724732067853690257536233971216884390878879262188202305529937132397194333083536231248870386416194361506529551267334207198502259771408638122015980894363561808597010080081622557455039101321981979045520049618583777721048046635533806616517023595097133203631578945644487800945620369784973459902004606886572701865867757842758530645706617127194967371083950603267501532435909029491516973738110897934782297684100117657987098185725131372267749706609250481876835516003714638685918913011736805218743265426063700710595364425062760458252336880552521181566417553430681181548267844169315284408461087588214317641649835663127518728182948655658524206852221830755306118393326934164459415342651778653397980580828158806300749952897558204686612590853678738603318442905510689778698417735603118111677563872589911516803236547002987989628986181014596471307916144369564690909518788574398821730583884980809523077569358851616027719521488998358632323127308909861560777386006984035267826785387215920936255817889813416247486456433211043194821421299793188104636399541496539441501383868748384870224681829391860319598667962363489309283087840712400431022706137591368056518861313458307990705003607588327248867879324093380071864152853317943535073401891193638546730000660453783784472469288830546979000131248952100446949032058838294923613919284305249167833012980192255157050378521810552961623637523647962685751660066539364142273063001648652613891842243501797455993616794063303522111829071597538821839777552812981538570168702202620274678647916644030729018445497956399844836807851997088201407769199261674991148329821854382718946282165387064858588646221611410343570342878862979083418871606214430014533275029715104673156021000043869510583773779766003460887624861640938645252177935289947578496255243925598620521409052346250847830487046492688313289470553891357290706967599556298586669559721686506052072801342104355762779184021797626656484580261591407173477009039475168017709900129391137881248534255949312866653465033728846390649968460644741907524313323903404908195233044389559060547854954620263256676813262435925020249516275607080900436460421497025691488555265022810327762115842282433269528629137662675481993546118143913367579700141255870143319434764035725376914388899683088262844616425575034001428982557620386364384137906519612917777354183694676232982904981261717676191554292570438432239918482261744350470199171258214687683172646078959690569981353264435973965173473319484798758064137926885413552523275720457329477215706850016950046959758389373527538622664943456437071610511521617176237598050900553232154896062817794302268640579555845730600598376482703339859420098582351400179507104569019191359062304102336798080907240196312675268916362136351032648077232914950859151265812143823371072949148088472355286394195993455684156344577951727033374238129903260198160571971183950662758220321837136059718025940870615534713104482272716848395524105913605919812444978458110854511231668173534838253724825347636777581712867205865148285317273569069839935110763432091319780314031658897379628301178409806410175016511072932907832177487566289310650383806093372841399226733384778203302020700517188941706465146238366720632742644336612174011766914919235570905644803016342294301837655263108450172510307540942604409687066288066265900569082451407632599158164499361455172452057020443093722305550217222299706209749268609762787409626448772056043078634808885709143464793241536214303199965695610753570417207285334250171325558818113295504095217830139465216436594262960768570585698507157151317262928960072587601564840556088613165411835958628710665496282599535127193244635791046554389165150954187306071015034430609582302257455974944275067630926322529966338219395202927917973247094559691016402983683080426309910481567503623509654924302589575273521412445149542462972258510120707802110188106722347972579330653187713438466713807546383471635428854957610942841898601794658721444495198801550804042506452191484989920400007310672369944655246020908767882300064337725657385010969899058191290957079866699453765080407917852438222041070599278889267745752084287526377986730360561230710723922581504781379172731261234878334034473833573601973235946604273704635201327182592410906040097638585857716958419563109577748529579836844756803121874818202833941887076311731615289811756429711334181497218078040465077657204457082859417475114926179367379999220181789399433337731146911970737861041963986422166045588965683206701337505745038872111332436739840284188639147633491695114032583475841514170325690161784931455706904169858050217798497637014758914810543205854914100662201721719726878930012101267481270235940855162601689425111458499658315589660460091525797881670384625905383256920520425791378948827579603278877535466861441826827797651258953563761485994485049706638406266121957141911063246061774180577212381659872472432252969098533628440799030007594546281549235506086481557928961969617060715201589825299772803520002610888814176506636216905928021516429198484077446143617891415191517976537848282687018750030264867608433204658525470555882410254654806040437372771834769014720664234434374255514129178503032471263418076525187802925534774001104853996960549926508093910691337614841834884596365621526610332239417467064368340504749943339802285610313083038484571294767389856293937641914407036507544622061186499127249643799875806537850203753189972618014404667793050140301580709266213229273649718653952866567538572115133606114457222800851183757899219543063413692302293139751143702404830227357629039911794499248480915071002444078482866598579406525539141041497342780203520135419925977628178182825372022920108186449448349255421793982723279357095828748597126780783134286180750497175747373730296280477376908932558914598141724852658299510882230055223242218586191394795184220131553319634363922684259164168669438122537135960710031743651959027712571604588486044820674410935215327906816032054215967959066411120187618531256710150212239401285668608469435937408158536481912528004920724042172170913983123118054043277015835629513656274610248827706488865037765175678806872498861657094846665770674577000207144332525555736557083150320019082992096545498737419756608619533492312940263904930982014700371161829485939931199955070455381196711289367735249958182011774799788636393286405807810818657337668157893827656450642917396685579555053188715314552353070355994740186225988149854660737787698781542360397080977412361518245964026869979609564523828584235953564615185448165799966460648261396618720304839119560250381111550938420209894591555760083897989949964566262540514195610780090298667014635238532066032574466820259430618801773091109212741138269148784355679352572808875543164693077235363768226036080174040660997151176880434927489197133087822951123746632635635328517394189466510943745768270782209928468034684157443127739811044186762032954475468077511126663685479944460934809992951875666499902261686019672053749149951226823637895865245462813439289338365156536992413109638102559114643923805213907862893561660998836479175633176725856523591069520326895990054884753424160586689820067483163174286329119633399132709086065074595260357157323069712106423424081597068328707624437165532750228797802598690981111226558888151520837482450034463046505984569690276166958278982913613535306291331427881888249342136442417833519319786543940201465328083410341785272489879050919932369270996567133507711905899945951923990615156165480300145359212550696405345263823452155999210578191371030188979206408883974767667144727314254467923500524618849237455307575734902707342496298879996942094595961008702501329453325358045689285707241207965919809225550560061971283541270202072583994171175520920820151096509526685113897577150810849443508285458749912943857563115668324566827992991861539009255871716840495663991959154034218364537212023678608655364745175654879318925644085274489190918193411667583563439758886046349413111875241038425467937999203546910411935443113219136068129657568583611774564654674861061988591414805799318725367531243470335482637527081353105570818049642498584646147973467599315946514787025065271083508782350656532331797738656666181652390017664988485456054961300215776115255813396184027067814900350252876823607822107397102339146870159735868589015297010347780503292154014359595298683404657471756232196640515401477953167461726208727304820634652469109953327375561090578378455945469160223687689641425960164689647106348074109928546482353083540132332924864037318003195202317476206537726163717445360549726690601711176761047774971666890152163838974311714180622222345718567941507299526201086205084783127474791909996889937275229053674785020500038630036526218800670926674104806027341997756660029427941090400064654281074454007616429525362460261476180471744322889953285828397762184600967669267581270302806519535452053173536808954589902180783145775891280203970053633193821100095443241244197949192916205234421346395653840781209416214835001155883618421164283992454027590719621537570187067083731012246141362048926555668109467076386536083015847614512581588569610030337081197058344452874666198891534664244887911940711423940115986970795745946337170243268484864632018986352827092313047089215684758207753034387689978702323438584381125011714013265769320554911860153519551654627941175593967947958810333935413289702528893533748106257875620364294270257512121137330213811951395756419122685155962476203282038726342066227347868223036522019655729325905068134849292299647248229359787842720945578267329975853818536442370617353517653060396801087899490506654491544577952166038552398013798104340564182403396162494910454712104839439200945914647542424785991096900046541371091630096785951563947332190934511838669964622788855817353221326876634958059123761251203010983867841195725887799206041260049865895027247133146763722204388398558347770112599424691208308595666787531942465131444389971195968105937957532155524204659410081418351120174196853432672343271868099625045432475688702055341969199545300952644398446384346598830418262932239295612610045884644244285011551557765935780379565026806130721758672048541797157896401554276881090475899564605488362989140226580026134158039480357971019004151547655018391755772677897148793477372747525743898158705040701968215101218826088040084551332795162841280679678965570163917067779841529149397403158167896865448841319046368332179115059107813898261026271979696826411179918656038993895418928488851750122504754778999508544083983800725431468842988412616042682248823097788556495765424017114510393927980290997604904428832198976751320535115230545666467143795931915272680278210241540629795828828466355623580986725638200565215519951793551069127710538552661926903526081367717666435071213453983711357500975854405939558661737828297120544693182260401670308530911657973113259516101749193468250063285777004686987177255226525708428745733039859744230639751837209975339055095883623642814493247460522424051972825153787541962759327436278819283740253185668545040893929401040561666867664402868211607294830305236465560955351079987185041352121321534713770667681396211443891632403235741573773787908838267618458756361026435182951815392455211729022985278518025598478407179607904114472041476091765804302984501746867981277584971731733287305281134969591668387877072315968334322509070204019030503595891994666652037530271923764252552910347950343816357721698115464329245608951158732012675424975710520894362639501382962152214033621065422821876739580121286442788547491928976959315766891987305176388698461503354594898541849550251690616888419122873385522699976822609645007504500096116866129171093180282355042553653997166054753907348915189650027442328981181709248273610863801576007240601649547082331349361582435128299050405405333992577071321011503713898695076713447940748097845416328110406350804863393555238405735580863718763530261867971725608155328716436111474875107033512913923595452951407437943144900950809932872153235195999616750297532475931909938012968640379783553559071355708369947311923538531051736669154087312467233440702525006918026747725078958903448856673081487299464807786497709361969389290891718228134002845552513917355978456150353144603409441211512001738697261466786933733154341007587514908295822756919350542184106448264951943804240543255345965248373785310657979037977505031436474651422484768831323479762673689855474944277949916560108528257618964374464656819789319422077536824661110427671936481836360534108748971066866318805026555929568123959680449295166615409802610781691689418764353363449482900125929366840591370059526914934421861891742142561071896846626335874414976973921566392767687720145153302241853125308442727245771161505550519076276250016522166274796257424425420546785767478190959486500575711016264847833741198041625940813327229905891486422127968042984725356237202887830051788539737909455265135144073130049869453403245984236934627060242579432563660640597549471239092372458126154582526667304702319359866523378856244229188278436440434628094888288712101968642736370461639297485616780079779959696843367730352483047478240669928277140069031660709951473154191919911453182543906294573298686613524886500574780251977607442660798300291573030523199052185718628543687577860915726925232573171665625274275808460620177046433101212443409281314659760221360416223031167750085960128475289259463348312408766740128170543067985261868949895004918275008304998926472034986965363326210919830621495095877228260815566702155693484634079776879525038204442326697479264829899016938511552124688935873289878336267819361764023681714606495185508780596635354698788205094762016350757090024201498400967867845405354130050482404996646978558002628931826518708714613909521454987992300431779500489569529280112698632533646737179519363094399609176354568799002814515169743717518330632232942199132137614506411391269837128970829395360832883050256072727563548374205497856659895469089938558918441085605111510354367477810778500572718180809661542709143010161515013086522842238721618109043183163796046431523184434669799904865336375319295967726080853457652274714047941973192220960296582500937408249714373040087376988068797038047223488825819819025644086847749767508999164153502160223967816357097637814023962825054332801828798160046910336602415904504637333597488119998663995617171089911809851197616486499233594328274275983382931099806461605360243604040848379619072542165869409486682092396143083817303621520642297839982533698027039931804024928814430649614747600087654305571672697259114631990688823893005380061568007730984416061355843701277573463708822073792921409548717956947854414951731561828176343929570234710460088230637509877521391223419548471196982303169544468045517922669260631327498272520906329003279972932906827204647650366969765227673645419031639887433042226322021325368176044169612053532174352764937901877252263626883107879345194133825996368795020985033021472307603375442346871647223795507794130304865403488955400210765171630884759704098331306109510294140865574071074640401937347718815339902047036749084359309086354777210564861918603858715882024476138160390378532660185842568914109194464566162667753712365992832481865739251429498555141512136758288423285957759412684479036912662015308418041737698963759002546999454131659341985624780714434977201991702665380714107259910648709897259362243300706760476097690456341576573395549588448948093604077155688747288451838106069038026528318275560395905381507241627615047252487759578650784894547389096573312763852962664517004459626327934637721151028545472312880039058405918498833810711366073657536918428084655898982349219315205257478363855266205400703561310260405145079325925798227406012199249391735122145336707913500607486561657301854049217477162051678486507913573336334257685988361252720250944019430674728667983441293018131344299088234006652915385763779110955708000600143579956351811596764725075668367726052352939773016348235753572874236648294604770429166438403558846422370760111774821079625901180265548868995181239470625954254584491340203400196442965370643088660925268811549596291166168612036195319253262662271108142149856132646467211954801142455133946382385908540917878668826947602781853283155445565265933912487885639504644196022475186011405239187543742526581685003052301877096152411653980646785444273124462179491306502631062903402737260479940181929954454297256377507172705659271779285537195547433852182309492703218343678206382655341157162788603990157495208065443409462446634653253581574814022471260618973060860559065082163068709634119751925774318683671722139063093061019303182326666420628155129647685313861018672921889347039342072245556791239578260248978371473556820782675452142687314252252601795889759116238720807580527221031327444754083319215135934526961397220564699247718289310588394769170851420631557192703636345039529604362885088555160008371973526383838996789184600327073682083234847108471706160879195227388252347506380811606090840124222431476103563328940609282430125462013806032608121942876847907192546246309055749298781661271916548229644317263587524548607563020667656942355342774617635549231817456159185668061686428714964129290560130053913469569829490891003991259088290348791943368696942620662946948514931472688923571615032405542263391673583102728579723061998175868700492227418629077079508809336215346303842967525604369606110193842723883107587771653594778681499030978765900869583480043137176832954871752604714113064847270887246697164585218774442100900090916189819413456305028950484575822161887397443918833085509908566008543102796375247476265353031558684515120283396640547496946343986288291957510384781539068343717740714095628337554413567955424664601335663617305811711646062717854078898495334329100315985673932305693426085376230981047171826940937686754301837015557540822371538037838383342702379535934403549452173960327095407712107332936507766465603712364707109272580867897181182493799540477008369348889220963814281561595610931815183701135104790176383595168144627670903450457460997444500166918675661035889313483800512736411157304599205955471122443903196476642761038164285918037488354360663299436899730090925177601162043761411616688128178292382311221745850238080733727204908880095181889576314103157447684338100457385008523652069340710078955916549813037292944462306371284357984809871964143085146878525033128989319500645722582281175483887671061073178169281242483613796475692482076321356427357261609825142445262515952514875273805633150964052552659776922077806644338105562443538136258941809788015677378951310313157361136026047890761945591820289365770116416881703644242694283057457471567494391573593353763114830246668754727566653059819746822346578699972291792416156043557665183382167059157867799311835820189855730344883681934418305987021880502259192818047775223884407167894780414701414651073580452021499197980812095692195622632313741870979731320870864552236740416185590793816745658234353037283309503729022429802768451559528656923189798000383061378732434546500582722712325031420712488100290697226311129067629080951145758060270806092801504406139446350643069742785469477459876821004441453438033759717384777232052065301037861326418823586036569054773343070911759152582503029410738914441818378779490613137536794654893375260322906277631983337976816641721083140551864133302224787118511817036598365960493964571491686005656771360533192423185262166760222073368844844409234470948568027905894191829969467724456269443308241243846160408284006424867072583661011433404214473683453638496544701067827313169538435919120440283949541956874453676459875488726170687163109591315801609722382049772577307454562979127906177531663252857205858766376754282917933549923678212008601904369428956102301731743150352204665675088491593025926618816581008701658499456495586855628208747248318351516339189292646558880593601275151838235485893426165223086697314511412035659916934103076974774451947043836739600076578628245472064617380804602903639144493859012422380173377038154675297645596518492676039300171943042511794045679862114630138402371099347243455794730048929825402680821621522346560274258486595687074510352794291633405915025075992398611224340312056999780516223878772230396359709132856830486160362127579561601328561866388146004722200580017580282279272167842720649966956840905752590774886105493806116954293569077377792821084159737469613143291808510446953973485067590503662391722108732333169909603363771705474725026941732982890400239372879549386540463828596742216318201530139629734398479588628632934746650690284066719018081265539973675916799759010867483920062877888531102781695087545740384607594616919584610655963327283485609570305572502494416337066573150237126843581984154103154401008430380631442183776750349813408169325201240813452285974626715177152223063741359255747513535160669108359443999692315898156732033027129284241219651936303734407981204656795322986357374589031654007016472204989445629050395873788912680565516464274460174738175296313458739390484560414203426465560422112239134631023161290836446988901247285192778589195228773637440432659264672239982186452797664826673070168802722052338600372842903155828454593854349099449420750911108532138744823216151007808922516285123275724355101999038195993350032641446053470357293073912578481757987468353429629749652545426864234949270336399427519354240001973125098882419600095766257217621860474573769577649582201796258392376391717855799468922496750179251915218219624653575570564228220399546682648329822996167217080156801080799777126517156274295763666959661983507435667132218383358509536665806605597148376773866922551603463644386269977295750658468929599809168949981898588529537874489519527097766262684177088590284321676352132630838812766335363319004134332844347630067982023716933653652880580156390360562722752187272454764258840995216482554453662083811789117725225682611478014242896970967121967502094421226279437073328703410646312100557376727450271638975234111426287828736758358819056742163061523416789476056879277154789714326222041069587947186435439940738639948986836168919377836648327137363654676901173760246643082285362494712605173293777247276797635865806019396287718060679122426813922872134061694882029506831654589707623668302556167559477498715183426989208952182644710514911419441192277010977616645850068963849426165593473112961064282379048216056210094265076173838082479030510998790719611852832556787472942907151041468948104916751035295897242381802288151276582257190705537652455285511598636421244284176256230139538669970308943645907600684938040875210854159851278070333207779865635907968462191534944587677170063778573171211036517486371634098385626541555573292664616402279791195975248525300376741774056125700303625811704838385391207273191845064713669122576415213769896260940351804147432053600369234179035440735703058314741623452840188940808983125191307741823338981880316339159565954543405777784331681162551898060409183018907512170192983622897099598983405484962284289398469847938668614293324543983592637036699355184231661615244505980576745765335552338715678211466689996845227042954589710922163652573965950289645637766038988037941517917867910675199009966139206238732318786758420544279396366759104126821843375015743069045967947046685602358283919759975285865384338189120042853787549302768972168199113340697282255535300044743958830079799736518459131437946494086272149669719100359399974735262764126125995350902609540048669398955899487421379590802893196914845826873123710180229775301190684280440780938156598081694611679374425663244656799606363751546304833112722231812338371779800439731087402647536582575657351059978314264831879619843765495877803685261751835391844920488198629786329743136948511780579298636452193232481339393090754566368038513630619718033957979522539508697432546502659123585049283028832934489284591373621624852528877442891851104093746333590660233239711922814450735588373324057814862662207486215513375036775585494138678352928273109003823116855374520901095101174796663003330352534143230024288248051396631446632656081582045216883922312025671065388459503224002320453633895521539919011035217362720909565500846486605368975498478995875596103167696587161281951919668893326641203784750417081752273735270989343717167642329956935697166213782736138899530515711822960896394055380431939398453970864418654291655853168697537052760701061488025700785387150835779480952313152747735711713643356413242974208137266896149109564214803567792270566625834289773407718710649866150447478726164249976671481383053947984958938064202886667951943482750168192023591633247099185942520392818083953020434979919361853380201407072481627304313418985942503858404365993281651941497377286729589582881907490040331593436076189609669494800067194371424058105327517721952474344983414191979918179909864631583246021516575531754156198940698289315745851842783390581029411600498699307751428513021286202539508732388779357409781288187000829944831476678183644656510024467827445695591845768068704978044824105799710771577579093525803824227377612436908709875189149049904225568041463131309240101049368241449253427992201346380538342369643767428862595140146178201810734100565466708236854312816339049676558789901487477972479202502227218169405159042170892104287552188658308608452708423928652597536146290037780167001654671681605343292907573031466562485809639550080023347676187068086526878722783177420214068980703410506200235273632267291964034093571225623659496432076928058165514428643204955256838543079254299909353199329432966018220787933122323225928276556048763399988478426451731890365879756498207607478270258861409976050788036706732268192473513646356758611212953074644777149423343867876705824452296605797007134458987594126654609414211447540007211790607458330686866231309155780005966522736183536340439991445294960728379007338249976020630448806064574892740547730693971337007962746135534442514745423654662752252624869916077111131569725392943756732215758704952417232428206555322808868670153681482911738542735797154157943689491063759749151524510096986573825654899585216747260540468342338610760823605782941948009334370046866568258579827323875158302566720152604684361412652956519894291184887986819088277339147282063794512260294515707367105637720023427811802621502691790400488001808901847311751199425460594416773315777951735444490965752131026306836047140331442314298077895617051256930051804287472368435536402764392777908638966566390166776625678575354239947427919442544664643315554138265543388487778859972063679660692327601733858843763144148113561693030468420017434061395220072403658812798249143261731617813894970955038369479594617979829257740992171922783223006387384996138434398468502234780438733784470928703890536420557474836284616809363650973790900204118525835525201575239280826462555785658190226958376345342663420946214426672453987171047721482128157607275305173330963455909323664528978019175132987747952929099598069790148515839540444283988381797511245355548426126784217797728268989735007954505834273726937288386902125284843370917479603207479554080911491866208687184899550445210616155437083299502854903659617362726552868081324793106686855857401668022408227992433394360936223390321499357262507480617409173636062365464458476384647869520547719533384203403990244761056010612777546471464177412625548519830144627405538601855708359981544891286863480720710061787059669365218674805943569985859699554089329219507269337550235821561424994538234781138316591662683103065194730233419384164076823699357668723462219641322516076261161976034708844046473083172682611277723613381938490606534404043904909864126903479263503943531836741051762565704797064478004684323069430241749029731181951132935746854550484711078742905499870600373983113761544808189067620753424526993443755719446665453524088287267537759197074526286322840219629557247932987132852479994638938924943286917770190128914220188747760484939855471168524810559991574441551507431214406120333762869533792439547155394213121021954430556748370425907553004950664994802614794524739012802842646689229455664958621308118913500279654910344806150170407268010067948926855360944990373928383520627992820181576427054962997401900837493444950600754365525758905546552402103412862124809003162941975876195941956592556732874237856112669741771367104424821916671499611728903944393665340294226514575682907490402153401026923964977275904729573320027982816062130523130658731513076913832317193626664465502290735017347656293033318520949298475227462534564256702254695786484819977513326393221579478212493307051107367474918016345667888810782101151826314878755138027101379868751299375133303843885631415175908928986956197561123025310875057188962535763225834275763348421016668109884514141469311719314272028007223449941999003964948245457520704922091620614222912795322688239046498239081592961111003756999529251250673688233852648213896986384052437049402152187547825163347082430303521036927849762517317825860862215614519165573478940019558704784741658847364803865995119651409542615026615147651220820245816010801218275982577477652393859159165067449846149161165153821266726927461290533753163055654440793427876550267301214578324885948736899073512166118397877342715872870912311383472485146035661382188014840560716074652441118841800734067898587159273982452147328317214621907330492060817440914125388918087968538960627860118193099489240811702350413554126823863744341209267781729790694714759018264824761112414556423937732224538665992861551475342773370683344173073150805440138894084087253197595538897613986400165639906934600670780501058567196636796167140097031535132386972899001749862948883362389858632127176571330142071330179992326381982094042993377790345261665892577931395405145369730429462079488033141099249907113241694504241391265397274078984953073730364134893688060340009640631540701820289244667315059736321311926231179142794944897281477264038321021720718017561601025111179022163703476297572233435788863537030535008357679180120653016668316780269873860755423748298548246360981608957670421903145684942967286646362305101773132268579232832164818921732941553151386988781837232271364011755881332524294135348699384658137175857614330952147617551708342432434174779579226338663454959438736807839569911987059388085500837507984051126658973018149321061950769007587519836861526164087252594820126991923916722273718430385263107266000047367872474915828601694439920041571102706081507270147619679971490141639274282889578424398001497985658130305740620028554097382687819891158955487586486645709231721825870342960508203415938806006561845735081804032347750084214100574577342802985404049555529215986404933246481040773076611691605586804857302606467764258503301836174306413323887707999698641372275526317649662882467901094531117120243890323410259937511584651917675138077575448307953064925086002835629697045016137935696266759775923436166369375035368699454550392874449940328328128905560530091416446608691247256021455381248285307613556149618444364923014290938289373215312818797541139219415606631622784836152140668972661027123715779503062132916001988806369127647416567067485490795342762338253943990022498972883660263920518704790601584084302914787302246651371144395418253441269003331181914268070735159284180415100555199146564934872796969351992963117195821262627236458009708099166752820365818699111948365866102758375863322993225541477479210421324166848264953111826527351008031659958888814809945737293785681411438021523876706455063233067233939551964260397443829874822322662036352861302543796600943104500158604854027036789711934695579989189112302233381602302236277726084846296189550730850698061500281436425336666311433321645213882557346329366870956708432252564333895997812402164189946978348320376011613913855499933990786652305860332060641949298931012423081105800169745975038516887112037747631577311831360002742502722451570906304496369230938382329175076469684003556425503797106891999812319602533733677437970687713814747552190142928586781724044248049323750330957002929126630316970587409214456472022710796484778657310660832173093768033821742156446602190335203981531618935787083561603302255162155107179460621892674335641960083663483835896703409115513087820138723494714321400450513941428998350576038799343355677628023346565854351219361896876831439866735726040869511136649881229957801618882834124004126142251475184552502502640896823664946401177803776799157180146386554733265278569418005501363433953502870836220605121839418516239153709790768084909674194289061134979961034672077354959593868862427986411437928435620575955500144308051267664432183688321434583708549082240014585748228606859593502657405750939203135881722442164955416889785558265198046245527898343289578416968890756237467281044803018524217706136533236073856228166664597654076844715963930782091017090763377917711485205493367936868430832404126789220929930411890501756484917499452393770674524578019171841679541825554377930299249277892416277257788147974770446005423669346157135208417428211847353652367573702352791459837645712257646122605628127852169580892808988394594406165340521932514843306105322700231133680378433377389724881307874325614952744243584753011150345103737688223837573804282007358586938044331529253129961025096113761670187568525921208929131354473196308440066835155160913925692912175784379179004808848023029304392630921342768601226558630456913133560978156776098711809238440656353136182676923761613389237802972720736243967239854144480757286813436768000573823963610796223140429490728058551444771338682314499547929338131259971996894072233847404542592316639781608209399269744676323921370773991899853301483814622364299493902073285072098040905300059160091641710175605409814301906444379905831277826625762288108104414704097708248077905168225857235732665234414956169007985520848841886027352780861218049418060017941147110410688703738674378147161236141950474056521041002268987858525470689031657094677131822113205505046579701869337769278257145248837213394613987859786320048011792814546859096532616616068403160077901584946840224344163938313618742275417712170336151163782359059685168880561304838542087505126933144171705880517278127917564053282929427357971823360842784676292324980318169828654166132873909074116734612367109059236155113860447246378721244612580406931724769152219217409096880209008801535633471775664392125733993165330324425899852598966724744126503608416484160724482125980550754851232313331300621490042708542735985913041306918279258584509440150719217604794274047740253314305451367710311947544521321732225875550489799267468541529538871443696399406391099267018219539890685186755868574434469213792094590683677929528246795437302263472495359466300235998990248299853826140395410812427393530207575128774273992824866921285637240069184859771126480352376025469714309316636539718514623865421671429236191647402172547787238964043145364190541101514371773797752463632741619269990461595895793940622986041489302535678633503526382069821487003578061101552210224486633247184367035502326672749787730470216165019711937442505629639916559369593557640005236360445141148916155147776301876302136068825296274460238077523189646894043033182148655637014692476427395401909403584437251915352134557610698046469739424511797999048754951422010043090235713636892619493763602673645872492900162675597083797995647487354531686531900176427222751039446099641439322672532108666047912598938351926694497553568096931962642014042788365702610390456105151611792018698900673027082384103280213487456720062839744828713298223957579105420819286308176631987048287388639069922461848323992902685392499812367091421613488781501234093387999776097433615750910992585468475923085725368613605356762146929424264323906626708602846163376051573599050869800314239735368928435294958099434465414316189806451480849292695749412903363373410480943579407321266012450796613789442208485840536446021616517885568969302685188950832476793300404851688934411125834396590422211152736276278672366665845757559585409486248261694480201791748223085835007862255216359325125768382924978090431102048708975715033330963651576804501966025215527080352103848176167004443740572131294252820989545456276344353575741673638980108310579931697917916718271145837435222026387771805250290791645414791173616253155840768495583288190293564201219633684854080865928095131505012602919562576032932512847250469881908146475324342363863860247943921015193235101390117789997483527186469346024554247028375300033725403910085997650987642832802908445662021678362267272292737780213652404028817217012490974899454430826861772239385250883760749742195942655217301733355851389407457348144161511380845358039740277795072051893487170722955427683655826706766313911972211811528466502223383490906676554168336907959409404576472940901354356409277969379842065738891481990225399022315913388145851487225126560927576795873759207013915029216513720851137197522734365458411622066281660256333632074449918511469174455062297146086578736313585389023662557285424516018080487167823688885575325066254262367702604215835160174851981885460860036597606743233346410471991027562358645341748631726556391320606407754779439671383653877377610828300019937359760370467245737880967939894493795829602910746901609451288456550071458091887879542641820145369659962842686882363495879277007025298960996798975941955735253914237782443302746708282008722602053415292735847582937522487377937899136764642153727843553986244015856488692101644781661602962113570056638347990334049623875941092886778920270077504951511405782565295015024484968204744379710872943108541684540513016310902267112951959140520827546866418137305837933236150599142045255880213558474751516267815309465541240524091663857551298894834797423322854504140527354235070335984964593699534959698554244978249586929179182415068053002553370412778703476446244329205906832901886692400222391918714603175399666877477960121790688623311002908668305431787009355066944389131913333586368037447530664502418437136030852288582121720231274167009740351431532131803978033680228154223490183737494117973254478594157962104378787072154814091725163615415163381388912588517924237727229603497305533840942889918919161186249580560073570527227874940321250645426206304469470804277945973817146810395192821550688079136701210109944220737024613687196031491162370967939354636396448139025711768057799751751298979667073292674886430097398814873780767363792886767781170520534367705731566895899181530825761606591843760505051704242093231358724816618683821026679970982966436224723644898648976857100173643547336955619347638598187756855912376232580849341570570863450733443976604780386678461711520325115528237161469200634713570383377229877321365028868868859434051205798386937002783312365427450532283462669786446920780944052138528653384627970748017872477988461146015077617116261800781557915472305214759943058006652042710117125674185860274188801377931279938153727692612114066810156521441903567333926116697140453812010040811760123270513163743154487571768761575554916236601762880220601068655524141619314312671535587154866747899398685510873576261006923021359580838145290642217792987748784161516349497309700794368305080955621264592795333690631936594413261117944256602433064619312002953123619348034504503004315096798588111896950537335671086336886944665564112662287921812114121425167348136472449021275252555647623248505638391391630760976364990288930588053406631352470996993362568102360392264043588787550723319888417590521211390376609272658409023873553418516426444865247805763826160023858280693148922231457758783791564902227590699346481624734399733206013058796068136378152964615963260698744961105368384203105364183675373594176373955988088591188920114871545460924735613515979992999722298041707112256996310945945097765566409972722824015293663094891067963296735505830412258608050740410916678539569261234499102819759563955711753011823480304181029089719655278245770283085321733741593938595853203645590564229716679900322284081259569032886928291260139267587858284765599075828016611120063145411315144108875767081854894287737618991537664505164279985451077400771946398046265077776614053524831090497899859510873112620613018757108643735744708366215377470972660188656210681516328000908086198554303597948479869789466434027029290899143432223920333487108261968698934611177160561910681226015874410833093070377506876977485840324132474643763087889666151972556180371472590029550718424245405129246729039791532535999005557334600111693557020225722442772950263840538309433999383388018839553821540371447394465152512354603526742382254148328248990134023054550811390236768038649723899924257800315803725555410178461863478690646045865826036072306952576113184134225274786464852363324759102670562466350802553058142201552282050989197818420425028259521880098846231828512448393059455162005455907776121981297954040150653985341579053629101777939776957892084510979265382905626736402636703151957650493344879513766262192237185642999150828898080904189181015450813145034385734032579549707819385285699926238835221520814478940626889936085239827537174490903769904145555260249190126341431327373827075950390882531223536876389814182564965563294518709637484074360669912550026080424160562533591856230955376566866124027875883101021495284600804805028045254063691285010599912421270508133194975917146762267305044225075915290251742774636494555052325186322411388406191257012917881384181566918237215400893603475101448554254698937834239606460813666829750019379115061709452680984785152862123171377897417492087541064556959508967969794980679770961683057941674310519254486327358885118436597143583348756027405400165571178309126113117314169066606067613797690123141099672013123730329707678988740099317309687380126740538923612230370779727025191340850390101739924877352408881040807749924412635346413181858792480760553268122881584307471326768283097203149049868884456187976015468233715478415429742230166504759393312132256510189175368566338139736836336126010908419590215582111816677413843969205870515074254852744810154541079359513596653630049188769523677579147319184225806802539818418929888943038224766186405856591859943091324575886587044653095332668532261321209825839180538360814144791320319699276037194760191286674308615217243049852806380129834255379486287824758850820609389214668693729881191560115633701248675404205911464930888219050248857645752083363921499441937170268576222251074166230901665867067714568862793343153513505688216165112807318529333124070912343832502302341169501745502360505475824093175657701604884577017762183184615567978427541088499501610912720817913532406784267161792013428902861583277304794830971705537485109380418091491750245433432217445924133037928381694330975012918544596923388733288616144238100112755828623259628572648121538348900698511503485369544461542161283241700533583180520082915722904696365553178152398468725451306350506984981006205514844020769539324155096762680887603572463913955278222246439122592651921288446961107463586148252820017348957533954255019475442643148903233373926763409115527189768429887783617346613535388507656327107814312435018965109238453660236940276060642119384227665755210663671879603217527184404651560427289869560206997012906367847161654793068868305846508082886614111979138822898112498261434559408961813509226857611474609406147937240008842153535862052780125014270055274468359151840373309373580494342483940467505708347927948338133276237937844629209323999417593374917899786484958148818865149169302451512835579818112344900827168644548306546633975256079615935830821400021951611342337058359111545217293721664061708131602078213341260356852013161345136871600980378712556766143923146458085652084039744217352744813741215277475202259244561520365608268890193913957991844109971588312780020898275935898106482117936157951837937026741451400902833064466209280549839169261068975151083963132117128513257434964510681479694782619701483204392206140109523453209269311762298139422044308117317394338867965739135764377642819353621467837436136161591167926578700137748127848510041447845416464568496606699139509524527949914769441031612575776863713634644477006787131066832417871556281779122339077841275184193161188155887229676749605752053192594847679397486414128879475647133049543555044790277128690095643357913405127375570391806822344718167939329121448449553897728696601037841520390662890781218240141299368590465146519209198605347788576842696538459445700169758422531241268031418456268722581132040056433413524302102739213788415250475704533878002467378571470021087314693254557923134757243640544448132093266582986850659125571745568328831440322798049274104403921761438405750750288608423536966715191668510428001748971774811216784160854454400190449242294333666338347684438072624307319019363571067447363413698467328522605570126450123348367412135721830146848071241856625742852208909104583727386227300781566668914250733456373259567253354316171586533339843321723688126003809020585719930855573100508771533737446465211874481748868710652311198691114058503492239156755462142467550498676710264926176510110766876596258810039163948397811986615585196216487695936398904500383258041054420595482859955239065758108017936807080830518996468540836412752905182813744878769639548306385089756146421874889271294890398025623046812175145502330254086076115859321603465240763923593699949180470780496764486889980902123735780457040380820770357387588525976042434608851075199334470112741787878845674656640471901619633546770714090590826954225196409446319547658653032104723804625249971910690110456227579220926904132753699634145768795242244563973018311291451151322757841320376225862458224784696669785947914981610522628786944136373683125108310682898766123782697506343047263278453719024447970975017396831214493357290791648779915089163278018852504558488782722376705263811803792477835540018117452957747339714012352011459901984753358434861297092928529424139865507522507808919352104173963493428604871342370429572757862549365917805401652536330410692033704691093097588782938291296447890613200063096560747882082122140978472301680600835812336957051454650181292694364578357815608503303392466039553797630836137289498678842851139853615593352782103740733076818433040893624460576706096188294529171362940967592507631348636606011346115980434147450705511490716640635688739020690279453438236930531133440901381392849163507484449076828386687476663619303412376248380175840467851210698290605196112357188811150723607303158506622574566366740720668999061320627793994112805759798332878792144188725498543014546662945079670707688135022230580562225942983096887732856788971494623888272184647618153045844390967248232348259587963698908456664795754200195991919240707615823002328977439748112690476546256873684352229063217889227643289360535947903046811114130586348244566489159211382258867880972564351646404364328416076247766114349880319792230537889671148058968061594279189647401954989466232962162567264739015818692956765601444248501821713300527995551312539849919933907083138030214072556753022600033565715934283182650908979350869698950542635843046765145668997627989606295925119763672907762567862769469947280606094290314917493590511523235698715397127866718077578671910380368991445381484562682604003456798248689847811138328054940490519768008320299631757043011485087384048591850157264392187414592464617404735275250506783992273121600117160338604710710015235631159734711153198198710616109850375758965576728904060387168114313084172893710817412764581206119054145955378853200366615264923610030157044627231777788649806700723598889528747481372190175074700005571108178930354895017924552067329003818814068686247959272205591627902292600592107710510448103392878991286820705448979977319695574374529708195463942431669050083984398993036790655541596099324867822475424361758944371791403787168166189093900243862038610001362193667280872414291108080291896093127526202667881902085595708111853836166128848729527875143202956393295910508349687029060692838441522579419764824996318479414814660898281725690484184326061946254276693688953540732363428302189694947766126078346328490315128061501009539164530614554234923393806214007779256337619373052025699319099789404390847443596972052065999017828537676265683558625452697455260991024576619614037537859594506363227095122489241931813728141668427013096050734578659047904243852086508154491350136491698639048125666610843702294730266721499164849610746803261583352580352858275799038584091667618877199539888680431991650866887781701439663176815592262016991396613153738021294160006906947533431677802632207226265881842757216055461439677336258462997385077307751473833315101468395296411397329672457933540390136107395245686243008096720460995545708974893048753897955544443791303790422346037768729236001386569593952300768091377768847789746299699489949016141866131552200856673695770822720338936659590666350594330040363762591189195691561626122704788696510356062748423100605472091437069471661080277379848576543481249822444235828329813543645124092220896643987201997945619030397327254617823136363375927622656301565813545578319730419339269008282952718252138855126583037630477490625995514925943105307478901043009876580816508144862607975129633326675259272351611791836777128931053144471668835182920514343609292493191180249366051791485330421043899773019267686085347768149502299280938065840007311767895491286098112311307002535600347898600653805084532572431553654422067661352337408211307834360326940015926958459588297845649462271300855594293344520727007718206398887404742186697709349647758173683580193168322111365547392288184271373843690526638607662451284299368435082612881367358536293873792369928837047900484722240370919885912556341130849457067599032002751632513926694249485692320904596897775676762684224768120033279577059394613185252356456291805905295974791266162882381429824622654141067246487216174351317397697122228010100668178786776119825961537643641828573481088089988571570279722274734750248439022607880448075724807701621064670166965100202654371260046641935546165838945950143502160890185703558173661823437491622669077311800121188299737319891006060966841193266075165452741829459541189277264192546108246351931647783837078295218389645376236304858042774417907169146356546201215125418664885396161542055152375000426794253417764590821513675258479774465114750438460596325820468809667795709044645884673847481638045635188183210386594798204376334738389017759714236223057776395541011294523488098341476645559342209402059733452337956309441446698222457026367119493286653989491344225517746402732596722993581333110831711807234044326813737231209669052411856734897392234152750707954137453460386506786693396236535556479102508529284294227710593056660625152290924148057080971159783458351173168204129645967070633303569271821496292272073250126955216172649821895790908865085382490848904421755530946832055636316431893917626269931034289485184392539670922412565933079102365485294162132200251193795272480340133135247014182195618419055761030190199521647459734401211601239235679307823190770288415814605647291481745105388060109787505925537152356112290181284710137917215124667428500061818271276125025241876177485994084521492727902567005925854431027704636911098800554312457229683836980470864041706010966962231877065395275783874454229129966623016408054769705821417128636329650130416501278156397799631957412627634011130135082721772287129164002237230234809031485343677016544959380750634285293053131127965945266651960426350406454862543383772209428482543536823186182982713182489884498260285705690699045790998144649193654563259496570044689011049923939218088155626191834404362264965506449848521612498442375928443642612004256628602157801140467879662339228190804577624109076487087406157070486658398144845855803277997327929143195789110373530019873110486895656281917362036703039179710646309906285483702836118486672219457621775034511770110458001291255925462680537427727378863726783016568351092332280649908459179620305691566806180826586923920561895421631986004793961133953226395999749526798801074576466538377400437463695133685671362553184054638475191646737948743270916620098057717103475575333102702706317395612448413745782734376330101853438497450236265733191742446567787499665000938706441886733491099877926005340862442833450486907338279348425305698737469497333364267191968992849534561045719338665222471536681145666596959735075972188416698767321649331898967182978657974612216573922404856900225324160367805329990925438960169901664189038843548375648056012628830409421321300206164540821986138099462721214327234457806819925823202851398237118926541234460723597174777907172041523181575194793527456442984630888846385381068621715274531612303165705848974316209831401326306699896632888532682145204083110738032052784669279984003137878996525635126885368435559620598057278951754498694219326972133205286374577983487319388899574634252048213337552584571056619586932031563299451502519194559691231437579991138301656117185508816658756751184338145761060365142858427872190232598107834593970738225147111878311540875777560020664124562293239116606733386480367086953749244898068000217666674827426925968686433731916548717750106343608307376281613984107392410037196754833838054369880310983922140260514297591221159148505938770679068701351029862207502287721123345624421024715163941251258954337788492834236361124473822814504596821452253550035968325337489186278678359443979041598043992124889848660795045011701169092519383155609441705397900600291315024253848282782826223304151370929502192196508374714697845805550615914539506437316401173317807741497557116733034632008408954066541694665746735785483133770133628948904397670025863002540635264006601631712883920305576358989492412827022489373848906764385339931878608019223108328847459816417701264089078551777830131616162049792779670521847212730327970738223860581986744668610994383049960437407323195784473254857416239738852016202384784256163512597161783106850156299135559874758848151014815490937380933394074455700842090155903853444962128368313687375166780513082594599771257467939781491953642874321122421579851584491669362551569370916855252644720786527971466476760328471332985501945689772758983450586004316822658631176606237201721007922216410188299330808409384014213759697185976897042759041500946595252763487628135867117352364964121058854934496645898651826545634382851159137631569519895230262881794959971545221250667461174394884433312659432286710965281109501693028351496524082850120190831078678067061851145740970787563117610746428835593915985421673115153096948758378955979586132649569817205284291038172721213138681565524428109871168862743968021885581515367531218374119972919471325465199144188500672036481975944167950887487934416759598361960010994838744709079104099785974656112459851972157558134628546189728615020774374529539536929655449012953097288963767713353842429715394179547179095580120134210175150931491664699052366350233024087218654727629639065723341455005903913890253699317155917179823065162679744711857951506573868504088229934804445549850597823297898617029498418376255258757455303112991914341109413088238114443068843062655305601658801408561023324210300218460588586954418502977463085858496130037238190325162225570729975710727306066072916922978033647048840958711228045188511908718588299514331534128549297173849768523136276076868494780364948299904475715771141080958058141208956059471668626290036145602625334863284986816039463372436667112964460292915746181117789169695839947080954788863503281129626899231110099889317815313946681882028368363373822281414974006917942192888817139116283910295684918233358930813360131488748366464224381776081007739183393749346933644748150564933649323157235306109385796839902153381449126925350768211098738352197507736653475499431740580563099143218212547336281359488317681489194306530426029773885492974570569448783077945878865062970895499843760181694031056909587141386804846359853684034105948341788438963179956468815791937174656705047441528027712541569401365862097760735632832966564135817028088013546326104892768731829917950379944446328158595181380144716817284996793061814177131912099236282922612543236071226270324572637946863533391758737446552006008819975294017572421299723542069630427857950608911113416534893431149175314953530067419744979017235181671568754163484949491289001739377451431928382431183263265079530371177806185851153508809998200482761808307209649636476943066172549186143700971387567940218696710148540307471561091358933165600167252126542502898612259306484105898847129649230941215144563947889999327145875969555737090855150648002321476443037232466147111552578583071024936898814562568786834745518893385181791667579054210421036349316257870476543126790661216644142285017446278477132740595579600648343288827864837043456066966456899746910373987712891593313271266247505582258634928427718355831641593667712218537642376222104779338956378722902509543014182257180331300148113377736941508488867501893156994849838936052666818012783912005801431596441910546663236810148207799356523056490420711364192200177189107935243234322761787712568251126481332974354926568682748715986654943041648468220593921673359485057849622807932422649812705271398407720995707236227009245067665680069149966555737866411877079767754867028786431817941521796178310655030287157272282250812017060713380339641841211253856248920130010782462165136989511064611133562443838185366273563783436921279354709230119655914915800561707258518503167289370411936374780625824298250726464801821523430268081486978164824349353456855843696378384153838051184406043696871666416514036129729992912630842812149152469877429332305214999981829046119471676727503742221367186614654042534463141660649871499001000660041544868437352208483059495953182872280520828676300361091734508632133033647289584176588755345227938480297724485711815574893561311524926772006362198369980664159549388683836411891430443767715498026544959061738265591178545999378510861446014967645550103653971251138583505085112442517772923814396233043724036032603181442991365750246012787514117944901305803452199992701148071712847770301254994886841867572975189214295652512486943983729047410363121899124217339550688778643130750024823361832738729697376598820053895902935486054979802320400472236873557411858132734337978931582039412878989728973298812553514507641535360519462112217000676321611195841029252568536561813138784086477147099724553013170761712163186600291464501378587854802096244703771373587720086738054108140042311418525803293267396324596914044834665722042880679280616029884043400536534009706581694636096660911110968789751801325224478246957913251892122653056085866541115373584912790254654369020869419871125588453729063224423222287139122012248769976837147645598526739225904997885514250047585260297929306159913444898341973583316070107516452301310796620382579278533125161760789984630103493496981494261055367836366022561213767081421091373531780682420175737470287189310207606953355721704357535177461573524838432101571399813798596607129664438314791296359275429627129436142685922138993054980645399144588692472767598544271527788443836760149912897358259961869729756588978741082189422337344547375227693199222635973520722998387368484349176841191020246627479579564349615012657433845758638834735832242535328142047826934473129971189346354502994681747128179298167439644524956655532311649920677163664580318205849626132234652606175413532444702007661807418914040158148560001030119994109595492321434406067634769713089513389171050503856336503545166431774489640061738861761193622676890576955693918707703942304940038440622614449572516631017080642923345170422426679607075404028551182398361531383751432493056398381877995594942545196756559181968690885283434886050828529642437578712929439366177362830136595872723080969468398938676366226456791132977469812675226595621009318322081754694778878755356188335083870248295346078597023609865656376722755704495258739871812593441903785275571333409842450127258596692434317689018966145404453679047136294238156127656824247864736176671770647002431119711090007474065945650315375044177982192306323700872039212085499569681061379189029961178936752146022386905665481382858280449537530160921422195940638787074787991194920898374091788534417523064715030278397979864517336625329511775105559014160459873338186887977858817291976604516353353556047648420520888811722831990044504284486852338334530105533929637308039738230604714104525470094899407601215247602819963846343554852932377161410869591950786873276075400085220065031871239272857835807010762542769655355964789450166013816295177908531139811092831583216931563867459747449584385282701658246192092219529134323496779345585613140207765996142546463288677356891785576835169608392864188830094883324700447958316931533832382377876344426323456301679513671047510469669001217777128065522453689371871451567394733440447280450959433090683667110655953338602938000999949010642769859623260401863733572846679531229683156358145420890540651226419162015504500430562136991850941034609601030543816694795964585804425194905110733387679946734471718615647723811737035654917628707589456035519195603962301157866323750234725054461073979402475184415558178087962822231972692984516683306919505079993357259165675557294585962182052650473353712351623662770479333289322136141858785972771685682725303734836891911847197133753088446777943274857148827821608844765700041403499921376794209627560883081509438030705666022764678117533361028187800710219794428777313146387857817205661409023041499923248268982477222109852189758140879763486146763606368674611966620347304608917277240045953051376938375381543486981101990651706961774052218247422657652138152740612699012706880875386408669901461740890540981877671880076124151967064152117653084325544261017536348281196837493395825742541244634247233586360777980960199745187758845459645895956779558869098404768259253477849930457883128541747079059795909431627722327844578918694214929451540174214623240300841907975296782445969183509474202123617940309048634960534054931299919496087957952586977170236680033862505764938088740994009589948109397983231108838769236490221499111120870639202892490698435333152727991330986335454324971441378059132240814960156485679843966464780280409057580889190254236606774500413415794312112501275232250148067232979652230488493751166084976116412777395311302041566848265531411348993243747890268935173904043294851610659785832253168204202834993641595980197343889883020994152152288611175126686173051956249367180053845637855129171848417841594797435580617856680758491080185805695567990185198397660693358224779136504562705766735170961550493338390452612404395517449136885115987454340932040102218982707539212403241042424451570052968378815749468441508011138612561164102477190903050040240662278945607061512108266146098662040425010583978098192019726759010749924884966139441184159734610382401178556739080566483321039073867083298691078093495828888707110651559651222542929154212923108071159723275797510859911398076844732639426419452063138217862260999160086752446265457028969067192282283045169111363652774517975842147102219099906257373383472726498678244401048998507631630668050267115944636293525120269424810854530602810627264236538250773340575475701704367039596467715959261029438313074897245505729085688496091346323165819468660587092144653716755655531962091865952628448253731353698162517351930115341581171353292035873164168839107994000677266031617527582917398395852606454113318985505747847121053505795649095931672167565624818782002769963734155880000867852567422461511406015760115910256449002264980039498403358091309140197877843650167960167465370287466062584346329708303725980494653589318912163976013193079476972058034710553111117215859219066231028099212084069283091906017370764654655683413207556315315006453462321007133584907633048328153458698497332599801187479664273140279381289961720524540674695271948079930396730194274036466594154400092799908634806622334906695224044652158992864203435098858422692019340575496840904812955522654754650713532842543496616084954788090727649930252702815067862810825243222979985391759845188868387004477101866772159439708514664612871148749531862180941719676843144666435175837688436786081446319641912566574047718699160915550910878919431253671945651261878486910876729910565595155159739659034383628124629118117760949411880105946336671039049777312004243578115790429823045072038322781246413671297959415082918378213212876890545963586369344879749784841123274921331663162812456388238288715648447883142417650147980187858215768793063001153788998014623690135803753306246148576074932567807682651045738059018831237617271889933790487113395588485234240255002352200613574914318259142479829367775490496399350755839668967578364316618369307625603528602940662803255416535431518013714821941772672244005268401996533334184004345525296592918502940131600651124395297874364222806977720437363717873457948420238745151249157913139411148608416429347958793681868609689684640858334131017858142710955416293375915178392341303110543328703526599993904966822112768158316511246866451167351378214345336650598328347443536290312393672084593164394941881138607974670134709640378534907149089842317891739783650654751982883367395714360000003439863363212091718954899055748693397700245632475954504411422582410783866837655467400137324322809113692670682805397549111166171102397437749479335174036135005397581475520834285772800986189401984375446435081498218360112577632447389452051636938585136484259964518361856989088721789764694721246807900330925083496645841656554261294195108847197209106605105540933731954888406444080280579549008076040034154662137669606444293774985897353625591959618552448187940317374508256072895120945456562159540405425814886929842786582357673195799285293120866275922366115137445767916063621675267440451221051052090834707443986137829082352772895849625656881972792768694795806100573787084121444815034797422312103295359297822377134077549545477791813823542607184617108389097825964406170543546968567030745411634244134486308676327949177682923093183221341455482591367202823284396549001805653203960795517074496039006696990334199278212696767771835209083959545341866777944872740383733381985235884202840150981579594685874537989503257362809837592216229258598599123843993575573285028613155970362934249814178056461615863415338635077223269996508860870999964899373049307170967888740149746147542880387421250689212155876692242387434701120990859082164073576380817386959755176083877600277517253037133445654852635661720197563001580049790223419586738061442401502436288957503206533690825756785507020555105572381878574650371086308158185862815883054564662297694803970618265491385181326737485227188267917919091354407852685476254126683398240534022469989966652573155637645862251862823092085424412805997628505488913098331761884983352975136073772030571342739638126588567405013841074788943393996603591853934198416322617654857376671943132840050626295140357877264680649549355746326408186979718630218760025813995719923601345374229758918285167511358171472625828596940798518571870075823122317068134867930884899275181661399609753105295773584618525865211893339375771859916335112163441037910451845019023066893064178977808158101360449495409665363660370075881004450265734935127707426742578608784898185628869980851665713320835842613381142623855420315774246613108873106318111989880289722849790551075148403702290580483052731884959994156606537314021296702220821915862905952604040620011815269664910068587592655660567562963361434230232810747488395040380984981860056164646099819257616235478710913832967563761506732550860683433720438748186791668975746563456020002562889601191100980453350423842063824039434163502977688802779835087481178298349417211674919425601608685332435385951152061809031241698182079314615062073826097180458265687043623935757495737332781578904386011378078508110273049446611821957450170106059384336519458628360682108585130499820420578458577175933849015564447305834515291412561679970569657426139901681932056241927977282026714297258700193234337873153939403115411184101414292741703537542003698760608765500109345299007034032401334806388514095769557147190364152027721127070187421548123931953220997506553022646844227700020589045922742423904937051507367764629844971682121994198274794049092601715727439368569721862936007387077810797440975556627807371228030350048829843919546433753355787895064018998685060281902452191177018634505171087023903398550540704454189088472042376499749035038518949505897971286631644699407490959473411581934618336692169573605081585080837952036335619947691937965065016808710250735070825260046821242820434367245824478859256555487861614478717581068572356895150707602217433511627331709472765932413249132702425519391509083601346239612335001086614623850633127072987745618984384288764099836164964775714638573247333226653894523588365972955159905187411779288608760239306160016168434070611663449248395156319152882728822831375458678269830696691220130954815935450754923554167766876455212545681242936427474153815692219503331560151614492247512488957534835926226263545406704767033866410025277276800886383266629488582740369655329362236090572479794734434077704284318507901973469071141230364111729224929307731939309795452877412451183953480382210373644697046967493042810911797232448615413264031578430955396671061468083815548947146733652483679138566431084747848676243012018489329109615281108087617422779131629345494425395422727309645057976122885347393189600810965202090151104579377602529543130188938184010247010134929317443562883578609861545691161669857388024973756940558138630581099823372565164920155443216861690537054630176154809626620800633059320775897175589925862195462096455464624399535391743228225433267174308492508396461328929584567927365409119947616225155964704061297047759818551878441419948614013153859322060745185909608884280218943358691959604936409651570327527570641500776261323783648149005245481413195989296398441371781402764122087644989688629798910870164270169014007825748311598976330612951195680427485317886333041169767175063822135213839779138443325644288490872919067009802496281560626258636942322658490628628035057282983101266919109637258378149363774960594515216932644945188292639525772348420077356021656909077097264985642831778694777804964343991762549216500608626285329471055602670413384500507827390640287529864161287496473708235188892189612641279553536442286955430551308700009878557534223100547153412810957024870812654319123261956462149376527526356402127388765103883255007364899937167183280028398832319373301564123277185395654932422977953016534830128490677845037490891749347389015649588574802194996722621185874361039774946338633057887487405540005440439344888192044102134790034598411927024921557026873700970995205391930979319495883265922171508324621942300185974396706491149559411733728199869021311629886680267446443489233020607003821262841723679627307191405008084085703978151998148822390059948911946474438682533745889962375133378280532928272016815977970066488394482446332210928320504045983008943565954267256879714918703447338237767914829203283196838105907715727191903042365315650957464549643425328069510396558733549803850995143463506175361480050195045201350200180281506933241918267855737764414097080945745624854867704904368368717590918057269794010465019484853146726642978667687697789291431128505043098192949736165944259471754765135205245072597538577958372797702972231435199958499522344049394502115428867244188717409524554771867484911475031801773304689909317974472957035192387686405544278134169807249382219749124257510162187439772902147704638010731470653154201300583810458905006764557332998149945854655105526374914354195867992595981412218735238407957416123372264063860431988936249867649693592569592128495906254446474331759999685163660305216426770428154681777589339252115538590526823311608302751194384823861552852465010329467297198112105314125898165100120742688143577590825227466863206188376830450921784582526239594189673003640808624233657620979111641766331328852352062487922978959456450333733139422384778582717195412347860434376165241568717943562570215636666680088531006728947033079540804583324192188488870712275670333173939262509073556164513677064199539111948881240659821685787131385056850623094155206877987539740658484250135205615103489821873770245063583314243624807432542464195984647411575625441010389671576677263196442524931941806472423789334668561083789808830313571333157729435664956078125304917594015895146954965223118559669048559467607968190167266634650186182955669893965019614544401768162810604465068448139561667220729261210164692339016793399632833013163850830967942792934551268435760356901970523138364640961311774904600772840862214747547653221505518116489887879087780918009050706040061220010051271575991225725282523378026809030528461581739558198122397010092017202251606352922464781615533532275453264543087093320924631855976580561717446840450048285353396546862678852330044967795580761661801833668792312510460809773895565488962815089519622093675058841609752282328250433712970186608193748968699961301486924694482420723632912367052542145464162968910442981633373266871675946715392611950649224725627254543274193495995569590243279097174392258098103601486364409101491734183079646345064833303404765711827040276868271418084574998493392039317445402616663674646668754385093967129918067471909885312710726724428584870694307099756567949198418996425748884764622030325637751112534060087936904565779272035205921345924272965206683338510673615276261016026647772485083344719891986802656197236420847504962661607797092906844757798251795569758235084371746103310387911789239441630112634077535773520558040066982523191225570519133631407211349723226549151062961739050617857127509403623146700931176133132018631158730886798239298009805089491510788371194099750375473674305745187265414016446924576792185753680363289139664155342066705623272936001177781498886100830877849571709880858667023104043242526785955562077310543072298032125941107957349146684680220501816192150766649106862033378713826058987655210423668198670177861672671972374156917880001690656659046965316154923604061891820982414006103779407166342002735828911994182647812782659666207030384795881442790246669264032799404016800137293477301530941805070587421153284642203006550763966756168318897005152026656649929417382840327305940740147117478464839241225676523593418554066440983706083636457657081801664285044258224551650808864421212113914352453935225522162483791737330329812349528984098613273709957407786789349311975204237925022851375880436791854547836416773151821457226504640800104202100410766027807729152555503218182387221708112766208665317651926458452495269685376314437998340336947124447247796973890514941120010934140073794061859447165516612674930799374705772930521750426383798367668159183589049652163726492960837147204067428996276720315410211504333742057182854090136325721437592054640471894328548696883599785122262130812989581571391597464534806099601555877223193450760315411663112963843719400333736013305526352571490454327925190794007111504785378036370897340146753465517470747096935814912797188187854376797751675927822300312945518595042883902735494672667647506072643698761394806879080593531793001711000214417701504495496412454361656210150919997862972495905809191825255486358703529320142005857057855419217730505342687533799076038746689684283402648733290888881745453047194740939258407362058242849349024756883352446212456101562729065130618520732925434179252299417447855189995098959999877410951464170076989305620163502192692653166599093238118295411937545448509428621839424186218067457128099385258842631930670182098008050900019819621758458932516877698594110522845465835679362969619219080897536813210484518784516230623911878024604050824909336069998094776253792973597037759066145994638578378211017122446355845171941670344732162722443265914858595797823752976323442911242311368603724514438765801271594060878788638511089680883165505046309006148832545452819908256238805872042843941834687865142541377686054291079721004271658
lib/std/compress/deflate/testdata/compress-gettysburg.txt deleted-29
......@@ -1,29 +0,0 @@
1 Four score and seven years ago our fathers brought forth on
2this continent, a new nation, conceived in Liberty, and dedicated
3to the proposition that all men are created equal.
4 Now we are engaged in a great Civil War, testing whether that
5nation, or any nation so conceived and so dedicated, can long
6endure.
7 We are met on a great battle-field of that war.
8 We have come to dedicate a portion of that field, as a final
9resting place for those who here gave their lives that that
10nation might live. It is altogether fitting and proper that
11we should do this.
12 But, in a larger sense, we can not dedicate - we can not
13consecrate - we can not hallow - this ground.
14 The brave men, living and dead, who struggled here, have
15consecrated it, far above our poor power to add or detract.
16The world will little note, nor long remember what we say here,
17but it can never forget what they did here.
18 It is for us the living, rather, to be dedicated here to the
19unfinished work which they who fought here have thus far so
20nobly advanced. It is rather for us to be here dedicated to
21the great task remaining before us - that from these honored
22dead we take increased devotion to that cause for which they
23gave the last full measure of devotion -
24 that we here highly resolve that these dead shall not have
25died in vain - that this nation, under God, shall have a new
26birth of freedom - and that government of the people, by the
27people, for the people, shall not perish from this earth.
28
29Abraham Lincoln, November 19, 1863, Gettysburg, Pennsylvania
lib/std/compress/deflate/testdata/compress-pi.txt deleted-1
......@@ -1 +0,0 @@
13.1415926535897932384626433832795028841971693993751058209749445923078164062862089986280348253421170679821480865132823066470938446095505822317253594081284811174502841027019385211055596446229489549303819644288109756659334461284756482337867831652712019091456485669234603486104543266482133936072602491412737245870066063155881748815209209628292540917153643678925903600113305305488204665213841469519415116094330572703657595919530921861173819326117931051185480744623799627495673518857527248912279381830119491298336733624406566430860213949463952247371907021798609437027705392171762931767523846748184676694051320005681271452635608277857713427577896091736371787214684409012249534301465495853710507922796892589235420199561121290219608640344181598136297747713099605187072113499999983729780499510597317328160963185950244594553469083026425223082533446850352619311881710100031378387528865875332083814206171776691473035982534904287554687311595628638823537875937519577818577805321712268066130019278766111959092164201989380952572010654858632788659361533818279682303019520353018529689957736225994138912497217752834791315155748572424541506959508295331168617278558890750983817546374649393192550604009277016711390098488240128583616035637076601047101819429555961989467678374494482553797747268471040475346462080466842590694912933136770289891521047521620569660240580381501935112533824300355876402474964732639141992726042699227967823547816360093417216412199245863150302861829745557067498385054945885869269956909272107975093029553211653449872027559602364806654991198818347977535663698074265425278625518184175746728909777727938000816470600161452491921732172147723501414419735685481613611573525521334757418494684385233239073941433345477624168625189835694855620992192221842725502542568876717904946016534668049886272327917860857843838279679766814541009538837863609506800642251252051173929848960841284886269456042419652850222106611863067442786220391949450471237137869609563643719172874677646575739624138908658326459958133904780275900994657640789512694683983525957098258226205224894077267194782684826014769909026401363944374553050682034962524517493996514314298091906592509372216964615157098583874105978859597729754989301617539284681382686838689427741559918559252459539594310499725246808459872736446958486538367362226260991246080512438843904512441365497627807977156914359977001296160894416948685558484063534220722258284886481584560285060168427394522674676788952521385225499546667278239864565961163548862305774564980355936345681743241125150760694794510965960940252288797108931456691368672287489405601015033086179286809208747609178249385890097149096759852613655497818931297848216829989487226588048575640142704775551323796414515237462343645428584447952658678210511413547357395231134271661021359695362314429524849371871101457654035902799344037420073105785390621983874478084784896833214457138687519435064302184531910484810053706146806749192781911979399520614196634287544406437451237181921799983910159195618146751426912397489409071864942319615679452080951465502252316038819301420937621378559566389377870830390697920773467221825625996615014215030680384477345492026054146659252014974428507325186660021324340881907104863317346496514539057962685610055081066587969981635747363840525714591028970641401109712062804390397595156771577004203378699360072305587631763594218731251471205329281918261861258673215791984148488291644706095752706957220917567116722910981690915280173506712748583222871835209353965725121083579151369882091444210067510334671103141267111369908658516398315019701651511685171437657618351556508849099898599823873455283316355076479185358932261854896321329330898570642046752590709154814165498594616371802709819943099244889575712828905923233260972997120844335732654893823911932597463667305836041428138830320382490375898524374417029132765618093773444030707469211201913020330380197621101100449293215160842444859637669838952286847831235526582131449576857262433441893039686426243410773226978028073189154411010446823252716201052652272111660396665573092547110557853763466820653109896526918620564769312570586356620185581007293606598764861179104533488503461136576867532494416680396265797877185560845529654126654085306143444318586769751456614068007002378776591344017127494704205622305389945613140711270004078547332699390814546646458807972708266830634328587856983052358089330657574067954571637752542021149557615814002501262285941302164715509792592309907965473761255176567513575178296664547791745011299614890304639947132962107340437518957359614589019389713111790429782856475032031986915140287080859904801094121472213179476477726224142548545403321571853061422881375850430633217518297986622371721591607716692547487389866549494501146540628433663937900397692656721463853067360965712091807638327166416274888800786925602902284721040317211860820419000422966171196377921337575114959501566049631862947265473642523081770367515906735023507283540567040386743513622224771589150495309844489333096340878076932599397805419341447377441842631298608099888687413260472156951623965864573021631598193195167353812974167729478672422924654366800980676928238280689964004824354037014163149658979409243237896907069779422362508221688957383798623001593776471651228935786015881617557829735233446042815126272037343146531977774160319906655418763979293344195215413418994854447345673831624993419131814809277771038638773431772075456545322077709212019051660962804909263601975988281613323166636528619326686336062735676303544776280350450777235547105859548702790814356240145171806246436267945612753181340783303362542327839449753824372058353114771199260638133467768796959703098339130771098704085913374641442822772634659470474587847787201927715280731767907707157213444730605700733492436931138350493163128404251219256517980694113528013147013047816437885185290928545201165839341965621349143415956258658655705526904965209858033850722426482939728584783163057777560688876446248246857926039535277348030480290058760758251047470916439613626760449256274204208320856611906254543372131535958450687724602901618766795240616342522577195429162991930645537799140373404328752628889639958794757291746426357455254079091451357111369410911939325191076020825202618798531887705842972591677813149699009019211697173727847684726860849003377024242916513005005168323364350389517029893922334517220138128069650117844087451960121228599371623130171144484640903890644954440061986907548516026327505298349187407866808818338510228334508504860825039302133219715518430635455007668282949304137765527939751754613953984683393638304746119966538581538420568533862186725233402830871123282789212507712629463229563989898935821167456270102183564622013496715188190973038119800497340723961036854066431939509790190699639552453005450580685501956730229219139339185680344903982059551002263535361920419947455385938102343955449597783779023742161727111723643435439478221818528624085140066604433258885698670543154706965747458550332323342107301545940516553790686627333799585115625784322988273723198987571415957811196358330059408730681216028764962867446047746491599505497374256269010490377819868359381465741268049256487985561453723478673303904688383436346553794986419270563872931748723320837601123029911367938627089438799362016295154133714248928307220126901475466847653576164773794675200490757155527819653621323926406160136358155907422020203187277605277219005561484255518792530343513984425322341576233610642506390497500865627109535919465897514131034822769306247435363256916078154781811528436679570611086153315044521274739245449454236828860613408414863776700961207151249140430272538607648236341433462351897576645216413767969031495019108575984423919862916421939949072362346468441173940326591840443780513338945257423995082965912285085558215725031071257012668302402929525220118726767562204154205161841634847565169998116141010029960783869092916030288400269104140792886215078424516709087000699282120660418371806535567252532567532861291042487761825829765157959847035622262934860034158722980534989650226291748788202734209222245339856264766914905562842503912757710284027998066365825488926488025456610172967026640765590429099456815065265305371829412703369313785178609040708667114965583434347693385781711386455873678123014587687126603489139095620099393610310291616152881384379099042317473363948045759314931405297634757481193567091101377517210080315590248530906692037671922033229094334676851422144773793937517034436619910403375111735471918550464490263655128162288244625759163330391072253837421821408835086573917715096828874782656995995744906617583441375223970968340800535598491754173818839994469748676265516582765848358845314277568790029095170283529716344562129640435231176006651012412006597558512761785838292041974844236080071930457618932349229279650198751872127267507981255470958904556357921221033346697499235630254947802490114195212382815309114079073860251522742995818072471625916685451333123948049470791191532673430282441860414263639548000448002670496248201792896476697583183271314251702969234889627668440323260927524960357996469256504936818360900323809293459588970695365349406034021665443755890045632882250545255640564482465151875471196218443965825337543885690941130315095261793780029741207665147939425902989695946995565761218656196733786236256125216320862869222103274889218654364802296780705765615144632046927906821207388377814233562823608963208068222468012248261177185896381409183903673672220888321513755600372798394004152970028783076670944474560134556417254370906979396122571429894671543578468788614445812314593571984922528471605049221242470141214780573455105008019086996033027634787081081754501193071412233908663938339529425786905076431006383519834389341596131854347546495569781038293097164651438407007073604112373599843452251610507027056235266012764848308407611830130527932054274628654036036745328651057065874882256981579367897669742205750596834408697350201410206723585020072452256326513410559240190274216248439140359989535394590944070469120914093870012645600162374288021092764579310657922955249887275846101264836999892256959688159205600101655256375678566722796619885782794848855834397518744545512965634434803966420557982936804352202770984294232533022576341807039476994159791594530069752148293366555661567873640053666564165473217043903521329543529169414599041608753201868379370234888689479151071637852902345292440773659495630510074210871426134974595615138498713757047101787957310422969066670214498637464595280824369445789772330048764765241339075920434019634039114732023380715095222010682563427471646024335440051521266932493419673977041595683753555166730273900749729736354964533288869844061196496162773449518273695588220757355176651589855190986665393549481068873206859907540792342402300925900701731960362254756478940647548346647760411463233905651343306844953979070903023460461470961696886885014083470405460742958699138296682468185710318879065287036650832431974404771855678934823089431068287027228097362480939962706074726455399253994428081137369433887294063079261595995462624629707062594845569034711972996409089418059534393251236235508134949004364278527138315912568989295196427287573946914272534366941532361004537304881985517065941217352462589548730167600298865925786628561249665523533829428785425340483083307016537228563559152534784459818313411290019992059813522051173365856407826484942764411376393866924803118364453698589175442647399882284621844900877769776312795722672655562596282542765318300134070922334365779160128093179401718598599933849235495640057099558561134980252499066984233017350358044081168552653117099570899427328709258487894436460050410892266917835258707859512983441729535195378855345737426085902908176515578039059464087350612322611200937310804854852635722825768203416050484662775045003126200800799804925485346941469775164932709504934639382432227188515974054702148289711177792376122578873477188196825462981268685817050740272550263329044976277894423621674119186269439650671515779586756482399391760426017633870454990176143641204692182370764887834196896861181558158736062938603810171215855272668300823834046564758804051380801633638874216371406435495561868964112282140753302655100424104896783528588290243670904887118190909494533144218287661810310073547705498159680772009474696134360928614849417850171807793068108546900094458995279424398139213505586422196483491512639012803832001097738680662877923971801461343244572640097374257007359210031541508936793008169980536520276007277496745840028362405346037263416554259027601834840306811381855105979705664007509426087885735796037324514146786703688098806097164258497595138069309449401515422221943291302173912538355915031003330325111749156969174502714943315155885403922164097229101129035521815762823283182342548326111912800928252561902052630163911477247331485739107775874425387611746578671169414776421441111263583553871361011023267987756410246824032264834641766369806637857681349204530224081972785647198396308781543221166912246415911776732253264335686146186545222681268872684459684424161078540167681420808850280054143613146230821025941737562389942075713627516745731891894562835257044133543758575342698699472547031656613991999682628247270641336222178923903176085428943733935618891651250424404008952719837873864805847268954624388234375178852014395600571048119498842390606136957342315590796703461491434478863604103182350736502778590897578272731305048893989009923913503373250855982655867089242612429473670193907727130706869170926462548423240748550366080136046689511840093668609546325002145852930950000907151058236267293264537382104938724996699339424685516483261134146110680267446637334375340764294026682973865220935701626384648528514903629320199199688285171839536691345222444708045923966028171565515656661113598231122506289058549145097157553900243931535190902107119457300243880176615035270862602537881797519478061013715004489917210022201335013106016391541589578037117792775225978742891917915522417189585361680594741234193398420218745649256443462392531953135103311476394911995072858430658361935369329699289837914941939406085724863968836903265564364216644257607914710869984315733749648835292769328220762947282381537409961545598798259891093717126218283025848112389011968221429457667580718653806506487026133892822994972574530332838963818439447707794022843598834100358385423897354243956475556840952248445541392394100016207693636846776413017819659379971557468541946334893748439129742391433659360410035234377706588867781139498616478747140793263858738624732889645643598774667638479466504074111825658378878454858148962961273998413442726086061872455452360643153710112746809778704464094758280348769758948328241239292960582948619196670918958089833201210318430340128495116203534280144127617285830243559830032042024512072872535581195840149180969253395075778400067465526031446167050827682772223534191102634163157147406123850425845988419907611287258059113935689601431668283176323567325417073420817332230462987992804908514094790368878687894930546955703072619009502076433493359106024545086453628935456862958531315337183868265617862273637169757741830239860065914816164049449650117321313895747062088474802365371031150898427992754426853277974311395143574172219759799359685252285745263796289612691572357986620573408375766873884266405990993505000813375432454635967504844235284874701443545419576258473564216198134073468541117668831186544893776979566517279662326714810338643913751865946730024434500544995399742372328712494834706044063471606325830649829795510109541836235030309453097335834462839476304775645015008507578949548931393944899216125525597701436858943585877526379625597081677643800125436502371412783467926101995585224717220177723700417808419423948725406801556035998390548985723546745642390585850216719031395262944554391316631345308939062046784387785054239390524731362012947691874975191011472315289326772533918146607300089027768963114810902209724520759167297007850580717186381054967973100167870850694207092232908070383263453452038027860990556900134137182368370991949516489600755049341267876436746384902063964019766685592335654639138363185745698147196210841080961884605456039038455343729141446513474940784884423772175154334260306698831768331001133108690421939031080143784334151370924353013677631084913516156422698475074303297167469640666531527035325467112667522460551199581831963763707617991919203579582007595605302346267757943936307463056901080114942714100939136913810725813781357894005599500183542511841721360557275221035268037357265279224173736057511278872181908449006178013889710770822931002797665935838758909395688148560263224393726562472776037890814458837855019702843779362407825052704875816470324581290878395232453237896029841669225489649715606981192186584926770403956481278102179913217416305810554598801300484562997651121241536374515005635070127815926714241342103301566165356024733807843028655257222753049998837015348793008062601809623815161366903341111386538510919367393835229345888322550887064507539473952043968079067086806445096986548801682874343786126453815834280753061845485903798217994599681154419742536344399602902510015888272164745006820704193761584547123183460072629339550548239557137256840232268213012476794522644820910235647752723082081063518899152692889108455571126603965034397896278250016110153235160519655904211844949907789992007329476905868577878720982901352956613978884860509786085957017731298155314951681467176959760994210036183559138777817698458758104466283998806006162298486169353373865787735983361613384133853684211978938900185295691967804554482858483701170967212535338758621582310133103877668272115726949518179589754693992642197915523385766231676275475703546994148929041301863861194391962838870543677743224276809132365449485366768000001065262485473055861598999140170769838548318875014293890899506854530765116803337322265175662207526951791442252808165171667766727930354851542040238174608923283917032754257508676551178593950027933895920576682789677644531840404185540104351348389531201326378369283580827193783126549617459970567450718332065034556644034490453627560011250184335607361222765949278393706478426456763388188075656121689605041611390390639601620221536849410926053876887148379895599991120991646464411918568277004574243434021672276445589330127781586869525069499364610175685060167145354315814801054588605645501332037586454858403240298717093480910556211671546848477803944756979804263180991756422809873998766973237695737015808068229045992123661689025962730430679316531149401764737693873514093361833216142802149763399189835484875625298752423873077559555955465196394401821840998412489826236737714672260616336432964063357281070788758164043814850188411431885988276944901193212968271588841338694346828590066640806314077757725705630729400492940302420498416565479736705485580445865720227637840466823379852827105784319753541795011347273625774080213476826045022851579795797647467022840999561601569108903845824502679265942055503958792298185264800706837650418365620945554346135134152570065974881916341359556719649654032187271602648593049039787489589066127250794828276938953521753621850796297785146188432719223223810158744450528665238022532843891375273845892384422535472653098171578447834215822327020690287232330053862163479885094695472004795231120150432932266282727632177908840087861480221475376578105819702226309717495072127248479478169572961423658595782090830733233560348465318730293026659645013718375428897557971449924654038681799213893469244741985097334626793321072686870768062639919361965044099542167627840914669856925715074315740793805323925239477557441591845821562518192155233709607483329234921034514626437449805596103307994145347784574699992128599999399612281615219314888769388022281083001986016549416542616968586788372609587745676182507275992950893180521872924610867639958916145855058397274209809097817293239301067663868240401113040247007350857828724627134946368531815469690466968693925472519413992914652423857762550047485295476814795467007050347999588867695016124972282040303995463278830695976249361510102436555352230690612949388599015734661023712235478911292547696176005047974928060721268039226911027772261025441492215765045081206771735712027180242968106203776578837166909109418074487814049075517820385653909910477594141321543284406250301802757169650820964273484146957263978842560084531214065935809041271135920041975985136254796160632288736181367373244506079244117639975974619383584574915988097667447093006546342423460634237474666080431701260052055928493695941434081468529815053947178900451835755154125223590590687264878635752541911288877371766374860276606349603536794702692322971868327717393236192007774522126247518698334951510198642698878471719396649769070825217423365662725928440620430214113719922785269984698847702323823840055655517889087661360130477098438611687052310553149162517283732728676007248172987637569816335415074608838663640693470437206688651275688266149730788657015685016918647488541679154596507234287730699853713904300266530783987763850323818215535597323530686043010675760838908627049841888595138091030423595782495143988590113185835840667472370297149785084145853085781339156270760356390763947311455495832266945702494139831634332378975955680856836297253867913275055542524491943589128405045226953812179131914513500993846311774017971512283785460116035955402864405902496466930707769055481028850208085800878115773817191741776017330738554758006056014337743299012728677253043182519757916792969965041460706645712588834697979642931622965520168797300035646304579308840327480771811555330909887025505207680463034608658165394876951960044084820659673794731680864156456505300498816164905788311543454850526600698230931577765003780704661264706021457505793270962047825615247145918965223608396645624105195510522357239739512881816405978591427914816542632892004281609136937773722299983327082082969955737727375667615527113922588055201898876201141680054687365580633471603734291703907986396522961312801782679717289822936070288069087768660593252746378405397691848082041021944719713869256084162451123980620113184541244782050110798760717155683154078865439041210873032402010685341947230476666721749869868547076781205124736792479193150856444775379853799732234456122785843296846647513336573692387201464723679427870042503255589926884349592876124007558756946413705625140011797133166207153715436006876477318675587148783989081074295309410605969443158477539700943988394914432353668539209946879645066533985738887866147629443414010498889931600512076781035886116602029611936396821349607501116498327856353161451684576956871090029997698412632665023477167286573785790857466460772283415403114415294188047825438761770790430001566986776795760909966936075594965152736349811896413043311662774712338817406037317439705406703109676765748695358789670031925866259410510533584384656023391796749267844763708474978333655579007384191473198862713525954625181604342253729962863267496824058060296421146386436864224724887283434170441573482481833301640566959668866769563491416328426414974533349999480002669987588815935073578151958899005395120853510357261373640343675347141048360175464883004078464167452167371904831096767113443494819262681110739948250607394950735031690197318521195526356325843390998224986240670310768318446607291248747540316179699411397387765899868554170318847788675929026070043212666179192235209382278788809886335991160819235355570464634911320859189796132791319756490976000139962344455350143464268604644958624769094347048293294140411146540923988344435159133201077394411184074107684981066347241048239358274019449356651610884631256785297769734684303061462418035852933159734583038455410337010916767763742762102137013548544509263071901147318485749233181672072137279355679528443925481560913728128406333039373562420016045664557414588166052166608738748047243391212955877763906969037078828527753894052460758496231574369171131761347838827194168606625721036851321566478001476752310393578606896111259960281839309548709059073861351914591819510297327875571049729011487171897180046961697770017913919613791417162707018958469214343696762927459109940060084983568425201915593703701011049747339493877885989417433031785348707603221982970579751191440510994235883034546353492349826883624043327267415540301619505680654180939409982020609994140216890900708213307230896621197755306659188141191577836272927461561857103721724710095214236964830864102592887457999322374955191221951903424452307535133806856807354464995127203174487195403976107308060269906258076020292731455252078079914184290638844373499681458273372072663917670201183004648190002413083508846584152148991276106513741539435657211390328574918769094413702090517031487773461652879848235338297260136110984514841823808120540996125274580881099486972216128524897425555516076371675054896173016809613803811914361143992106380050832140987604599309324851025168294467260666138151745712559754953580239983146982203613380828499356705575524712902745397762140493182014658008021566536067765508783804304134310591804606800834591136640834887408005741272586704792258319127415739080914383138456424150940849133918096840251163991936853225557338966953749026620923261318855891580832455571948453875628786128859004106006073746501402627824027346962528217174941582331749239683530136178653673760642166778137739951006589528877427662636841830680190804609849809469763667335662282915132352788806157768278159588669180238940333076441912403412022316368577860357276941541778826435238131905028087018575047046312933353757285386605888904583111450773942935201994321971171642235005644042979892081594307167019857469273848653833436145794634175922573898588001698014757420542995801242958105456510831046297282937584161162532562516572498078492099897990620035936509934721582965174135798491047111660791587436986541222348341887722929446335178653856731962559852026072947674072616767145573649812105677716893484917660771705277187601199908144113058645577910525684304811440261938402322470939249802933550731845890355397133088446174107959162511714864874468611247605428673436709046678468670274091881014249711149657817724279347070216688295610877794405048437528443375108828264771978540006509704033021862556147332117771174413350281608840351781452541964320309576018694649088681545285621346988355444560249556668436602922195124830910605377201980218310103270417838665447181260397190688462370857518080035327047185659499476124248110999288679158969049563947624608424065930948621507690314987020673533848349550836366017848771060809804269247132410009464014373603265645184566792456669551001502298330798496079949882497061723674493612262229617908143114146609412341593593095854079139087208322733549572080757165171876599449856937956238755516175754380917805280294642004472153962807463602113294255916002570735628126387331060058910652457080244749375431841494014821199962764531068006631183823761639663180931444671298615527598201451410275600689297502463040173514891945763607893528555053173314164570504996443890936308438744847839616840518452732884032345202470568516465716477139323775517294795126132398229602394548579754586517458787713318138752959809412174227300352296508089177705068259248822322154938048371454781647213976820963320508305647920482085920475499857320388876391601995240918938945576768749730856955958010659526503036266159750662225084067428898265907510637563569968211510949669744580547288693631020367823250182323708459790111548472087618212477813266330412076216587312970811230758159821248639807212407868878114501655825136178903070860870198975889807456643955157415363193191981070575336633738038272152798849350397480015890519420879711308051233933221903466249917169150948541401871060354603794643379005890957721180804465743962806186717861017156740967662080295766577051291209907944304632892947306159510430902221439371849560634056189342513057268291465783293340524635028929175470872564842600349629611654138230077313327298305001602567240141851520418907011542885799208121984493156999059182011819733500126187728036812481995877070207532406361259313438595542547781961142935163561223496661522614735399674051584998603552953329245752388810136202347624669055816438967863097627365504724348643071218494373485300606387644566272186661701238127715621379746149861328744117714552444708997144522885662942440230184791205478498574521634696448973892062401943518310088283480249249085403077863875165911302873958787098100772718271874529013972836614842142871705531796543076504534324600536361472618180969976933486264077435199928686323835088756683595097265574815431940195576850437248001020413749831872259677387154958399718444907279141965845930083942637020875635398216962055324803212267498911402678528599673405242031091797899905718821949391320753431707980023736590985375520238911643467185582906853711897952626234492483392496342449714656846591248918556629589329909035239233333647435203707701010843880032907598342170185542283861617210417603011645918780539367447472059985023582891833692922337323999480437108419659473162654825748099482509991833006976569367159689364493348864744213500840700660883597235039532340179582557036016936990988671132109798897070517280755855191269930673099250704070245568507786790694766126298082251633136399521170984528092630375922426742575599892892783704744452189363203489415521044597261883800300677617931381399162058062701651024458869247649246891924612125310275731390840470007143561362316992371694848132554200914530410371354532966206392105479824392125172540132314902740585892063217589494345489068463993137570910346332714153162232805522972979538018801628590735729554162788676498274186164218789885741071649069191851162815285486794173638906653885764229158342500673612453849160674137340173572779956341043326883569507814931378007362354180070619180267328551191942676091221035987469241172837493126163395001239599240508454375698507957046222664619000103500490183034153545842833764378111988556318777792537201166718539541835984438305203762819440761594106820716970302285152250573126093046898423433152732131361216582808075212631547730604423774753505952287174402666389148817173086436111389069420279088143119448799417154042103412190847094080254023932942945493878640230512927119097513536000921971105412096683111516328705423028470073120658032626417116165957613272351566662536672718998534199895236884830999302757419916463841427077988708874229277053891227172486322028898425125287217826030500994510824783572905691988555467886079462805371227042466543192145281760741482403827835829719301017888345674167811398954750448339314689630763396657226727043393216745421824557062524797219978668542798977992339579057581890622525473582205236424850783407110144980478726691990186438822932305382318559732869780922253529591017341407334884761005564018242392192695062083183814546983923664613639891012102177095976704908305081854704194664371312299692358895384930136356576186106062228705599423371631021278457446463989738188566746260879482018647487672727222062676465338099801966883680994159075776852639865146253336312450536402610569605513183813174261184420189088853196356986962795036738424313011331753305329802016688817481342988681585577810343231753064784983210629718425184385534427620128234570716988530518326179641178579608888150329602290705614476220915094739035946646916235396809201394578175891088931992112260073928149169481615273842736264298098234063200244024495894456129167049508235812487391799648641133480324757775219708932772262349486015046652681439877051615317026696929704928316285504212898146706195331970269507214378230476875280287354126166391708245925170010714180854800636923259462019002278087409859771921805158532147392653251559035410209284665925299914353791825314545290598415817637058927906909896911164381187809435371521332261443625314490127454772695739393481546916311624928873574718824071503995009446731954316193855485207665738825139639163576723151005556037263394867208207808653734942440115799667507360711159351331959197120948964717553024531364770942094635696982226673775209945168450643623824211853534887989395673187806606107885440005508276570305587448541805778891719207881423351138662929667179643468760077047999537883387870348718021842437342112273940255717690819603092018240188427057046092622564178375265263358324240661253311529423457965569502506810018310900411245379015332966156970522379210325706937051090830789479999004999395322153622748476603613677697978567386584670936679588583788795625946464891376652199588286933801836011932368578558558195556042156250883650203322024513762158204618106705195330653060606501054887167245377942831338871631395596905832083416898476065607118347136218123246227258841990286142087284956879639325464285343075301105285713829643709990356948885285190402956047346131138263878897551788560424998748316382804046848618938189590542039889872650697620201995548412650005394428203930127481638158530396439925470201672759328574366661644110962566337305409219519675148328734808957477775278344221091073111351828046036347198185655572957144747682552857863349342858423118749440003229690697758315903858039353521358860079600342097547392296733310649395601812237812854584317605561733861126734780745850676063048229409653041118306671081893031108871728167519579675347188537229309616143204006381322465841111157758358581135018569047815368938137718472814751998350504781297718599084707621974605887423256995828892535041937958260616211842368768511418316068315867994601652057740529423053601780313357263267054790338401257305912339601880137825421927094767337191987287385248057421248921183470876629667207272325650565129333126059505777727542471241648312832982072361750574673870128209575544305968395555686861188397135522084452852640081252027665557677495969626612604565245684086139238265768583384698499778726706555191854468698469478495734622606294219624557085371272776523098955450193037732166649182578154677292005212667143463209637891852323215018976126034373684067194193037746880999296877582441047878123266253181845960453853543839114496775312864260925211537673258866722604042523491087026958099647595805794663973419064010036361904042033113579336542426303561457009011244800890020801478056603710154122328891465722393145076071670643556827437743965789067972687438473076346451677562103098604092717090951280863090297385044527182892749689212106670081648583395537735919136950153162018908887484210798706899114804669270650940762046502772528650728905328548561433160812693005693785417861096969202538865034577183176686885923681488475276498468821949739729707737187188400414323127636504814531122850990020742409255859252926103021067368154347015252348786351643976235860419194129697690405264832347009911154242601273438022089331096686367898694977994001260164227609260823493041180643829138347354679725399262338791582998486459271734059225620749105308531537182911681637219395188700957788181586850464507699343940987433514431626330317247747486897918209239480833143970840673084079589358108966564775859905563769525232653614424780230826811831037735887089240613031336477371011628214614661679404090518615260360092521947218890918107335871964142144478654899528582343947050079830388538860831035719306002771194558021911942899922722353458707566246926177663178855144350218287026685610665003531050216318206017609217984684936863161293727951873078972637353717150256378733579771808184878458866504335824377004147710414934927438457587107159731559439426412570270965125108115548247939403597681188117282472158250109496096625393395380922195591918188552678062149923172763163218339896938075616855911752998450132067129392404144593862398809381240452191484831646210147389182510109096773869066404158973610476436500068077105656718486281496371118832192445663945814491486165500495676982690308911185687986929470513524816091743243015383684707292898982846022237301452655679898627767968091469798378268764311598832109043715611299766521539635464420869197567370005738764978437686287681792497469438427465256316323005551304174227341646455127812784577772457520386543754282825671412885834544435132562054464241011037955464190581168623059644769587054072141985212106734332410756767575818456990693046047522770167005684543969234041711089888993416350585157887353430815520811772071880379104046983069578685473937656433631979786803671873079693924236321448450354776315670255390065423117920153464977929066241508328858395290542637687668968805033317227800185885069736232403894700471897619347344308437443759925034178807972235859134245813144049847701732361694719765715353197754997162785663119046912609182591249890367654176979903623755286526375733763526969344354400473067198868901968147428767790866979688522501636949856730217523132529265375896415171479559538784278499866456302878831962099830494519874396369070682762657485810439112232618794059941554063270131989895703761105323606298674803779153767511583043208498720920280929752649812569163425000522908872646925284666104665392171482080130502298052637836426959733707053922789153510568883938113249757071331029504430346715989448786847116438328050692507766274500122003526203709466023414648998390252588830148678162196775194583167718762757200505439794412459900771152051546199305098386982542846407255540927403132571632640792934183342147090412542533523248021932277075355546795871638358750181593387174236061551171013123525633485820365146141870049205704372018261733194715700867578539336078622739558185797587258744102542077105475361294047460100094095444959662881486915903899071865980563617137692227290764197755177720104276496949611056220592502420217704269622154958726453989227697660310524980855759471631075870133208861463266412591148633881220284440694169488261529577625325019870359870674380469821942056381255833436421949232275937221289056420943082352544084110864545369404969271494003319782861318186188811118408257865928757426384450059944229568586460481033015388911499486935436030221810943466764000022362550573631294626296096198760564259963946138692330837196265954739234624134597795748524647837980795693198650815977675350553918991151335252298736112779182748542008689539658359421963331502869561192012298889887006079992795411188269023078913107603617634779489432032102773359416908650071932804017163840644987871753756781185321328408216571107549528294974936214608215583205687232185574065161096274874375098092230211609982633033915469494644491004515280925089745074896760324090768983652940657920198315265410658136823791984090645712468948470209357761193139980246813405200394781949866202624008902150166163813538381515037735022966074627952910384068685569070157516624192987244482719429331004854824454580718897633003232525821581280327467962002814762431828622171054352898348208273451680186131719593324711074662228508710666117703465352839577625997744672185715816126411143271794347885990892808486694914139097716736900277758502686646540565950394867841110790116104008572744562938425494167594605487117235946429105850909950214958793112196135908315882620682332156153086833730838173279328196983875087083483880463884784418840031847126974543709373298362402875197920802321878744882872843727378017827008058782410749357514889978911739746129320351081432703251409030487462262942344327571260086642508333187688650756429271605525289544921537651751492196367181049435317858383453865255656640657251363575064353236508936790431702597878177190314867963840828810209461490079715137717099061954969640070867667102330048672631475510537231757114322317411411680622864206388906210192355223546711662137499693269321737043105987225039456574924616978260970253359475020913836673772894438696400028110344026084712899000746807764844088711341352503367877316797709372778682166117865344231732264637847697875144332095340001650692130546476890985050203015044880834261845208730530973189492916425322933612431514306578264070283898409841602950309241897120971601649265613413433422298827909921786042679812457285345801338260995877178113102167340256562744007296834066198480676615805021691833723680399027931606420436812079900316264449146190219458229690992122788553948783538305646864881655562294315673128274390826450611628942803501661336697824051770155219626522725455850738640585299830379180350432876703809252167907571204061237596327685674845079151147313440001832570344920909712435809447900462494313455028900680648704293534037436032625820535790118395649089354345101342969617545249573960621490288728932792520696535386396443225388327522499605986974759882329916263545973324445163755334377492928990581175786355555626937426910947117002165411718219750519831787137106051063795558588905568852887989084750915764639074693619881507814685262133252473837651192990156109189777922008705793396463827490680698769168197492365624226087154176100430608904377976678519661891404144925270480881971498801542057787006521594009289777601330756847966992955433656139847738060394368895887646054983871478968482805384701730871117761159663505039979343869339119789887109156541709133082607647406305711411098839388095481437828474528838368079418884342666222070438722887413947801017721392281911992365405516395893474263953824829609036900288359327745855060801317988407162446563997948275783650195514221551339281978226984278638391679715091262410548725700924070045488485692950448110738087996547481568913935380943474556972128919827177020766613602489581468119133614121258783895577357194986317210844398901423948496659251731388171602663261931065366535041473070804414939169363262373767777095850313255990095762731957308648042467701212327020533742667053142448208168130306397378736642483672539837487690980602182785786216512738563513290148903509883270617258932575363993979055729175160097615459044771692265806315111028038436017374742152476085152099016158582312571590733421736576267142390478279587281505095633092802668458937649649770232973641319060982740633531089792464242134583740901169391964250459128813403498810635400887596820054408364386516617880557608956896727531538081942077332597917278437625661184319891025007491829086475149794003160703845549465385946027452447466812314687943441610993338908992638411847425257044572517459325738989565185716575961481266020310797628254165590506042479114016957900338356574869252800743025623419498286467914476322774005529460903940177536335655471931000175430047504719144899841040015867946179241610016454716551337074073950260442769538553834397550548871099785205401175169747581344926079433689543783221172450687344231989878844128542064742809735625807066983106979935260693392135685881391214807354728463227784908087002467776303605551232386656295178853719673034634701222939581606792509153217489030840886516061119011498443412350124646928028805996134283511884715449771278473361766285062169778717743824362565711779450064477718370221999106695021656757644044997940765037999954845002710665987813603802314126836905783190460792765297277694043613023051787080546511542469395265127101052927070306673024447125973939950514628404767431363739978259184541176413327906460636584152927019030276017339474866960348694976541752429306040727005059039503148522921392575594845078867977925253931765156416197168443524369794447355964260633391055126826061595726217036698506473281266724521989060549880280782881429796336696744124805982192146339565745722102298677599746738126069367069134081559412016115960190237753525556300606247983261249881288192937343476862689219239777833910733106588256813777172328315329082525092733047850724977139448333892552081175608452966590553940965568541706001179857293813998258319293679100391844099286575605993598910002969864460974714718470101531283762631146774209145574041815908800064943237855839308530828305476076799524357391631221886057549673832243195650655460852881201902363644712703748634421727257879503428486312944916318475347531435041392096108796057730987201352484075057637199253650470908582513936863463863368042891767107602111159828875539940120076013947033661793715396306139863655492213741597905119083588290097656647300733879314678913181465109316761575821351424860442292445304113160652700974330088499034675405518640677342603583409608605533747362760935658853109760994238347382222087292464497684560579562516765574088410321731345627735856052358236389532038534024842273371639123973215995440828421666636023296545694703577184873442034227706653837387506169212768015766181095420097708363604361110592409117889540338021426523948929686439808926114635414571535194342850721353453018315875628275733898268898523557799295727645229391567477566676051087887648453493636068278050564622813598885879259940946446041705204470046315137975431737187756039815962647501410906658866162180038266989961965580587208639721176995219466789857011798332440601811575658074284182910615193917630059194314434605154047710570054339000182453117733718955857603607182860506356479979004139761808955363669603162193113250223851791672055180659263518036251214575926238369348222665895576994660491938112486609099798128571823494006615552196112207203092277646200999315244273589488710576623894693889446495093960330454340842102462401048723328750081749179875543879387381439894238011762700837196053094383940063756116458560943129517597713935396074322792489221267045808183313764165818269562105872892447740035947009268662659651422050630078592002488291860839743732353849083964326147000532423540647042089499210250404726781059083644007466380020870126664209457181702946752278540074508552377720890581683918446592829417018288233014971554235235911774818628592967605048203864343108779562892925405638946621948268711042828163893975711757786915430165058602965217459581988878680408110328432739867198621306205559855266036405046282152306154594474489908839081999738747452969810776201487134000122535522246695409315213115337915798026979555710508507473874750758068765376445782524432638046143042889235934852961058269382103498000405248407084403561167817170512813378805705643450616119330424440798260377951198548694559152051960093041271007277849301555038895360338261929343797081874320949914159593396368110627557295278004254863060054523839151068998913578820019411786535682149118528207852130125518518493711503422159542244511900207393539627400208110465530207932867254740543652717595893500716336076321614725815407642053020045340183572338292661915308354095120226329165054426123619197051613839357326693760156914429944943744856809775696303129588719161129294681884936338647392747601226964158848900965717086160598147204467428664208765334799858222090619802173211614230419477754990738738567941189824660913091691772274207233367635032678340586301930193242996397204445179288122854478211953530898910125342975524727635730226281382091807439748671453590778633530160821559911314144205091447293535022230817193663509346865858656314855575862447818620108711889760652969899269328178705576435143382060141077329261063431525337182243385263520217735440715281898137698755157574546939727150488469793619500477720970561793913828989845327426227288647108883270173723258818244658436249580592560338105215606206155713299156084892064340303395262263451454283678698288074251422567451806184149564686111635404971897682154227722479474033571527436819409892050113653400123846714296551867344153741615042563256713430247655125219218035780169240326699541746087592409207004669340396510178134857835694440760470232540755557764728450751826890418293966113310160131119077398632462778219023650660374041606724962490137433217246454097412995570529142438208076098364823465973886691349919784013108015581343979194852830436739012482082444814128095443773898320059864909159505322857914576884962578665885999179867520554558099004556461178755249370124553217170194282884617402736649978475508294228020232901221630102309772151569446427909802190826689868834263071609207914085197695235553488657743425277531197247430873043619511396119080030255878387644206085044730631299277888942729189727169890575925244679660189707482960949190648764693702750773866432391919042254290235318923377293166736086996228032557185308919284403805071030064776847863243191000223929785255372375566213644740096760539439838235764606992465260089090624105904215453927904411529580345334500256244101006359530039598864466169595626351878060688513723462707997327233134693971456285542615467650632465676620279245208581347717608521691340946520307673391841147504140168924121319826881568664561485380287539331160232292555618941042995335640095786495340935115266454024418775949316930560448686420862757201172319526405023099774567647838488973464317215980626787671838005247696884084989185086149003432403476742686245952395890358582135006450998178244636087317754378859677672919526111213859194725451400301180503437875277664402762618941017576872680428176623860680477885242887430259145247073950546525135339459598789619778911041890292943818567205070964606263541732944649576612651953495701860015412623962286413897796733329070567376962156498184506842263690367849555970026079867996261019039331263768556968767029295371162528005543100786408728939225714512481135778627664902425161990277471090335933309304948380597856628844787441469841499067123764789582263294904679812089984857163571087831191848630254501620929805829208334813638405421720056121989353669371336733392464416125223196943471206417375491216357008573694397305979709719726666642267431117762176403068681310351899112271339724036887000996862922546465006385288620393800504778276912835603372548255793912985251506829969107754257647488325341412132800626717094009098223529657957997803018282428490221470748111124018607613415150387569830918652780658896682362523937845272634530420418802508442363190383318384550522367992357752929106925043261446950109861088899914658551881873582528164302520939285258077969737620845637482114433988162710031703151334402309526351929588680690821355853680161000213740851154484912685841268695899174149133820578492800698255195740201818105641297250836070356851055331787840829000041552511865779453963317538532092149720526607831260281961164858098684587525129997404092797683176639914655386108937587952214971731728131517932904431121815871023518740757222100123768721944747209349312324107065080618562372526732540733324875754482967573450019321902199119960797989373383673242576103938985349278777473980508080015544764061053522202325409443567718794565430406735896491017610775948364540823486130254718476485189575836674399791508512858020607820554462991723202028222914886959399729974297471155371858924238493855858595407438104882624648788053304271463011941589896328792678327322456103852197011130466587100500083285177311776489735230926661234588873102883515626446023671996644554727608310118788389151149340939344750073025855814756190881398752357812331342279866503522725367171230756861045004548970360079569827626392344107146584895780241408158405229536937499710665594894459246286619963556350652623405339439142111271810691052290024657423604130093691889255865784668461215679554256605416005071276641766056874274200329577160643448606201239821698271723197826816628249938714995449137302051843669076723577400053932662622760323659751718925901801104290384274185507894887438832703063283279963007200698012244365116394086922220745320244624121155804354542064215121585056896157356414313068883443185280853975927734433655384188340303517822946253702015782157373265523185763554098954033236382319219892171177449469403678296185920803403867575834111518824177439145077366384071880489358256868542011645031357633355509440319236720348651010561049872726472131986543435450409131859513145181276437310438972507004981987052176272494065214619959232142314439776546708351714749367986186552791715824080651063799500184295938799158350171580759883784962257398512129810326379376218322456594236685376799113140108043139732335449090824910499143325843298821033984698141715756010829706583065211347076803680695322971990599904451209087275776225351040902392888779424630483280319132710495478599180196967835321464441189260631526618167443193550817081875477050802654025294109218264858213857526688155584113198560022135158887210365696087515063187533002942118682221893775546027227291290504292259787710667873840000616772154638441292371193521828499824350920891801685572798156421858191197490985730570332667646460728757430565372602768982373259745084479649545648030771598153955827779139373601717422996027353102768719449444917939785144631597314435351850491413941557329382048542123508173912549749819308714396615132942045919380106231421774199184060180347949887691051557905554806953878540066453375981862846419905220452803306263695626490910827627115903856995051246529996062855443838330327638599800792922846659503551211245284087516229060262011857775313747949362055496401073001348853150735487353905602908933526400713274732621960311773433943673385759124508149335736911664541281788171454023054750667136518258284898099512139193995633241336556777098003081910272040997148687418134667006094051021462690280449159646545330107754695413088714165312544813061192407821188690056027781824235022696189344352547633573536485619363254417756613981703930632872166905722259745209192917262199844409646158269456380239502837121686446561785235565164127712826918688615572716201474934052276946595712198314943381622114006936307430444173284786101777743837977037231795255434107223445512555589998646183876764903972461167959018100035098928641204195163551108763204267612979826529425882951141275841262732790798807559751851576841264742209479721843309352972665210015662514552994745127631550917636730259462132930190402837954246323258550301096706922720227074863419005438302650681214142135057154175057508639907673946335146209082888934938376439399256900604067311422093312195936202982972351163259386772241477911629572780752395056251581603133359382311500518626890530658368129988108663263271980611271548858798093487912913707498230575929091862939195014721197586067270092547718025750337730799397134539532646195269996596385654917590458333585799102012713204583903200853878881633637685182083727885131175227769609787962142372162545214591281831798216044111311671406914827170981015457781939202311563871950805024679725792497605772625913328559726371211201905720771409148645074094926718035815157571514050397610963846755569298970383547314100223802583468767350129775413279532060971154506484212185936490997917766874774481882870632315515865032898164228288232746866106592732197907162384642153489852476216789050260998045266483929542357287343977680495774091449538391575565485459058976495198513801007958010783759945775299196700547602252552034453988712538780171960718164078124847847257912407824544361682345239570689514272269750431873633263011103053423335821609333191218806608268341428910415173247216053355849993224548730778822905252324234861531520976938461042582849714963475341837562003014915703279685301868631572488401526639835689563634657435321783493199825542117308467745297085839507616458229630324424328237737450517028560698067889521768198156710781633405266759539424926280756968326107495323390536223090807081455919837355377748742029039018142937311529334644468151212945097596534306284215319445727118614900017650558177095302468875263250119705209476159416768727784472000192789137251841622857783792284439084301181121496366424659033634194540657183544771912446621259392656620306888520055599121235363718226922531781458792593750441448933981608657900876165024635197045828895481793756681046474614105142498870252139936870509372305447734112641354892806841059107716677821238332810262185587751312721179344448201440425745083063944738363793906283008973306241380614589414227694747931665717623182472168350678076487573420491557628217583972975134478990696589532548940335615613167403276472469212505759116251529654568544633498114317670257295661844775487469378464233737238981920662048511894378868224807279352022501796545343757274163910791972952950812942922205347717304184477915673991738418311710362524395716152714669005814700002633010452643547865903290733205468338872078735444762647925297690170912007874183736735087713376977683496344252419949951388315074877537433849458259765560996555954318040920178497184685497370696212088524377013853757681416632722412634423982152941645378000492507262765150789085071265997036708726692764308377229685985169122305037462744310852934305273078865283977335246017463527703205938179125396915621063637625882937571373840754406468964783100704580613446731271591194608435935825987782835266531151065041623295329047772174083559349723758552138048305090009646676088301540612824308740645594431853413755220166305812111033453120745086824339432159043594430312431227471385842030390106070940315235556172767994160020393975099897629335325855575624808996691829864222677502360193257974726742578211119734709402357457222271212526852384295874273501563660093188045493338989741571490544182559738080871565281430102670460284316819230392535297795765862414392701549740879273131051636119137577008929564823323648298263024607975875767745377160102490804624301856524161756655600160859121534556267602192689982855377872583145144082654583484409478463178777374794653580169960779405568701192328608041130904629350871827125934668712766694873899824598527786499569165464029458935064964335809824765965165142090986755203808309203230487342703468288751604071546653834619611223013759451579252696743642531927390036038608236450762698827497618723575476762889950752114804852527950845033958570838130476937881321123674281319487950228066320170022460331989671970649163741175854851878484012054844672588851401562725019821719066960812627785485964818369621410721714214986361918774754509650308957099470934337856981674465828267911940611956037845397855839240761276344105766751024307559814552786167815949657062559755074306521085301597908073343736079432866757890533483669555486803913433720156498834220893399971641479746938696905480089193067138057171505857307148815649920714086758259602876056459782423770242469805328056632787041926768467116266879463486950464507420219373945259262668613552940624781361206202636498199999498405143868285258956342264328707663299304891723400725471764188685351372332667877921738347541480022803392997357936152412755829569276837231234798989446274330454566790062032420516396282588443085438307201495672106460533238537203143242112607424485845094580494081820927639140008540422023556260218564348994145439950410980591817948882628052066441086319001688568155169229486203010738897181007709290590480749092427141018933542818429995988169660993836961644381528877214085268088757488293258735809905670755817017949161906114001908553744882726200936685604475596557476485674008177381703307380305476973609786543859382187220583902344443508867499866506040645874346005331827436296177862518081893144363251205107094690813586440519229512932450078833398788429339342435126343365204385812912834345297308652909783300671261798130316794385535726296998740359570458452230856390098913179475948752126397078375944861139451960286751210561638976008880092746115860800207803341591451797073036835196977766076373785333012024120112046988609209339085365773222392412449051532780950955866459477634482269986074813297302630975028812103517723124465095349653693090018637764094094349837313251321862080214809922685502948454661814715557444709669530177690434272031892770604717784527939160472281534379803539679861424370956683221491465438014593829277393396032754048009552231816667380357183932757077142046723838624617803976292377131209580789363841447929802588065522129262093623930637313496640186619510811583471173312025805866727639992763579078063818813069156366274125431259589936119647626101405563503399523140323113819656236327198961837254845333702062563464223952766943568376761368711962921818754576081617053031590728828700712313666308722754918661395773730546065997437810987649802414011242142773668082751390959313404155826266789510846776118665957660165998178089414985754976284387856100263796543178313634025135814161151902096499133548733131115022700681930135929595971640197196053625033558479980963488718039111612813595968565478868325856437896173159762002419621552896297904819822199462269487137462444729093456470028537694958859591606789282491054412515996300781368367490209374915732896270028656829344431342347351239298259166739503425995868970697267332582735903121288746660451461487850346142827765991608090398652575717263081833494441820193533385071292345774375579344062178711330063106003324053991693682603746176638565758877580201229366353270267100681261825172914608202541892885935244491070138206211553827793565296914576502048643282865557934707209634807372692141186895467322767751335690190153723669036865389161291688887876407525493494249733427181178892759931596719354758988097924525262363659036320070854440784544797348291802082044926670634420437555325050527522833778887040804033531923407685630109347772125639088640413101073817853338316038135280828119040832564401842053746792992622037698718018061122624490909242641985820861751177113789051609140381575003366424156095216328197122335023167422600567941281406217219641842705784328959802882335059828208196666249035857789940333152274817776952843681630088531769694783690580671064828083598046698841098135158654906933319522394363287923990534810987830274500172065433699066117784554364687723631844464768069142828004551074686645392805399409108754939166095731619715033166968309929466349142798780842257220697148875580637480308862995118473187124777291910070227588893486939456289515802965372150409603107761289831263589964893410247036036645058687287589051406841238124247386385427908282733827973326885504935874303160274749063129572349742611221517417153133618622410913869500688835898962349276317316478340077460886655598733382113829928776911495492184192087771606068472874673681886167507221017261103830671787856694812948785048943063086169948798703160515884108282351274153538513365895332948629494495061868514779105804696039069372662670386512905201137810858616188886947957607413585534585151768051973334433495230120395770739623771316030242887200537320998253008977618973129817881944671731160647231476248457551928732782825127182446807824215216469567819294098238926284943760248852279003620219386696482215628093605373178040863727268426696421929946819214908701707533361094791381804063287387593848269535583077395761447997270003472880182785281389503217986345216111066608839314053226944905455527867894417579202440021450780192099804461382547805858048442416404775031536054906591430078158372430123137511562284015838644270890718284816757527123846782459534334449622010096071051370608461801187543120725491334994247617115633321408934609156561550600317384218701570226103101916603887064661438897736318780940711527528174689576401581047016965247557740891644568677717158500583269943401677202156767724068128366565264122982439465133197359199709403275938502669557470231813203243716420586141033606524536939160050644953060161267822648942437397166717661231048975031885732165554988342121802846912529086101485527815277625623750456375769497734336846015607727035509629049392487088406281067943622418704747008368842671022558302403599841645951122485272633632645114017395248086194635840783753556885622317115520947223065437092606797351000565549381224575483728545711797393615756167641692895805257297522338558611388322171107362265816218842443178857488798109026653793426664216990914056536432249301334867988154886628665052346997235574738424830590423677143278792316422403877764330192600192284778313837632536121025336935812624086866699738275977365682227907215832478888642369346396164363308730139814211430306008730666164803678984091335926293402304324974926887831643602681011309570716141912830686577323532639653677390317661361315965553584999398600565155921936759977717933019744688148371103206503693192894521402650915465184309936553493337183425298433679915939417466223900389527673813330617747629574943868716978453767219493506590875711917720875477107189937960894774512654757501871194870738736785890200617373321075693302216320628432065671192096950585761173961632326217708945426214609858410237813215817727602222738133495410481003073275107799948991977963883530734443457532975914263768405442264784216063122769646967156473999043715903323906560726644116438605404838847161912109008701019130726071044114143241976796828547885524779476481802959736049439700479596040292746299203572099761950140348315380947714601056333446998820822120587281510729182971211917876424880354672316916541852256729234429187128163232596965413548589577133208339911288775917226115273379010341362085614577992398778325083550730199818459025958355989260553299673770491722454935329683300002230181517226575787524058832249085821280089747909326100762578770428656006996176212176845478996440705066241710213327486796237430229155358200780141165348065647488230615003392068983794766255036549822805329662862117930628430170492402301985719978948836897183043805182174419147660429752437251683435411217038631379411422095295885798060152938752753799030938871683572095760715221900279379292786303637268765822681241993384808166021603722154710143007377537792699069587121289288019052031601285861825494413353820784883465311632650407642428390870121015194231961652268422003711230464300673442064747718021353070124098860353399152667923871101706221865883573781210935179775604425634694999787251125440854522274810914874307259869602040275941178942581281882159952359658979181144077653354321757595255536158128001163846720319346507296807990793963714961774312119402021297573125165253768017359101557338153772001952444543620071848475663415407442328621060997613243487548847434539665981338717466093020535070271952983943271425371155766600025784423031073429551533945060486222764966687624079324353192992639253731076892135352572321080889819339168668278948281170472624501948409700975760920983724090074717973340788141825195842598096241747610138252643955135259311885045636264188300338539652435997416931322894719878308427600401368074703904097238473945834896186539790594118599310356168436869219485382055780395773881360679549900085123259442529724486666766834641402189915944565309423440650667851948417766779470472041958822043295380326310537494883122180391279678446100139726753892195119117836587662528083690053249004597410947068772912328214304635337283519953648274325833119144459017809607782883583730111857543659958982724531925310588115026307542571493943024453931870179923608166611305426253995833897942971602070338767815033010280120095997252222280801423571094760351925544434929986767817891045559063015953809761875920358937341978962358931125983902598310267193304189215109689156225069659119828323455503059081730735195503721665870288053992138576037035377105178021280129566841984140362872725623214428754302210909472721073474134975514190737043318276626177275996888826027225247133683353452816692779591328861381766349857728936900965749562287103024362590772412219094300871755692625758065709912016659622436080242870024547362036394841255954881727272473653467783647201918303998717627037515724649922289467932322693619177641614618795613956699567783068290316589699430767333508234990790624100202506134057344300695745474682175690441651540636584680463692621274211075399042188716127617787014258864825775223889184599523376292377915585744549477361295525952226578636462118377598473700347971408206994145580719080213590732269233100831759510659019121294795408603640757358750205890208704579670007055262505811420663907459215273309406823649441590891009220296680523325266198911311842016291631076894084723564366808182168657219688268358402785500782804043453710183651096951782335743030504852653738073531074185917705610397395062640355442275156101107261779370634723804990666922161971194259120445084641746383589938239946517395509000859479990136026674261494290066467115067175422177038774507673563742154782905911012619157555870238957001405117822646989944917908301795475876760168094100135837613578591356924455647764464178667115391951357696104864922490083446715486383054477914330097680486878348184672733758436892724310447406807685278625585165092088263813233623148733336714764520450876627614950389949504809560460989604329123358348859990294526400284994280878624039811814884767301216754161106629995553668193123287425702063738352020086863691311733469731741219153633246745325630871347302792174956227014687325867891734558379964351358800959350877556356248810493852999007675135513527792412429277488565888566513247302514710210575352516511814850902750476845518252096331899068527614435138213662152368890578786699432288816028377482035506016029894009119713850179871683633744139275973644017007014763706655703504338121113576415018451821413619823495159601064752712575935185304332875537783057509567425442684712219618709178560783936144511383335649103256405733898667178123972237519316430617013859539474367843392670986712452211189690840236327411496601243483098929941738030588417166613073040067588380432111555379440605497721705942821514886165672771240903387727745629097110134885184374118695655449745736845218066982911045058004299887953899027804383596282409421860556287788428802127553884803728640019441614257499904272009595204654170598104989967504511936471172772220436102614079750809686975176600237187748348016120310234680567112644766123747627852190241202569943534716226660893675219833111813511146503854895025120655772636145473604426859498074396932331297127377157347099713952291182653485155587137336629120242714302503763269501350911612952993785864681307226486008270881333538193703682598867893321238327053297625857382790097826460545598555131836688844628265133798491667839409761353766251798258249663458771950124384040359140849209733754642474488176184070023569580177410177696925077814893386672557898564589851056891960924398841569280696983352240225634570497312245269354193837004843183357196516626721575524193401933099018319309196582920969656247667683659647019595754739345514337413708761517323677204227385674279170698204549953095918872434939524094441678998846319845504852393662972079777452814399418256789457795712552426826089940863317371538896262889629402112108884427376568624527612130371017300785135715404533041507959447776143597437803742436646973247138410492124314138903579092416036406314038149831481905251720937103964026808994832572297954564042701757722904173234796073618787889913318305843069394825961318713816423467218730845133877219086975104942843769325024981656673816260615941768252509993741672883951744066932549653403101452225316189009235376486378482881344209870048096227171226407489571939002918573307460104360729190945767994614929290427981687729426487729952858434647775386906950148984133924540394144680263625402118614317031251117577642829914644533408920976961699098372652361768745605894704968170136974909523072082682887890730190018253425805343421705928713931737993142410852647390948284596418093614138475831136130576108462366837237695913492615824516221552134879244145041756848064120636520170386330129532777699023118648020067556905682295016354931992305914246396217025329747573114094220180199368035026495636955866425906762685687372110339156793839895765565193177883000241613539562437777840801748819373095020699900890899328088397430367736595524891300156633294077907139615464534088791510300651321934486673248275907946807879819425019582622320395131252014109960531260696555404248670549986786923021746989009547850725672978794769888831093487464426400718183160331655511534276155622405474473378049246214952133258527698847336269182649174338987824789278468918828054669982303689939783413747587025805716349413568433929396068192061773331791738208562436433635359863494496890781064019674074436583667071586924521182997893804077137501290858646578905771426833582768978554717687184427726120509266486102051535642840632368481807287940717127966820060727559555904040233178749447346454760628189541512139162918444297651066947969354016866010055196077687335396511614930937570968554559381513789569039251014953265628147011998326992200066392875374713135236421589265126204072887716578358405219646054105435443642166562244565042999010256586927279142752931172082793937751326106052881235373451068372939893580871243869385934389175713376300720319760816604464683937725806909237297523486702916910426369262090199605204121024077648190316014085863558427609537086558164273995349346546314504040199528537252004957805254656251154109252437991326262713609099402902262062836752132305065183934057450112099341464918433323646569371725914489324159006242020612885732926133596808726500045628284557574596592120530341310111827501306961509835515632004310784601906565493806542525229161991819959602752327702249855738824899882707465936355768582560518068964285376850772012220347920993936179268206590142165615925306737944568949070853263568196831861772268249911472615732035807646298116244013316737892788689229032593349861797021994981925739617673075834417098559222170171825712777534491508205278430904619460835217402005838672849709411023266953921445461066215006410674740207009189911951376466904481267253691537162290791385403937560077835153374167747942100384002308951850994548779039346122220865060160500351776264831611153325587705073541279249909859373473787081194253055121436979749914951860535920403830235716352727630874693219622190064260886183676103346002255477477813641012691906569686495012688376296907233961276287223041141813610060264044030035996988919945827397624114613744804059697062576764723766065541618574690527229238228275186799156983390747671146103022776606020061246876477728819096791613354019881402757992174167678799231603963569492851513633647219540611171767387372555728522940054361785176502307544693869307873499110352182532929726044553210797887711449898870911511237250604238753734841257086064069052058452122754533848008205302450456517669518576913200042816758054924811780519832646032445792829730129105318385636821206215531288668564956512613892261367064093953334570526986959692350353094224543865278677673027540402702246384483553239914751363441044050092330361271496081355490531539021002299595756583705381261965683144286057956696622154721695620870013727768536960840704833325132793112232507148630206951245395003735723346807094656483089209801534878705633491092366057554050864111521441481434630437273271045027768661953107858323334857840297160925215326092558932655600672124359464255065996771770388445396181632879614460817789272171836908880126778207430106422524634807454300476492885553409062185153654355474125476152769772667769772777058315801412185688011705028365275543214803488004442979998062157904564161957212784508928489806426497427090579129069217807298769477975112447305991406050629946894280931034216416629935614828130998870745292716048433630818404126469637925843094185442216359084576146078558562473814931427078266215185541603870206876980461747400808324343665382354555109449498431093494759944672673665352517662706772194183191977196378015702169933675083760057163454643671776723387588643405644871566964321041282595645349841388412890420682047007615596916843038999348366793542549210328113363184722592305554383058206941675629992013373175489122037230349072681068534454035993561823576312837767640631013125335212141994611869350833176587852047112364331226765129964171325217513553261867681942338790365468908001827135283584888444111761234101179918709236507184857856221021104009776994453121795022479578069506532965940383987369907240797679040826794007618729547835963492793904576973661643405359792219285870574957481696694062334272619733518136626063735982575552496509807260123668283605928341855848026958413772558970883789942910549800331113884603401939166122186696058491571485733568286149500019097591125218800396419762163559375743718011480559442298730418196808085647265713547612831629200449880315402105530597076666362749328308916880932359290081787411985738317192616728834918402429721290434965526942726402559641463525914348400675867690350382320572934132981593533044446496829441367323442158380761694831219333119819061096142952201536170298575105594326461468505452684975764807808009221335811378197749271768545075538328768874474591593731162470601091244609829424841287520224462594477638749491997840446829257360968534549843266536862844489365704111817793806441616531223600214918768769467398407517176307516849856359201486892943105940202457969622924566644881967576294349535326382171613395757790766370764569570259738800438415805894336137106551859987600754924187211714889295221737721146081154344982665479872580056674724051122007383459271575727715218589946948117940644466399432370044291140747218180224825837736017346685300744985564715420036123593397312914458591522887408719508708632218837288262822884631843717261903305777147651564143822306791847386039147683108141358275755853643597721650028277803713422869688787349795096031108899196143386664068450697420787700280509367203387232629637856038653216432348815557557018469089074647879122436375556668678067610544955017260791142930831285761254481944449473244819093795369008206384631678225064809531810406570254327604385703505922818919878065865412184299217273720955103242251079718077833042609086794273428955735559252723805511440438001239041687716445180226491681641927401106451622431101700056691121733189423400547959684669804298017362570406733282129962153684881404102194463424646220745575643960452985313071409084608499653767803793201899140865814662175319337665970114330608625009829566917638846056762972931464911493704624469351984039534449135141193667933301936617663652555149174982307987072280860859626112660504289296966535652516688885572112276802772743708917389639772257564890533401038855931125679991516589025016486961427207005916056166159702451989051832969278935550303934681219761582183980483960562523091462638447386296039848924386187298507775928792722068554807210497817653286210187476766897248841139560349480376727036316921007350834073865261684507482496448597428134936480372426116704266870831925040997615319076855770327421785010006441984124207396400139603601583810565928413684574119102736420274163723488214524101347716529603128408658419787951116511529827814620379139855006399960326591248525308493690313130100799977191362230866011099929142871249388541612038020411340188887219693477904497527454288072803509305828754420755134816660927879353566521255620139988249628478726214432362853676502591450468377635282587652139156480972141929675549384375582600253168536356731379262475878049445944183429172756988376226261846365452743497662411138451305481449836311789784489732076719508784158618879692955819733250699951402601511675529750575437810242238957925786562128432731202200716730574069286869363930186765958251326499145950260917069347519408975357464016830811798846452473618956056479426358070562563281189269663026479535951097127659136233180866921535788607812759910537171402204506186075374866306350591483916467656723205714516886170790984695932236724946737583099607042589220481550799132752088583781117685214269334786921895240622657921043620348852926267984013953216458791151579050460579710838983371864038024417511347226472547010794793996953554669619726763255229914654933499663234185951450360980344092212206712567698723427940708857070474293173329188523896721971353924492426178641188637790962814486917869468177591717150669111480020759432012061969637795103227089029566085562225452602610460736131368869009281721068198618553780982018471154163630326265699283424155023600978046417108525537612728905335045506135684143775854429677977014660294387687225115363801191758154028120818255606485410787933598921064427244898618961629413418001295130683638609294100083136673372153008352696235737175330738653338204842190308186449184093723944033405244909554558016406460761581010301767488475017661908692946098769201691202181688291040870709560951470416921147027413390052253340834812870353031023919699978597413908593605433599697075604460134242453682496098772581311024732798562072126572499003468293886872304895562253204463602639854225258416464324271611419817802482595563544907219226583863662663750835944314877635156145710745528016159677048442714194435183275698407552677926411261765250615965235457187956673170913319358761628255920783080185206890151504713340386100310055914817852110384754542933389188444120517943969970194112695119526564919594189975418393234647424290702718875223534393673633663200307232747037407123982562024662651974090199762452056198557625760008708173083288344381831070054514493545885422678578551915372292379555494333410174420169600090696415612732297770221217951868376359082255128816470021992348864043959153018464004714321186360622527011541122283802778538911098490201342741014121559769965438877197485376431158229838533123071751132961904559007938064276695819014842627991221792947987348901868471676503827328552059082984529806259250352128451925927986593506132961946796252373972565584157853744567558998032405492186962888490332560851455344391660226257775512916200772796852629387937530454181080729285891989715381797343496187232927614747850192611450413274873242970583408471112333746274617274626582415324271059322506255302314738759251724787322881491455915605036334575424233779160374952502493022351481961381162563911415610326844958072508273431765944054098269765269344579863479709743124498271933113863873159636361218623497261409556079920628316999420072054811525353393946076850019909886553861433495781650089961649079678142901148387645682174914075623767618453775144031475411206760160726460556859257799322070337333398916369504346690694828436629980037414527627716547623825546170883189810868806847853705536480469350958818025360529740793538676511195079373282083146268960071075175520614433784114549950136432446328193346389050936545714506900864483440180428363390513578157273973334537284263372174065775771079830517555721036795976901889958494130195999573017901240193908681356585539661941371794487632079868800371607303220547423572266896801882123424391885984168972277652194032493227314793669234004848976059037958094696041754279613782553781223947646147832926976545162290281701100437846038756544151739433960048915318817576650500951697402415644771293656614253949368884230517400129920556854289853897942669956777027089146513736892206104415481662156804219838476730871787590279209175900695273456682026513373111518000181434120962601658629821076663523361774007837783423709152644063054071807843358061072961105550020415131696373046849213356837265400307509829089364612047891114753037049893952833457824082817386441322710002968311940203323456420826473276233830294639378998375836554559919340866235090967961134004867027123176526663710778725111860354037554487418693519733656621772359229396776463251562023487570113795712096237723431370212031004965152111976013176419408203437348512852602913334915125083119802850177855710725373149139215709105130965059885999931560863655477403551898166733535880048214665099741433761182777723351910741217572841592580872591315074606025634903777263373914461377038021318347447301113032670296917335047701632106616227830027269283365584011791419447808748253360714403296252285775009808599609040936312635621328162071453406104224112083010008587264252112262480142647519426184325853386753874054743491072710049754281159466017136122590440158991600229827801796035194080046513534752698777609527839984368086908989197839693532179980139135442552717910225397010810632143048511378291498511381969143043497500189980681644412123273328307192824362406733196554692677851193152775113446468905504248113361434984604849051258345683266441528489713972376040328212660253516693914082049947320486021627759791771234751097502403078935759937715095021751693555827072533911892334070223832077585802137174778378778391015234132098489423459613692340497998279304144463162707214796117456975719681239291913740982925805561955207434243295982898980529233366415419256367380689494201471241340525072204061794355252555225008748790086568314542835167750542294803274783044056438581591952666758282929705226127628711040134801787224801789684052407924360582742467443076721645270313451354167649668901274786801010295133862698649748212118629040337691568576240699296372493097201628707200189835423690364149270236961938547372480329855045112089192879829874467864129159417531675602533435310626745254507114181483239880607297140234725520713490798398982355268723950909365667878992383712578976248755990443228895388377317348941122757071410959790047919301046740750411435381782464630795989555638991884773781341347070246747362112048986226991888517456251732519341352038115863350123913054441910073628447567514161050410973505852762044489190978901984315485280533985777844313933883994310444465669244550885946314081751220331390681596592510546858013133838152176418210433429788826119630443111388796258746090226130900849975430395771243230616906262919403921439740270894777663702488155499322458825979020631257436910946393252806241642476868495455324938017639371615636847859823715902385421265840615367228607131702674740131145261063765383390315921943469817605358380310612887852051546933639241088467632009567089718367490578163085158138161966882222047570437590614338040725853862083565176998426774523195824182683698270160237414938363496629351576854061397342746470899685618170160551104880971554859118617189668025973541705423985135560018720335079060946421271143993196046527424050882225359773481519135438571253258540493946010865793798058620143366078825219717809025817370870916460452727977153509910340736425020386386718220522879694458387652947951048660717390229327455426785669776865939923416834122274663015062155320502655341460995249356050854921756549134830958906536175693817637473644183378974229700703545206663170929607591989627732423090252397443861014263098687733913882518684316501027964911497737582888913450341148865948670215492101084328080783428089417298008983297536940644969903125399863919581601468995220880662285408414864274786281975546629278814621607171381880180840572084715868906836919393381864278454537956719272397972364651667592011057995663962598535512763558768140213409829016296873429850792471846056874828331381259161962476156902875901072733103299140623864608333378638257926302391590003557609032477281338887339178096966601469615031754226751125993315529674213336300222964906480934582008181061802100227664580400278213336758573019011371754672763059044353131319036092489097246427928455549913490005180295707082919052556781889913899625138662319380053611346224294610248954072404857123256628888931722116432947816190554868054943441034090680716088028227959686950133643814268252170472870863010137301155236861416908375675747637239763185757038109443390564564468524183028148107998376918512127201935044041804604721626939445788377090105974693219720558114078775989772072009689382249303236830515862657281114637996983137517937623215111252349734305240622105244234353732905655163406669506165892878218707756794176080712973781335187117931650033155523822487730653444179453415395202424449703410120874072188109388268167512042299404948179449472732894770111574139441228455521828424922240658752689172272780607116754046973008037039618787796694882555614674384392570115829546661358678671897661297311267200072971553613027503556167817765442287442114729881614802705243806817653573275578602505847084013208837932816008769081300492491473682517035382219619039014999523495387105997351143478292339499187936608692301375596368532373806703591144243268561512109404259582639301678017128669239283231057658851714020211196957064799814031505633045141564414623163763809904402816256917576489142569714163598439317433270237812336938043012892626375382667795034169334323607500248175741808750388475094939454896209740485442635637164995949920980884294790363666297526003243856352945844728944547166209297495496616877414120882130477022816116456044007236351581149729739218966737382647204722642221242016560150284971306332795814302516013694825567014780935790889657134926158161346901806965089556310121218491805847922720691871696316330044858020102860657858591269974637661741463934159569539554203314628026518951167938074573315759846086173702687867602943677780500244673391332431669880354073232388281847501051641331189537036488422690270478052742490603492082954755054003457160184072574536938145531175354210726557835615499874447480427323457880061873149341566046352979779455075359304795687209316724536547208381685855606043801977030764246083489876101345709394877002946175792061952549255757109038525171488525265671045349813419803390641529876343695420256080277614421914318921393908834543131769685101840103844472348948869520981943531906506555354617335814045544837884752526253949665869992058417652780125341033896469818642430034146791380619028059607854888010789705516946215228773090104467462497979992627120951684779568482583341402266477210843362437593741610536734041954738964197895425335036301861400951534766961476255651873823292468547356935802896011536791787303553159378363082248615177770541577576561759358512016692943111138863582159667618830326104164651714846979385422621687161400122378213779774131268977266712992025922017408770076956283473932201088159356286281928563571893384958850603853158179760679479840878360975960149733420572704603521790605647603285569276273495182203236144112584182426247712012035776388895974318232827871314608053533574494297621796789034568169889553518504478325616380709476951699086247100019748809205009521943632378719764870339223811540363475488626845956159755193765410115014067001226927474393888589943859730245414801061235908036274585288493563251585384383242493252666087588908318700709100237377106576985056433928854337658342596750653715005333514489908293887737352051459333049626531415141386124437935885070944688045486975358170212908490787347806814366323322819415827345671356443171537967818058195852464840084032909981943781718177302317003989733050495387356116261023999433259780126893432605584710278764901070923443884634011735556865903585244919370181041626208504299258697435817098133894045934471937493877624232409852832762266604942385129709453245586252103600829286649724174919141988966129558076770979594795306013119159011773943104209049079424448868513086844493705909026006120649425744710353547657859242708130410618546219881830090634588187038755856274911587375421064667951346487586771543838018521348281915812462599335160198935595167968932852205824799421034512715877163345222995418839680448835529753361286837225935390079201666941339091168758803988828869216002373257361588207163516271332810518187602104852180675526648673908900907195138058626735124312215691637902277328705410842037841525683288718046987952513073266340278519059417338920358540395677035611329354482585628287610610698229721420961993509331312171187891078766872044548876089410174798647137882462153955933333275562009439580434537919782280590395959927436913793778664940964048777841748336432684026282932406260081908081804390914556351936856063045089142289645219987798849347477729132797266027658401667890136490508741142126861969862044126965282981087045479861559545338021201155646979976785738920186243599326777689454060508218838227909833627167124490026761178498264377033002081844590009717235204331994708242098771514449751017055643029542821819670009202515615844174205933658148134902693111517093872260026458630561325605792560927332265579346280805683443921373688405650434307396574061017779370141424615493070741360805442100295600095663588977899267630517718781943706761498217564186590116160865408635391513039201316805769034172596453692350806417446562351523929050409479953184074862151210561833854566176652606393713658802521666223576132201941701372664966073252010771947931265282763302413805164907174565964853748354669194523580315301969160480994606814904037819829732360930087135760798621425422096419004367905479049930078372421581954535418371129368658430553842717628035279128821129308351575656599944741788438381565148434229858704245592434693295232821803508333726283791830216591836181554217157448465778420134329982594566884558266171979012180849480332448787258183774805522268151011371745368417870280274452442905474518234674919564188551244421337783521423865979925988203287085109338386829906571994614906290257427686038850511032638544540419184958866538545040571323629681069146814847869659166861842756798460041868762298055562963045953227923051616721591968675849523635298935788507746081537321454642984792310511676357749494622952569497660359473962430995343310404994209677883827002714478494069037073249106444151696053256560586778757417472110827435774315194060757983563629143326397812218946287447798119807225646714664054850131009656786314880090303749338875364183165134982546694673316118123364854397649325026179549357204305402182974871251107404011611405899911093062492312813116340549262571356721818628932786138833718028535056503591952741400869510926167541476792668032109237467087213606278332922386413619594121339278036118276324106004740971111048140003623342714514483334641675466354699731494756643423659493496845884551524150756376605086632827424794136062876041290644913828519456402643153225858624043141838669590633245063000392213192647625962691510904457695301444054618037857503036686212462278639752746667870121003392984873375014475600322100622358029343774955032037012738468163061026570300872275462966796880890587127676361066225722352229739206443093524327228100859973095132528630601105497915644791845004618046762408928925680912930592960642357021061524646205023248966593987324933967376952023991760898474571843531936646529125848064480196520162838795189499336759241485626136995945307287254532463291529110128763770605570609531377527751867923292134955245133089867969165129073841302167573238637575820080363575728002754490327953079900799442541108725693188014667935595834676432868876966610097395749967836593397846346959948950610490383647409504695226063858046758073069912290474089879166872117147527644711604401952718169508289733537148530928937046384420893299771125856840846608339934045689026787516008775461267988015465856522061210953490796707365539702576199431376639960606061106406959330828171876426043573425361756943784848495250108266488395159700490598380812105221111091943323951136051446459834210799058082093716464523127704023160072138543723461267260997870385657091998507595634613248460188409850194287687902268734556500519121546544063829253851276317663922050938345204300773017029940362615434001322763910912988327863920412300445551684054889809080779174636092439334912641164240093880746356607262336695842764583698268734815881961058571835767462009650526065929263548291499045768307210893245857073701660717398194485028842603963660746031184786225831056580870870305567595861341700745402965687634774176431051751036732869245558582082372038601781739405175130437994868822320044378043103170921034261674998000073016094814586374488778522273076330495383944345382770608760763542098445008306247630253572781032783461766970544287155315340016497076657195985041748199087201490875686037783591994719343352772947285537925787684832301101859365800717291186967617655053775030293033830706448912811412025506150896411007623824574488655182581058140345320124754723269087547507078577659732542844459353044992070014538748948226556442223696365544194225441338212225477497535494624827680533336983284156138692363443358553868471111430498248398991803165458638289353799130535222833430137953372954016257623228081138499491876144141322933767106563492528814528239506209022357876684650116660097382753660405446941653422239052108314585847035529352219928272760574821266065291385530345549744551470344939486863429459658431024190785923680224560763936784166270518555178702904073557304620639692453307795782245949710420188043000183881429008173039450507342787013124466860092778581811040911511729374873627887874907465285565434748886831064110051023020875107768918781525622735251550379532444857787277617001964853703555167655209119339343762866284619844026295252183678522367475108809781507098978413086245881522660963551401874495836926917799047120726494905737264286005211403581231076006699518536124862746756375896225299116496066876508261734178484789337295056739007878617925351440621045366250640463728815698232317500596261080921955211150859302955654967538862612972339914628358476048627627027309739202001432248707582337354915246085608210328882974183906478869923273691360048837436615223517058437705545210815513361262142911815615301758882573594892507108879262128641392443309383797333867806131795237315266773820858024701433527009243803266951742119507670884326346442749127558907746863582162166042741315170212458586056233631493164646913946562497471741958354218607748711057338458433689939645913740603382159352243594751626239188685307822821763983237306180204246560477527943104796189724299533029792497481684052893791044947004590864991872727345413508101983881864673609392571930511968645601855782450218231065889437986522432050677379966196955472440585922417953006820451795370043472451762893566770508490213107736625751697335527462302943031203596260953423574397249659211010657817826108745318874803187430823573699195156340957162700992444929749105489851519658664740148225106335367949737142510229341882585117371994499115097583746130105505064197721531929354875371191630262030328588658528480193509225875775597425276584011721342323648084027143356367542046375182552524944329657043861387865901965738802868401894087672816714137033661732650120578653915780703088714261519075001492576112927675193096728453971160213606303090542243966320674323582797889332324405779199278484633339777737655901870574806828678347965624146102899508487399692970750432753029972872297327934442988646412725348160603779707298299173029296308695801996312413304939350493325412355071054461182591141116454534710329881047844067780138077131465400099386306481266614330858206811395838319169545558259426895769841428893743467084107946318932539106963955780706021245974898293564613560788983472419979478564362042094613412387613198865352358312996862268948608408456655606876954501274486631405054735351746873009806322780468912246821460806727627708402402266155485024008952891657117617439020337584877842911289623247059191874691042005848326140677333751027195653994697162517248312230633919328707983800748485726516123434933273356664473358556430235280883924348278760886164943289399166399210488307847777048045728491456303353265070029588906265915498509407972767567129795010098229476228961891591441520032283878773485130979081019129267227103778898053964156362364169154985768408398468861684375407065121039062506128107663799047908879674778069738473170475253442156390387201238806323688037017949308954900776331523063548374256816653361606641980030188287123767481898330246836371488309259283375902278942588060087286038859168849730693948020511221766359138251524278670094406942355120201568377778851824670025651708509249623747726813694284350062938814429987905301056217375459182679973217735029368928065210025396268807498092643458011655715886700443503976505323478287327368840863540002740676783821963522226539290939807367391364082898722017776747168118195856133721583119054682936083236976113450281757830202934845982925000895682630271263295866292147653142233351793093387951357095346377183684092444422096319331295620305575517340067973740614162107923633423805646850092037167152642556371853889571416419772387422610596667396997173168169415435095283193556417705668622215217991151355639707143312893657553844648326201206424338016955862698561022460646069330793847858814367407000599769703649019273328826135329363112403650698652160638987250267238087403396744397830258296894256896741864336134979475245526291426522842419243083388103580053787023999542172113686550275341362211693140694669513186928102574795985605145005021715913317751609957865551981886193211282110709442287240442481153406055895958355815232012184605820563592699303478851132068626627588771446035996656108430725696500563064489187599466596772847171539573612108180841547273142661748933134174632662354222072600146012701206934639520564445543291662986660783089068118790090815295063626782075614388815781351134695366303878412092346942868730839320432333872775496805210302821544324723388845215343727250128589747691460808314404125868181540049187772287869801853454537006526655649170915429522756709222217474112062720656622989806032891672068743654948246108697367225547404812889242471854323605753411672850757552057131156697954584887398742228135887985840783135060548290551482785294891121905383195624228719484759407859398047901094194070671764439032730712135887385049993638838205501683402777496070276844880281912220636888636811043569529300652195528261526991271637277388418993287130563464688227398288763198645709836308917786487086676185485680047672552675414742851028145807403152992197814557756843681110185317498167016426647884090262682824448258027532094549915104518517716546311804904567985713257528117913656278158111288816562285876030875974963849435275676612168959261485030785362045274507752950631012480341804584059432926079854435620093708091821523920371790678121992280496069738238743312626730306795943960954957189577217915597300588693646845576676092450906088202212235719254536715191834872587423919410890444115959932760044506556206461164655665487594247369252336955993030355095817626176231849561906494839673002037763874369343999829430209147073618947932692762445186560239559053705128978163455423320114975994896278424327483788032701418676952621180975006405149755889650293004867605208010491537885413909424531691719987628941277221129464568294860281493181560249677887949813777216229359437811004448060797672429276249510784153446429150842764520002042769470698041775832209097020291657347251582904630910359037842977572651720877244740952267166306005469716387943171196873484688738186656751279298575016363411314627530499019135646823804329970695770150789337728658035712790913767420805655493624646
lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-null-max.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-null-max.input deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.input and /dev/null differ
lib/std/compress/deflate/testdata/huffman-null-max.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-null-max.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-pi.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-pi.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-pi.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-pi.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-pi.input deleted-1
......@@ -1 +0,0 @@
13.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
\ No newline at end of file
lib/std/compress/deflate/testdata/huffman-pi.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-pi.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-pi.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-1k.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-1k.input deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.input and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-1k.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-limit.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-limit.input deleted-4
......@@ -1,4 +0,0 @@
1aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2���vH
3��%������ ��ɷ���}��>���ls���m�IGH����1Y�4�[�� 0ˆ[|]o#�
4�-#���ul���pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O���{����k�*�w(��b� ��kQC9/��lu>�5�C.��u�
lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-limit.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-max.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-max.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-rand-max.input deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-rand-max.input and /dev/null differ
lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-shifts.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-shifts.input deleted-2
......@@ -1,2 +0,0 @@
1101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
\ No newline at end of file
lib/std/compress/deflate/testdata/huffman-shifts.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-shifts.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text-shift.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text-shift.input deleted-14
......@@ -1,14 +0,0 @@
1//Copyright2009ThGoAuthor.Allrightrrvd.
2//UofthiourccodigovrndbyBSD-tyl
3//licnthtcnbfoundinthLICENSEfil.
4
5pckgmin
6
7import"o"
8
9funcmin(){
10 vrb=mk([]byt,65535)
11 f,_:=o.Crt("huffmn-null-mx.in")
12 f.Writ(b)
13}
14ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
\ No newline at end of file
lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text-shift.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text.input deleted-14
......@@ -1,14 +0,0 @@
1// zig v0.10.0
2// create a file filled with 0x00
3const std = @import("std");
4
5pub fn main() !void {
6 var b = [1]u8{0} ** 65535;
7 const f = try std.fs.cwd().createFile(
8 "huffman-null-max.in",
9 .{ .read = true },
10 );
11 defer f.close();
12
13 _ = try f.writeAll(b[0..]);
14}
lib/std/compress/deflate/testdata/huffman-text.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-text.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-zero.dyn.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-zero.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/huffman-zero.golden deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-zero.golden and /dev/null differ
lib/std/compress/deflate/testdata/huffman-zero.input deleted-1
......@@ -1 +0,0 @@
100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
\ No newline at end of file
lib/std/compress/deflate/testdata/huffman-zero.wb.expect deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-zero.wb.expect and /dev/null differ
lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/huffman-zero.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/null-long-match.dyn.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput deleted
Binary files a/lib/std/compress/deflate/testdata/null-long-match.wb.expect-noinput and /dev/null differ
lib/std/compress/deflate/testdata/rfc1951.txt deleted-955
......@@ -1,955 +0,0 @@
1
2
3
4
5
6
7Network Working Group P. Deutsch
8Request for Comments: 1951 Aladdin Enterprises
9Category: Informational May 1996
10
11
12 DEFLATE Compressed Data Format Specification version 1.3
13
14Status of This Memo
15
16 This memo provides information for the Internet community. This memo
17 does not specify an Internet standard of any kind. Distribution of
18 this memo is unlimited.
19
20IESG Note:
21
22 The IESG takes no position on the validity of any Intellectual
23 Property Rights statements contained in this document.
24
25Notices
26
27 Copyright (c) 1996 L. Peter Deutsch
28
29 Permission is granted to copy and distribute this document for any
30 purpose and without charge, including translations into other
31 languages and incorporation into compilations, provided that the
32 copyright notice and this notice are preserved, and that any
33 substantive changes or deletions from the original are clearly
34 marked.
35
36 A pointer to the latest version of this and related documentation in
37 HTML format can be found at the URL
38 <ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40Abstract
41
42 This specification defines a lossless compressed data format that
43 compresses data using a combination of the LZ77 algorithm and Huffman
44 coding, with efficiency comparable to the best currently available
45 general-purpose compression methods. The data can be produced or
46 consumed, even for an arbitrarily long sequentially presented input
47 data stream, using only an a priori bounded amount of intermediate
48 storage. The format can be implemented readily in a manner not
49 covered by patents.
50
51
52
53
54
55
56
57
58Deutsch Informational [Page 1]
59
60RFC 1951 DEFLATE Compressed Data Format Specification May 1996
61
62
63Table of Contents
64
65 1. Introduction ................................................... 2
66 1.1. Purpose ................................................... 2
67 1.2. Intended audience ......................................... 3
68 1.3. Scope ..................................................... 3
69 1.4. Compliance ................................................ 3
70 1.5. Definitions of terms and conventions used ................ 3
71 1.6. Changes from previous versions ............................ 4
72 2. Compressed representation overview ............................. 4
73 3. Detailed specification ......................................... 5
74 3.1. Overall conventions ....................................... 5
75 3.1.1. Packing into bytes .................................. 5
76 3.2. Compressed block format ................................... 6
77 3.2.1. Synopsis of prefix and Huffman coding ............... 6
78 3.2.2. Use of Huffman coding in the "deflate" format ....... 7
79 3.2.3. Details of block format ............................. 9
80 3.2.4. Non-compressed blocks (BTYPE=00) ................... 11
81 3.2.5. Compressed blocks (length and distance codes) ...... 11
82 3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12
83 3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13
84 3.3. Compliance ............................................... 14
85 4. Compression algorithm details ................................. 14
86 5. References .................................................... 16
87 6. Security Considerations ....................................... 16
88 7. Source code ................................................... 16
89 8. Acknowledgements .............................................. 16
90 9. Author's Address .............................................. 17
91
921. Introduction
93
94 1.1. Purpose
95
96 The purpose of this specification is to define a lossless
97 compressed data format that:
98 * Is independent of CPU type, operating system, file system,
99 and character set, and hence can be used for interchange;
100 * Can be produced or consumed, even for an arbitrarily long
101 sequentially presented input data stream, using only an a
102 priori bounded amount of intermediate storage, and hence
103 can be used in data communications or similar structures
104 such as Unix filters;
105 * Compresses data with efficiency comparable to the best
106 currently available general-purpose compression methods,
107 and in particular considerably better than the "compress"
108 program;
109 * Can be implemented readily in a manner not covered by
110 patents, and hence can be practiced freely;
111
112
113
114Deutsch Informational [Page 2]
115
116RFC 1951 DEFLATE Compressed Data Format Specification May 1996
117
118
119 * Is compatible with the file format produced by the current
120 widely used gzip utility, in that conforming decompressors
121 will be able to read data produced by the existing gzip
122 compressor.
123
124 The data format defined by this specification does not attempt to:
125
126 * Allow random access to compressed data;
127 * Compress specialized data (e.g., raster graphics) as well
128 as the best currently available specialized algorithms.
129
130 A simple counting argument shows that no lossless compression
131 algorithm can compress every possible input data set. For the
132 format defined here, the worst case expansion is 5 bytes per 32K-
133 byte block, i.e., a size increase of 0.015% for large data sets.
134 English text usually compresses by a factor of 2.5 to 3;
135 executable files usually compress somewhat less; graphical data
136 such as raster images may compress much more.
137
138 1.2. Intended audience
139
140 This specification is intended for use by implementors of software
141 to compress data into "deflate" format and/or decompress data from
142 "deflate" format.
143
144 The text of the specification assumes a basic background in
145 programming at the level of bits and other primitive data
146 representations. Familiarity with the technique of Huffman coding
147 is helpful but not required.
148
149 1.3. Scope
150
151 The specification specifies a method for representing a sequence
152 of bytes as a (usually shorter) sequence of bits, and a method for
153 packing the latter bit sequence into bytes.
154
155 1.4. Compliance
156
157 Unless otherwise indicated below, a compliant decompressor must be
158 able to accept and decompress any data set that conforms to all
159 the specifications presented here; a compliant compressor must
160 produce data sets that conform to all the specifications presented
161 here.
162
163 1.5. Definitions of terms and conventions used
164
165 Byte: 8 bits stored or transmitted as a unit (same as an octet).
166 For this specification, a byte is exactly 8 bits, even on machines
167
168
169
170Deutsch Informational [Page 3]
171
172RFC 1951 DEFLATE Compressed Data Format Specification May 1996
173
174
175 which store a character on a number of bits different from eight.
176 See below, for the numbering of bits within a byte.
177
178 String: a sequence of arbitrary bytes.
179
180 1.6. Changes from previous versions
181
182 There have been no technical changes to the deflate format since
183 version 1.1 of this specification. In version 1.2, some
184 terminology was changed. Version 1.3 is a conversion of the
185 specification to RFC style.
186
1872. Compressed representation overview
188
189 A compressed data set consists of a series of blocks, corresponding
190 to successive blocks of input data. The block sizes are arbitrary,
191 except that non-compressible blocks are limited to 65,535 bytes.
192
193 Each block is compressed using a combination of the LZ77 algorithm
194 and Huffman coding. The Huffman trees for each block are independent
195 of those for previous or subsequent blocks; the LZ77 algorithm may
196 use a reference to a duplicated string occurring in a previous block,
197 up to 32K input bytes before.
198
199 Each block consists of two parts: a pair of Huffman code trees that
200 describe the representation of the compressed data part, and a
201 compressed data part. (The Huffman trees themselves are compressed
202 using Huffman encoding.) The compressed data consists of a series of
203 elements of two types: literal bytes (of strings that have not been
204 detected as duplicated within the previous 32K input bytes), and
205 pointers to duplicated strings, where a pointer is represented as a
206 pair <length, backward distance>. The representation used in the
207 "deflate" format limits distances to 32K bytes and lengths to 258
208 bytes, but does not limit the size of a block, except for
209 uncompressible blocks, which are limited as noted above.
210
211 Each type of value (literals, distances, and lengths) in the
212 compressed data is represented using a Huffman code, using one code
213 tree for literals and lengths and a separate code tree for distances.
214 The code trees for each block appear in a compact form just before
215 the compressed data for that block.
216
217
218
219
220
221
222
223
224
225
226Deutsch Informational [Page 4]
227
228RFC 1951 DEFLATE Compressed Data Format Specification May 1996
229
230
2313. Detailed specification
232
233 3.1. Overall conventions In the diagrams below, a box like this:
234
235 +---+
236 | | <-- the vertical bars might be missing
237 +---+
238
239 represents one byte; a box like this:
240
241 +==============+
242 | |
243 +==============+
244
245 represents a variable number of bytes.
246
247 Bytes stored within a computer do not have a "bit order", since
248 they are always treated as a unit. However, a byte considered as
249 an integer between 0 and 255 does have a most- and least-
250 significant bit, and since we write numbers with the most-
251 significant digit on the left, we also write bytes with the most-
252 significant bit on the left. In the diagrams below, we number the
253 bits of a byte so that bit 0 is the least-significant bit, i.e.,
254 the bits are numbered:
255
256 +--------+
257 |76543210|
258 +--------+
259
260 Within a computer, a number may occupy multiple bytes. All
261 multi-byte numbers in the format described here are stored with
262 the least-significant byte first (at the lower memory address).
263 For example, the decimal number 520 is stored as:
264
265 0 1
266 +--------+--------+
267 |00001000|00000010|
268 +--------+--------+
269 ^ ^
270 | |
271 | + more significant byte = 2 x 256
272 + less significant byte = 8
273
274 3.1.1. Packing into bytes
275
276 This document does not address the issue of the order in which
277 bits of a byte are transmitted on a bit-sequential medium,
278 since the final data format described here is byte- rather than
279
280
281
282Deutsch Informational [Page 5]
283
284RFC 1951 DEFLATE Compressed Data Format Specification May 1996
285
286
287 bit-oriented. However, we describe the compressed block format
288 in below, as a sequence of data elements of various bit
289 lengths, not a sequence of bytes. We must therefore specify
290 how to pack these data elements into bytes to form the final
291 compressed byte sequence:
292
293 * Data elements are packed into bytes in order of
294 increasing bit number within the byte, i.e., starting
295 with the least-significant bit of the byte.
296 * Data elements other than Huffman codes are packed
297 starting with the least-significant bit of the data
298 element.
299 * Huffman codes are packed starting with the most-
300 significant bit of the code.
301
302 In other words, if one were to print out the compressed data as
303 a sequence of bytes, starting with the first byte at the
304 *right* margin and proceeding to the *left*, with the most-
305 significant bit of each byte on the left as usual, one would be
306 able to parse the result from right to left, with fixed-width
307 elements in the correct MSB-to-LSB order and Huffman codes in
308 bit-reversed order (i.e., with the first bit of the code in the
309 relative LSB position).
310
311 3.2. Compressed block format
312
313 3.2.1. Synopsis of prefix and Huffman coding
314
315 Prefix coding represents symbols from an a priori known
316 alphabet by bit sequences (codes), one code for each symbol, in
317 a manner such that different symbols may be represented by bit
318 sequences of different lengths, but a parser can always parse
319 an encoded string unambiguously symbol-by-symbol.
320
321 We define a prefix code in terms of a binary tree in which the
322 two edges descending from each non-leaf node are labeled 0 and
323 1 and in which the leaf nodes correspond one-for-one with (are
324 labeled with) the symbols of the alphabet; then the code for a
325 symbol is the sequence of 0's and 1's on the edges leading from
326 the root to the leaf labeled with that symbol. For example:
327
328
329
330
331
332
333
334
335
336
337
338Deutsch Informational [Page 6]
339
340RFC 1951 DEFLATE Compressed Data Format Specification May 1996
341
342
343 /\ Symbol Code
344 0 1 ------ ----
345 / \ A 00
346 /\ B B 1
347 0 1 C 011
348 / \ D 010
349 A /\
350 0 1
351 / \
352 D C
353
354 A parser can decode the next symbol from an encoded input
355 stream by walking down the tree from the root, at each step
356 choosing the edge corresponding to the next input bit.
357
358 Given an alphabet with known symbol frequencies, the Huffman
359 algorithm allows the construction of an optimal prefix code
360 (one which represents strings with those symbol frequencies
361 using the fewest bits of any possible prefix codes for that
362 alphabet). Such a code is called a Huffman code. (See
363 reference [1] in Chapter 5, references for additional
364 information on Huffman codes.)
365
366 Note that in the "deflate" format, the Huffman codes for the
367 various alphabets must not exceed certain maximum code lengths.
368 This constraint complicates the algorithm for computing code
369 lengths from symbol frequencies. Again, see Chapter 5,
370 references for details.
371
372 3.2.2. Use of Huffman coding in the "deflate" format
373
374 The Huffman codes used for each alphabet in the "deflate"
375 format have two additional rules:
376
377 * All codes of a given bit length have lexicographically
378 consecutive values, in the same order as the symbols
379 they represent;
380
381 * Shorter codes lexicographically precede longer codes.
382
383
384
385
386
387
388
389
390
391
392
393
394Deutsch Informational [Page 7]
395
396RFC 1951 DEFLATE Compressed Data Format Specification May 1996
397
398
399 We could recode the example above to follow this rule as
400 follows, assuming that the order of the alphabet is ABCD:
401
402 Symbol Code
403 ------ ----
404 A 10
405 B 0
406 C 110
407 D 111
408
409 I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are
410 lexicographically consecutive.
411
412 Given this rule, we can define the Huffman code for an alphabet
413 just by giving the bit lengths of the codes for each symbol of
414 the alphabet in order; this is sufficient to determine the
415 actual codes. In our example, the code is completely defined
416 by the sequence of bit lengths (2, 1, 3, 3). The following
417 algorithm generates the codes as integers, intended to be read
418 from most- to least-significant bit. The code lengths are
419 initially in tree[I].Len; the codes are produced in
420 tree[I].Code.
421
422 1) Count the number of codes for each code length. Let
423 bl_count[N] be the number of codes of length N, N >= 1.
424
425 2) Find the numerical value of the smallest code for each
426 code length:
427
428 code = 0;
429 bl_count[0] = 0;
430 for (bits = 1; bits <= MAX_BITS; bits++) {
431 code = (code + bl_count[bits-1]) << 1;
432 next_code[bits] = code;
433 }
434
435 3) Assign numerical values to all codes, using consecutive
436 values for all codes of the same length with the base
437 values determined at step 2. Codes that are never used
438 (which have a bit length of zero) must not be assigned a
439 value.
440
441 for (n = 0; n <= max_code; n++) {
442 len = tree[n].Len;
443 if (len != 0) {
444 tree[n].Code = next_code[len];
445 next_code[len]++;
446 }
447
448
449
450Deutsch Informational [Page 8]
451
452RFC 1951 DEFLATE Compressed Data Format Specification May 1996
453
454
455 }
456
457 Example:
458
459 Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3,
460 3, 2, 4, 4). After step 1, we have:
461
462 N bl_count[N]
463 - -----------
464 2 1
465 3 5
466 4 2
467
468 Step 2 computes the following next_code values:
469
470 N next_code[N]
471 - ------------
472 1 0
473 2 0
474 3 2
475 4 14
476
477 Step 3 produces the following code values:
478
479 Symbol Length Code
480 ------ ------ ----
481 A 3 010
482 B 3 011
483 C 3 100
484 D 3 101
485 E 3 110
486 F 2 00
487 G 4 1110
488 H 4 1111
489
490 3.2.3. Details of block format
491
492 Each block of compressed data begins with 3 header bits
493 containing the following data:
494
495 first bit BFINAL
496 next 2 bits BTYPE
497
498 Note that the header bits do not necessarily begin on a byte
499 boundary, since a block does not necessarily occupy an integral
500 number of bytes.
501
502
503
504
505
506Deutsch Informational [Page 9]
507
508RFC 1951 DEFLATE Compressed Data Format Specification May 1996
509
510
511 BFINAL is set if and only if this is the last block of the data
512 set.
513
514 BTYPE specifies how the data are compressed, as follows:
515
516 00 - no compression
517 01 - compressed with fixed Huffman codes
518 10 - compressed with dynamic Huffman codes
519 11 - reserved (error)
520
521 The only difference between the two compressed cases is how the
522 Huffman codes for the literal/length and distance alphabets are
523 defined.
524
525 In all cases, the decoding algorithm for the actual data is as
526 follows:
527
528 do
529 read block header from input stream.
530 if stored with no compression
531 skip any remaining bits in current partially
532 processed byte
533 read LEN and NLEN (see next section)
534 copy LEN bytes of data to output
535 otherwise
536 if compressed with dynamic Huffman codes
537 read representation of code trees (see
538 subsection below)
539 loop (until end of block code recognized)
540 decode literal/length value from input stream
541 if value < 256
542 copy value (literal byte) to output stream
543 otherwise
544 if value = end of block (256)
545 break from loop
546 otherwise (value = 257..285)
547 decode distance from input stream
548
549 move backwards distance bytes in the output
550 stream, and copy length bytes from this
551 position to the output stream.
552 end loop
553 while not last block
554
555 Note that a duplicated string reference may refer to a string
556 in a previous block; i.e., the backward distance may cross one
557 or more block boundaries. However a distance cannot refer past
558 the beginning of the output stream. (An application using a
559
560
561
562Deutsch Informational [Page 10]
563
564RFC 1951 DEFLATE Compressed Data Format Specification May 1996
565
566
567 preset dictionary might discard part of the output stream; a
568 distance can refer to that part of the output stream anyway)
569 Note also that the referenced string may overlap the current
570 position; for example, if the last 2 bytes decoded have values
571 X and Y, a string reference with <length = 5, distance = 2>
572 adds X,Y,X,Y,X to the output stream.
573
574 We now specify each compression method in turn.
575
576 3.2.4. Non-compressed blocks (BTYPE=00)
577
578 Any bits of input up to the next byte boundary are ignored.
579 The rest of the block consists of the following information:
580
581 0 1 2 3 4...
582 +---+---+---+---+================================+
583 | LEN | NLEN |... LEN bytes of literal data...|
584 +---+---+---+---+================================+
585
586 LEN is the number of data bytes in the block. NLEN is the
587 one's complement of LEN.
588
589 3.2.5. Compressed blocks (length and distance codes)
590
591 As noted above, encoded data blocks in the "deflate" format
592 consist of sequences of symbols drawn from three conceptually
593 distinct alphabets: either literal bytes, from the alphabet of
594 byte values (0..255), or <length, backward distance> pairs,
595 where the length is drawn from (3..258) and the distance is
596 drawn from (1..32,768). In fact, the literal and length
597 alphabets are merged into a single alphabet (0..285), where
598 values 0..255 represent literal bytes, the value 256 indicates
599 end-of-block, and values 257..285 represent length codes
600 (possibly in conjunction with extra bits following the symbol
601 code) as follows:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618Deutsch Informational [Page 11]
619
620RFC 1951 DEFLATE Compressed Data Format Specification May 1996
621
622
623 Extra Extra Extra
624 Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
625 ---- ---- ------ ---- ---- ------- ---- ---- -------
626 257 0 3 267 1 15,16 277 4 67-82
627 258 0 4 268 1 17,18 278 4 83-98
628 259 0 5 269 2 19-22 279 4 99-114
629 260 0 6 270 2 23-26 280 4 115-130
630 261 0 7 271 2 27-30 281 5 131-162
631 262 0 8 272 2 31-34 282 5 163-194
632 263 0 9 273 3 35-42 283 5 195-226
633 264 0 10 274 3 43-50 284 5 227-257
634 265 1 11,12 275 3 51-58 285 0 258
635 266 1 13,14 276 3 59-66
636
637 The extra bits should be interpreted as a machine integer
638 stored with the most-significant bit first, e.g., bits 1110
639 represent the value 14.
640
641 Extra Extra Extra
642 Code Bits Dist Code Bits Dist Code Bits Distance
643 ---- ---- ---- ---- ---- ------ ---- ---- --------
644 0 0 1 10 4 33-48 20 9 1025-1536
645 1 0 2 11 4 49-64 21 9 1537-2048
646 2 0 3 12 5 65-96 22 10 2049-3072
647 3 0 4 13 5 97-128 23 10 3073-4096
648 4 1 5,6 14 6 129-192 24 11 4097-6144
649 5 1 7,8 15 6 193-256 25 11 6145-8192
650 6 2 9-12 16 7 257-384 26 12 8193-12288
651 7 2 13-16 17 7 385-512 27 12 12289-16384
652 8 3 17-24 18 8 513-768 28 13 16385-24576
653 9 3 25-32 19 8 769-1024 29 13 24577-32768
654
655 3.2.6. Compression with fixed Huffman codes (BTYPE=01)
656
657 The Huffman codes for the two alphabets are fixed, and are not
658 represented explicitly in the data. The Huffman code lengths
659 for the literal/length alphabet are:
660
661 Lit Value Bits Codes
662 --------- ---- -----
663 0 - 143 8 00110000 through
664 10111111
665 144 - 255 9 110010000 through
666 111111111
667 256 - 279 7 0000000 through
668 0010111
669 280 - 287 8 11000000 through
670 11000111
671
672
673
674Deutsch Informational [Page 12]
675
676RFC 1951 DEFLATE Compressed Data Format Specification May 1996
677
678
679 The code lengths are sufficient to generate the actual codes,
680 as described above; we show the codes in the table for added
681 clarity. Literal/length values 286-287 will never actually
682 occur in the compressed data, but participate in the code
683 construction.
684
685 Distance codes 0-31 are represented by (fixed-length) 5-bit
686 codes, with possible additional bits as shown in the table
687 shown in Paragraph 3.2.5, above. Note that distance codes 30-
688 31 will never actually occur in the compressed data.
689
690 3.2.7. Compression with dynamic Huffman codes (BTYPE=10)
691
692 The Huffman codes for the two alphabets appear in the block
693 immediately after the header bits and before the actual
694 compressed data, first the literal/length code and then the
695 distance code. Each code is defined by a sequence of code
696 lengths, as discussed in Paragraph 3.2.2, above. For even
697 greater compactness, the code length sequences themselves are
698 compressed using a Huffman code. The alphabet for code lengths
699 is as follows:
700
701 0 - 15: Represent code lengths of 0 - 15
702 16: Copy the previous code length 3 - 6 times.
703 The next 2 bits indicate repeat length
704 (0 = 3, ... , 3 = 6)
705 Example: Codes 8, 16 (+2 bits 11),
706 16 (+2 bits 10) will expand to
707 12 code lengths of 8 (1 + 6 + 5)
708 17: Repeat a code length of 0 for 3 - 10 times.
709 (3 bits of length)
710 18: Repeat a code length of 0 for 11 - 138 times
711 (7 bits of length)
712
713 A code length of 0 indicates that the corresponding symbol in
714 the literal/length or distance alphabet will not occur in the
715 block, and should not participate in the Huffman code
716 construction algorithm given earlier. If only one distance
717 code is used, it is encoded using one bit, not zero bits; in
718 this case there is a single code length of one, with one unused
719 code. One distance code of zero bits means that there are no
720 distance codes used at all (the data is all literals).
721
722 We can now define the format of the block:
723
724 5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286)
725 5 Bits: HDIST, # of Distance codes - 1 (1 - 32)
726 4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19)
727
728
729
730Deutsch Informational [Page 13]
731
732RFC 1951 DEFLATE Compressed Data Format Specification May 1996
733
734
735 (HCLEN + 4) x 3 bits: code lengths for the code length
736 alphabet given just above, in the order: 16, 17, 18,
737 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
738
739 These code lengths are interpreted as 3-bit integers
740 (0-7); as above, a code length of 0 means the
741 corresponding symbol (literal/length or distance code
742 length) is not used.
743
744 HLIT + 257 code lengths for the literal/length alphabet,
745 encoded using the code length Huffman code
746
747 HDIST + 1 code lengths for the distance alphabet,
748 encoded using the code length Huffman code
749
750 The actual compressed data of the block,
751 encoded using the literal/length and distance Huffman
752 codes
753
754 The literal/length symbol 256 (end of data),
755 encoded using the literal/length Huffman code
756
757 The code length repeat codes can cross from HLIT + 257 to the
758 HDIST + 1 code lengths. In other words, all code lengths form
759 a single sequence of HLIT + HDIST + 258 values.
760
761 3.3. Compliance
762
763 A compressor may limit further the ranges of values specified in
764 the previous section and still be compliant; for example, it may
765 limit the range of backward pointers to some value smaller than
766 32K. Similarly, a compressor may limit the size of blocks so that
767 a compressible block fits in memory.
768
769 A compliant decompressor must accept the full range of possible
770 values defined in the previous section, and must accept blocks of
771 arbitrary size.
772
7734. Compression algorithm details
774
775 While it is the intent of this document to define the "deflate"
776 compressed data format without reference to any particular
777 compression algorithm, the format is related to the compressed
778 formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below);
779 since many variations of LZ77 are patented, it is strongly
780 recommended that the implementor of a compressor follow the general
781 algorithm presented here, which is known not to be patented per se.
782 The material in this section is not part of the definition of the
783
784
785
786Deutsch Informational [Page 14]
787
788RFC 1951 DEFLATE Compressed Data Format Specification May 1996
789
790
791 specification per se, and a compressor need not follow it in order to
792 be compliant.
793
794 The compressor terminates a block when it determines that starting a
795 new block with fresh trees would be useful, or when the block size
796 fills up the compressor's block buffer.
797
798 The compressor uses a chained hash table to find duplicated strings,
799 using a hash function that operates on 3-byte sequences. At any
800 given point during compression, let XYZ be the next 3 input bytes to
801 be examined (not necessarily all different, of course). First, the
802 compressor examines the hash chain for XYZ. If the chain is empty,
803 the compressor simply writes out X as a literal byte and advances one
804 byte in the input. If the hash chain is not empty, indicating that
805 the sequence XYZ (or, if we are unlucky, some other 3 bytes with the
806 same hash function value) has occurred recently, the compressor
807 compares all strings on the XYZ hash chain with the actual input data
808 sequence starting at the current point, and selects the longest
809 match.
810
811 The compressor searches the hash chains starting with the most recent
812 strings, to favor small distances and thus take advantage of the
813 Huffman encoding. The hash chains are singly linked. There are no
814 deletions from the hash chains; the algorithm simply discards matches
815 that are too old. To avoid a worst-case situation, very long hash
816 chains are arbitrarily truncated at a certain length, determined by a
817 run-time parameter.
818
819 To improve overall compression, the compressor optionally defers the
820 selection of matches ("lazy matching"): after a match of length N has
821 been found, the compressor searches for a longer match starting at
822 the next input byte. If it finds a longer match, it truncates the
823 previous match to a length of one (thus producing a single literal
824 byte) and then emits the longer match. Otherwise, it emits the
825 original match, and, as described above, advances N bytes before
826 continuing.
827
828 Run-time parameters also control this "lazy match" procedure. If
829 compression ratio is most important, the compressor attempts a
830 complete second search regardless of the length of the first match.
831 In the normal case, if the current match is "long enough", the
832 compressor reduces the search for a longer match, thus speeding up
833 the process. If speed is most important, the compressor inserts new
834 strings in the hash table only when no match was found, or when the
835 match is not "too long". This degrades the compression ratio but
836 saves time since there are both fewer insertions and fewer searches.
837
838
839
840
841
842Deutsch Informational [Page 15]
843
844RFC 1951 DEFLATE Compressed Data Format Specification May 1996
845
846
8475. References
848
849 [1] Huffman, D. A., "A Method for the Construction of Minimum
850 Redundancy Codes", Proceedings of the Institute of Radio
851 Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101.
852
853 [2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data
854 Compression", IEEE Transactions on Information Theory, Vol. 23,
855 No. 3, pp. 337-343.
856
857 [3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources,
858 available in ftp://ftp.uu.net/pub/archiving/zip/doc/
859
860 [4] Gailly, J.-L., and Adler, M., GZIP documentation and sources,
861 available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/
862
863 [5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix
864 encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169.
865
866 [6] Hirschberg and Lelewer, "Efficient decoding of prefix codes,"
867 Comm. ACM, 33,4, April 1990, pp. 449-459.
868
8696. Security Considerations
870
871 Any data compression method involves the reduction of redundancy in
872 the data. Consequently, any corruption of the data is likely to have
873 severe effects and be difficult to correct. Uncompressed text, on
874 the other hand, will probably still be readable despite the presence
875 of some corrupted bytes.
876
877 It is recommended that systems using this data format provide some
878 means of validating the integrity of the compressed data. See
879 reference [3], for example.
880
8817. Source code
882
883 Source code for a C language implementation of a "deflate" compliant
884 compressor and decompressor is available within the zlib package at
885 ftp://ftp.uu.net/pub/archiving/zip/zlib/.
886
8878. Acknowledgements
888
889 Trademarks cited in this document are the property of their
890 respective owners.
891
892 Phil Katz designed the deflate format. Jean-Loup Gailly and Mark
893 Adler wrote the related software described in this specification.
894 Glenn Randers-Pehrson converted this document to RFC and HTML format.
895
896
897
898Deutsch Informational [Page 16]
899
900RFC 1951 DEFLATE Compressed Data Format Specification May 1996
901
902
9039. Author's Address
904
905 L. Peter Deutsch
906 Aladdin Enterprises
907 203 Santa Margarita Ave.
908 Menlo Park, CA 94025
909
910 Phone: (415) 322-0103 (AM only)
911 FAX: (415) 322-1734
912 EMail: <ghost@aladdin.com>
913
914 Questions about the technical content of this specification can be
915 sent by email to:
916
917 Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
918 Mark Adler <madler@alumni.caltech.edu>
919
920 Editorial comments on this specification can be sent by email to:
921
922 L. Peter Deutsch <ghost@aladdin.com> and
923 Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954Deutsch Informational [Page 17]
955
lib/std/compress/deflate/token.zig deleted-103
......@@ -1,103 +0,0 @@
1// 2 bits: type, can be 0 (literal), 1 (EOF), 2 (Match) or 3 (Unused).
2// 8 bits: xlength (length - MIN_MATCH_LENGTH).
3// 22 bits: xoffset (offset - MIN_OFFSET_SIZE), or literal.
4const length_shift = 22;
5const offset_mask = (1 << length_shift) - 1; // 4_194_303
6const literal_type = 0 << 30; // 0
7pub const match_type = 1 << 30; // 1_073_741_824
8
9// The length code for length X (MIN_MATCH_LENGTH <= X <= MAX_MATCH_LENGTH)
10// is length_codes[length - MIN_MATCH_LENGTH]
11var length_codes = [_]u32{
12 0, 1, 2, 3, 4, 5, 6, 7, 8, 8,
13 9, 9, 10, 10, 11, 11, 12, 12, 12, 12,
14 13, 13, 13, 13, 14, 14, 14, 14, 15, 15,
15 15, 15, 16, 16, 16, 16, 16, 16, 16, 16,
16 17, 17, 17, 17, 17, 17, 17, 17, 18, 18,
17 18, 18, 18, 18, 18, 18, 19, 19, 19, 19,
18 19, 19, 19, 19, 20, 20, 20, 20, 20, 20,
19 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20 21, 21, 21, 21, 21, 21, 21, 21, 21, 21,
21 21, 21, 21, 21, 21, 21, 22, 22, 22, 22,
22 22, 22, 22, 22, 22, 22, 22, 22, 22, 22,
23 22, 22, 23, 23, 23, 23, 23, 23, 23, 23,
24 23, 23, 23, 23, 23, 23, 23, 23, 24, 24,
25 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
26 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
27 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
28 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
29 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
30 25, 25, 25, 25, 25, 25, 25, 25, 25, 25,
31 25, 25, 26, 26, 26, 26, 26, 26, 26, 26,
32 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
33 26, 26, 26, 26, 26, 26, 26, 26, 26, 26,
34 26, 26, 26, 26, 27, 27, 27, 27, 27, 27,
35 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
36 27, 27, 27, 27, 27, 27, 27, 27, 27, 27,
37 27, 27, 27, 27, 27, 28,
38};
39
40var offset_codes = [_]u32{
41 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7,
42 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9,
43 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
44 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11,
45 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
46 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
47 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
48 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13,
49 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
50 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
51 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
52 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
53 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
54 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
55 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
56 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15,
57};
58
59pub const Token = u32;
60
61// Convert a literal into a literal token.
62pub fn literalToken(lit: u32) Token {
63 return literal_type + lit;
64}
65
66// Convert a < xlength, xoffset > pair into a match token.
67pub fn matchToken(xlength: u32, xoffset: u32) Token {
68 return match_type + (xlength << length_shift) + xoffset;
69}
70
71// Returns the literal of a literal token
72pub fn literal(t: Token) u32 {
73 return @as(u32, @intCast(t - literal_type));
74}
75
76// Returns the extra offset of a match token
77pub fn offset(t: Token) u32 {
78 return @as(u32, @intCast(t)) & offset_mask;
79}
80
81pub fn length(t: Token) u32 {
82 return @as(u32, @intCast((t - match_type) >> length_shift));
83}
84
85pub fn lengthCode(len: u32) u32 {
86 return length_codes[len];
87}
88
89// Returns the offset code corresponding to a specific offset
90pub fn offsetCode(off: u32) u32 {
91 if (off < @as(u32, @intCast(offset_codes.len))) {
92 return offset_codes[off];
93 }
94 if (off >> 7 < @as(u32, @intCast(offset_codes.len))) {
95 return offset_codes[off >> 7] + 14;
96 }
97 return offset_codes[off >> 14] + 28;
98}
99
100test {
101 const std = @import("std");
102 try std.testing.expectEqual(@as(Token, 3_401_581_099), matchToken(555, 555));
103}
lib/std/compress/gzip.zig deleted-382
......@@ -1,382 +0,0 @@
1//
2// Compressor/Decompressor for GZIP data streams (RFC1952)
3
4const std = @import("../std.zig");
5const io = std.io;
6const fs = std.fs;
7const testing = std.testing;
8const mem = std.mem;
9const deflate = @import("deflate.zig");
10
11const magic = &[2]u8{ 0x1f, 0x8b };
12
13// Flags for the FLG field in the header
14const FTEXT = 1 << 0;
15const FHCRC = 1 << 1;
16const FEXTRA = 1 << 2;
17const FNAME = 1 << 3;
18const FCOMMENT = 1 << 4;
19
20const max_string_len = 1024;
21
22pub const Header = struct {
23 extra: ?[]const u8 = null,
24 filename: ?[]const u8 = null,
25 comment: ?[]const u8 = null,
26 modification_time: u32 = 0,
27 operating_system: u8 = 255,
28};
29
30pub fn Decompress(comptime ReaderType: type) type {
31 return struct {
32 const Self = @This();
33
34 pub const Error = ReaderType.Error ||
35 deflate.Decompressor(ReaderType).Error ||
36 error{ CorruptedData, WrongChecksum };
37 pub const Reader = io.Reader(*Self, Error, read);
38
39 allocator: mem.Allocator,
40 inflater: deflate.Decompressor(ReaderType),
41 in_reader: ReaderType,
42 hasher: std.hash.Crc32,
43 read_amt: u32,
44
45 info: Header,
46
47 fn init(allocator: mem.Allocator, in_reader: ReaderType) !Self {
48 var hasher = std.compress.hashedReader(in_reader, std.hash.Crc32.init());
49 const hashed_reader = hasher.reader();
50
51 // gzip header format is specified in RFC1952
52 const header = try hashed_reader.readBytesNoEof(10);
53
54 // Check the ID1/ID2 fields
55 if (!std.mem.eql(u8, header[0..2], magic))
56 return error.BadHeader;
57
58 const CM = header[2];
59 // The CM field must be 8 to indicate the use of DEFLATE
60 if (CM != 8) return error.InvalidCompression;
61 // Flags
62 const FLG = header[3];
63 // Modification time, as a Unix timestamp.
64 // If zero there's no timestamp available.
65 const MTIME = mem.readInt(u32, header[4..8], .little);
66 // Extra flags
67 const XFL = header[8];
68 // Operating system where the compression took place
69 const OS = header[9];
70 _ = XFL;
71
72 const extra = if (FLG & FEXTRA != 0) blk: {
73 const len = try hashed_reader.readInt(u16, .little);
74 const tmp_buf = try allocator.alloc(u8, len);
75 errdefer allocator.free(tmp_buf);
76
77 try hashed_reader.readNoEof(tmp_buf);
78 break :blk tmp_buf;
79 } else null;
80 errdefer if (extra) |p| allocator.free(p);
81
82 const filename = if (FLG & FNAME != 0)
83 try hashed_reader.readUntilDelimiterAlloc(allocator, 0, max_string_len)
84 else
85 null;
86 errdefer if (filename) |p| allocator.free(p);
87
88 const comment = if (FLG & FCOMMENT != 0)
89 try hashed_reader.readUntilDelimiterAlloc(allocator, 0, max_string_len)
90 else
91 null;
92 errdefer if (comment) |p| allocator.free(p);
93
94 if (FLG & FHCRC != 0) {
95 const hash = try in_reader.readInt(u16, .little);
96 if (hash != @as(u16, @truncate(hasher.hasher.final())))
97 return error.WrongChecksum;
98 }
99
100 return .{
101 .allocator = allocator,
102 .inflater = try deflate.decompressor(allocator, in_reader, null),
103 .in_reader = in_reader,
104 .hasher = std.hash.Crc32.init(),
105 .info = .{
106 .filename = filename,
107 .comment = comment,
108 .extra = extra,
109 .modification_time = MTIME,
110 .operating_system = OS,
111 },
112 .read_amt = 0,
113 };
114 }
115
116 pub fn deinit(self: *Self) void {
117 self.inflater.deinit();
118 if (self.info.extra) |extra|
119 self.allocator.free(extra);
120 if (self.info.filename) |filename|
121 self.allocator.free(filename);
122 if (self.info.comment) |comment|
123 self.allocator.free(comment);
124 }
125
126 /// Implements the io.Reader interface
127 pub fn read(self: *Self, buffer: []u8) Error!usize {
128 if (buffer.len == 0)
129 return 0;
130
131 // Read from the compressed stream and update the computed checksum
132 const r = try self.inflater.read(buffer);
133 if (r != 0) {
134 self.hasher.update(buffer[0..r]);
135 self.read_amt +%= @truncate(r);
136 return r;
137 }
138
139 try self.inflater.close();
140
141 // We've reached the end of stream, check if the checksum matches
142 const hash = try self.in_reader.readInt(u32, .little);
143 if (hash != self.hasher.final())
144 return error.WrongChecksum;
145
146 // The ISIZE field is the size of the uncompressed input modulo 2^32
147 const input_size = try self.in_reader.readInt(u32, .little);
148 if (self.read_amt != input_size)
149 return error.CorruptedData;
150
151 return 0;
152 }
153
154 pub fn reader(self: *Self) Reader {
155 return .{ .context = self };
156 }
157 };
158}
159
160pub fn decompress(allocator: mem.Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
161 return Decompress(@TypeOf(reader)).init(allocator, reader);
162}
163
164pub const CompressOptions = struct {
165 header: Header = .{},
166 hash_header: bool = true,
167 level: deflate.Compression = .default_compression,
168};
169
170pub fn Compress(comptime WriterType: type) type {
171 return struct {
172 const Self = @This();
173
174 pub const Error = WriterType.Error ||
175 deflate.Compressor(WriterType).Error;
176 pub const Writer = io.Writer(*Self, Error, write);
177
178 allocator: mem.Allocator,
179 deflater: deflate.Compressor(WriterType),
180 out_writer: WriterType,
181 hasher: std.hash.Crc32,
182 write_amt: u32,
183
184 fn init(allocator: mem.Allocator, out_writer: WriterType, options: CompressOptions) !Self {
185 var hasher = std.compress.hashedWriter(out_writer, std.hash.Crc32.init());
186 const hashed_writer = hasher.writer();
187
188 // ID1/ID2
189 try hashed_writer.writeAll(magic);
190 // CM
191 try hashed_writer.writeByte(8);
192 // Flags
193 try hashed_writer.writeByte(
194 @as(u8, if (options.hash_header) FHCRC else 0) |
195 @as(u8, if (options.header.extra) |_| FEXTRA else 0) |
196 @as(u8, if (options.header.filename) |_| FNAME else 0) |
197 @as(u8, if (options.header.comment) |_| FCOMMENT else 0),
198 );
199 // Modification time
200 try hashed_writer.writeInt(u32, options.header.modification_time, .little);
201 // Extra flags
202 try hashed_writer.writeByte(0);
203 // Operating system
204 try hashed_writer.writeByte(options.header.operating_system);
205
206 if (options.header.extra) |extra| {
207 try hashed_writer.writeInt(u16, @intCast(extra.len), .little);
208 try hashed_writer.writeAll(extra);
209 }
210
211 if (options.header.filename) |filename| {
212 try hashed_writer.writeAll(filename);
213 try hashed_writer.writeByte(0);
214 }
215
216 if (options.header.comment) |comment| {
217 try hashed_writer.writeAll(comment);
218 try hashed_writer.writeByte(0);
219 }
220
221 if (options.hash_header) {
222 try out_writer.writeInt(
223 u16,
224 @truncate(hasher.hasher.final()),
225 .little,
226 );
227 }
228
229 return .{
230 .allocator = allocator,
231 .deflater = try deflate.compressor(allocator, out_writer, .{ .level = options.level }),
232 .out_writer = out_writer,
233 .hasher = std.hash.Crc32.init(),
234 .write_amt = 0,
235 };
236 }
237
238 pub fn deinit(self: *Self) void {
239 self.deflater.deinit();
240 }
241
242 /// Implements the io.Writer interface
243 pub fn write(self: *Self, buffer: []const u8) Error!usize {
244 if (buffer.len == 0)
245 return 0;
246
247 // Write to the compressed stream and update the computed checksum
248 const r = try self.deflater.write(buffer);
249 self.hasher.update(buffer[0..r]);
250 self.write_amt +%= @truncate(r);
251 return r;
252 }
253
254 pub fn writer(self: *Self) Writer {
255 return .{ .context = self };
256 }
257
258 pub fn flush(self: *Self) Error!void {
259 try self.deflater.flush();
260 }
261
262 pub fn close(self: *Self) Error!void {
263 try self.deflater.close();
264 try self.out_writer.writeInt(u32, self.hasher.final(), .little);
265 try self.out_writer.writeInt(u32, self.write_amt, .little);
266 }
267 };
268}
269
270pub fn compress(allocator: mem.Allocator, writer: anytype, options: CompressOptions) !Compress(@TypeOf(writer)) {
271 return Compress(@TypeOf(writer)).init(allocator, writer, options);
272}
273
274fn testReader(expected: []const u8, data: []const u8) !void {
275 var in_stream = io.fixedBufferStream(data);
276
277 var gzip_stream = try decompress(testing.allocator, in_stream.reader());
278 defer gzip_stream.deinit();
279
280 // Read and decompress the whole file
281 const buf = try gzip_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
282 defer testing.allocator.free(buf);
283
284 // Check against the reference
285 try testing.expectEqualSlices(u8, expected, buf);
286}
287
288fn testWriter(expected: []const u8, data: []const u8, options: CompressOptions) !void {
289 var actual = std.ArrayList(u8).init(testing.allocator);
290 defer actual.deinit();
291
292 var gzip_stream = try compress(testing.allocator, actual.writer(), options);
293 defer gzip_stream.deinit();
294
295 // Write and compress the whole file
296 try gzip_stream.writer().writeAll(data);
297 try gzip_stream.close();
298
299 // Check against the reference
300 try testing.expectEqualSlices(u8, expected, actual.items);
301}
302
303// All the test cases are obtained by compressing the RFC1952 text
304//
305// https://tools.ietf.org/rfc/rfc1952.txt length=25037 bytes
306// SHA256=164ef0897b4cbec63abf1b57f069f3599bd0fb7c72c2a4dee21bd7e03ec9af67
307test "compressed data" {
308 const plain = @embedFile("testdata/rfc1952.txt");
309 const compressed = @embedFile("testdata/rfc1952.txt.gz");
310 try testReader(plain, compressed);
311 try testWriter(compressed, plain, .{
312 .header = .{
313 .filename = "rfc1952.txt",
314 .modification_time = 1706533053,
315 .operating_system = 3,
316 },
317 });
318}
319
320test "sanity checks" {
321 // Truncated header
322 try testing.expectError(
323 error.EndOfStream,
324 testReader(undefined, &[_]u8{ 0x1f, 0x8B }),
325 );
326 // Wrong CM
327 try testing.expectError(
328 error.InvalidCompression,
329 testReader(undefined, &[_]u8{
330 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
331 0x00, 0x03,
332 }),
333 );
334 // Wrong checksum
335 try testing.expectError(
336 error.WrongChecksum,
337 testReader(undefined, &[_]u8{
338 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
339 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
340 0x00, 0x00, 0x00, 0x00,
341 }),
342 );
343 // Truncated checksum
344 try testing.expectError(
345 error.EndOfStream,
346 testReader(undefined, &[_]u8{
347 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
348 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
349 }),
350 );
351 // Wrong initial size
352 try testing.expectError(
353 error.CorruptedData,
354 testReader(undefined, &[_]u8{
355 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
356 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
357 0x00, 0x00, 0x00, 0x01,
358 }),
359 );
360 // Truncated initial size field
361 try testing.expectError(
362 error.EndOfStream,
363 testReader(undefined, &[_]u8{
364 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
365 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
366 0x00, 0x00, 0x00,
367 }),
368 );
369}
370
371test "header checksum" {
372 try testReader("", &[_]u8{
373 // GZIP header
374 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00,
375
376 // header.FHCRC (should cover entire header)
377 0x99, 0xd6,
378
379 // GZIP data
380 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
381 });
382}
lib/std/compress/testdata/rfc1951.txt deleted-955
......@@ -1,955 +0,0 @@
1
2
3
4
5
6
7Network Working Group P. Deutsch
8Request for Comments: 1951 Aladdin Enterprises
9Category: Informational May 1996
10
11
12 DEFLATE Compressed Data Format Specification version 1.3
13
14Status of This Memo
15
16 This memo provides information for the Internet community. This memo
17 does not specify an Internet standard of any kind. Distribution of
18 this memo is unlimited.
19
20IESG Note:
21
22 The IESG takes no position on the validity of any Intellectual
23 Property Rights statements contained in this document.
24
25Notices
26
27 Copyright (c) 1996 L. Peter Deutsch
28
29 Permission is granted to copy and distribute this document for any
30 purpose and without charge, including translations into other
31 languages and incorporation into compilations, provided that the
32 copyright notice and this notice are preserved, and that any
33 substantive changes or deletions from the original are clearly
34 marked.
35
36 A pointer to the latest version of this and related documentation in
37 HTML format can be found at the URL
38 <ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40Abstract
41
42 This specification defines a lossless compressed data format that
43 compresses data using a combination of the LZ77 algorithm and Huffman
44 coding, with efficiency comparable to the best currently available
45 general-purpose compression methods. The data can be produced or
46 consumed, even for an arbitrarily long sequentially presented input
47 data stream, using only an a priori bounded amount of intermediate
48 storage. The format can be implemented readily in a manner not
49 covered by patents.
50
51
52
53
54
55
56
57
58Deutsch Informational [Page 1]
59
60RFC 1951 DEFLATE Compressed Data Format Specification May 1996
61
62
63Table of Contents
64
65 1. Introduction ................................................... 2
66 1.1. Purpose ................................................... 2
67 1.2. Intended audience ......................................... 3
68 1.3. Scope ..................................................... 3
69 1.4. Compliance ................................................ 3
70 1.5. Definitions of terms and conventions used ................ 3
71 1.6. Changes from previous versions ............................ 4
72 2. Compressed representation overview ............................. 4
73 3. Detailed specification ......................................... 5
74 3.1. Overall conventions ....................................... 5
75 3.1.1. Packing into bytes .................................. 5
76 3.2. Compressed block format ................................... 6
77 3.2.1. Synopsis of prefix and Huffman coding ............... 6
78 3.2.2. Use of Huffman coding in the "deflate" format ....... 7
79 3.2.3. Details of block format ............................. 9
80 3.2.4. Non-compressed blocks (BTYPE=00) ................... 11
81 3.2.5. Compressed blocks (length and distance codes) ...... 11
82 3.2.6. Compression with fixed Huffman codes (BTYPE=01) .... 12
83 3.2.7. Compression with dynamic Huffman codes (BTYPE=10) .. 13
84 3.3. Compliance ............................................... 14
85 4. Compression algorithm details ................................. 14
86 5. References .................................................... 16
87 6. Security Considerations ....................................... 16
88 7. Source code ................................................... 16
89 8. Acknowledgements .............................................. 16
90 9. Author's Address .............................................. 17
91
921. Introduction
93
94 1.1. Purpose
95
96 The purpose of this specification is to define a lossless
97 compressed data format that:
98 * Is independent of CPU type, operating system, file system,
99 and character set, and hence can be used for interchange;
100 * Can be produced or consumed, even for an arbitrarily long
101 sequentially presented input data stream, using only an a
102 priori bounded amount of intermediate storage, and hence
103 can be used in data communications or similar structures
104 such as Unix filters;
105 * Compresses data with efficiency comparable to the best
106 currently available general-purpose compression methods,
107 and in particular considerably better than the "compress"
108 program;
109 * Can be implemented readily in a manner not covered by
110 patents, and hence can be practiced freely;
111
112
113
114Deutsch Informational [Page 2]
115
116RFC 1951 DEFLATE Compressed Data Format Specification May 1996
117
118
119 * Is compatible with the file format produced by the current
120 widely used gzip utility, in that conforming decompressors
121 will be able to read data produced by the existing gzip
122 compressor.
123
124 The data format defined by this specification does not attempt to:
125
126 * Allow random access to compressed data;
127 * Compress specialized data (e.g., raster graphics) as well
128 as the best currently available specialized algorithms.
129
130 A simple counting argument shows that no lossless compression
131 algorithm can compress every possible input data set. For the
132 format defined here, the worst case expansion is 5 bytes per 32K-
133 byte block, i.e., a size increase of 0.015% for large data sets.
134 English text usually compresses by a factor of 2.5 to 3;
135 executable files usually compress somewhat less; graphical data
136 such as raster images may compress much more.
137
138 1.2. Intended audience
139
140 This specification is intended for use by implementors of software
141 to compress data into "deflate" format and/or decompress data from
142 "deflate" format.
143
144 The text of the specification assumes a basic background in
145 programming at the level of bits and other primitive data
146 representations. Familiarity with the technique of Huffman coding
147 is helpful but not required.
148
149 1.3. Scope
150
151 The specification specifies a method for representing a sequence
152 of bytes as a (usually shorter) sequence of bits, and a method for
153 packing the latter bit sequence into bytes.
154
155 1.4. Compliance
156
157 Unless otherwise indicated below, a compliant decompressor must be
158 able to accept and decompress any data set that conforms to all
159 the specifications presented here; a compliant compressor must
160 produce data sets that conform to all the specifications presented
161 here.
162
163 1.5. Definitions of terms and conventions used
164
165 Byte: 8 bits stored or transmitted as a unit (same as an octet).
166 For this specification, a byte is exactly 8 bits, even on machines
167
168
169
170Deutsch Informational [Page 3]
171
172RFC 1951 DEFLATE Compressed Data Format Specification May 1996
173
174
175 which store a character on a number of bits different from eight.
176 See below, for the numbering of bits within a byte.
177
178 String: a sequence of arbitrary bytes.
179
180 1.6. Changes from previous versions
181
182 There have been no technical changes to the deflate format since
183 version 1.1 of this specification. In version 1.2, some
184 terminology was changed. Version 1.3 is a conversion of the
185 specification to RFC style.
186
1872. Compressed representation overview
188
189 A compressed data set consists of a series of blocks, corresponding
190 to successive blocks of input data. The block sizes are arbitrary,
191 except that non-compressible blocks are limited to 65,535 bytes.
192
193 Each block is compressed using a combination of the LZ77 algorithm
194 and Huffman coding. The Huffman trees for each block are independent
195 of those for previous or subsequent blocks; the LZ77 algorithm may
196 use a reference to a duplicated string occurring in a previous block,
197 up to 32K input bytes before.
198
199 Each block consists of two parts: a pair of Huffman code trees that
200 describe the representation of the compressed data part, and a
201 compressed data part. (The Huffman trees themselves are compressed
202 using Huffman encoding.) The compressed data consists of a series of
203 elements of two types: literal bytes (of strings that have not been
204 detected as duplicated within the previous 32K input bytes), and
205 pointers to duplicated strings, where a pointer is represented as a
206 pair <length, backward distance>. The representation used in the
207 "deflate" format limits distances to 32K bytes and lengths to 258
208 bytes, but does not limit the size of a block, except for
209 uncompressible blocks, which are limited as noted above.
210
211 Each type of value (literals, distances, and lengths) in the
212 compressed data is represented using a Huffman code, using one code
213 tree for literals and lengths and a separate code tree for distances.
214 The code trees for each block appear in a compact form just before
215 the compressed data for that block.
216
217
218
219
220
221
222
223
224
225
226Deutsch Informational [Page 4]
227
228RFC 1951 DEFLATE Compressed Data Format Specification May 1996
229
230
2313. Detailed specification
232
233 3.1. Overall conventions In the diagrams below, a box like this:
234
235 +---+
236 | | <-- the vertical bars might be missing
237 +---+
238
239 represents one byte; a box like this:
240
241 +==============+
242 | |
243 +==============+
244
245 represents a variable number of bytes.
246
247 Bytes stored within a computer do not have a "bit order", since
248 they are always treated as a unit. However, a byte considered as
249 an integer between 0 and 255 does have a most- and least-
250 significant bit, and since we write numbers with the most-
251 significant digit on the left, we also write bytes with the most-
252 significant bit on the left. In the diagrams below, we number the
253 bits of a byte so that bit 0 is the least-significant bit, i.e.,
254 the bits are numbered:
255
256 +--------+
257 |76543210|
258 +--------+
259
260 Within a computer, a number may occupy multiple bytes. All
261 multi-byte numbers in the format described here are stored with
262 the least-significant byte first (at the lower memory address).
263 For example, the decimal number 520 is stored as:
264
265 0 1
266 +--------+--------+
267 |00001000|00000010|
268 +--------+--------+
269 ^ ^
270 | |
271 | + more significant byte = 2 x 256
272 + less significant byte = 8
273
274 3.1.1. Packing into bytes
275
276 This document does not address the issue of the order in which
277 bits of a byte are transmitted on a bit-sequential medium,
278 since the final data format described here is byte- rather than
279
280
281
282Deutsch Informational [Page 5]
283
284RFC 1951 DEFLATE Compressed Data Format Specification May 1996
285
286
287 bit-oriented. However, we describe the compressed block format
288 in below, as a sequence of data elements of various bit
289 lengths, not a sequence of bytes. We must therefore specify
290 how to pack these data elements into bytes to form the final
291 compressed byte sequence:
292
293 * Data elements are packed into bytes in order of
294 increasing bit number within the byte, i.e., starting
295 with the least-significant bit of the byte.
296 * Data elements other than Huffman codes are packed
297 starting with the least-significant bit of the data
298 element.
299 * Huffman codes are packed starting with the most-
300 significant bit of the code.
301
302 In other words, if one were to print out the compressed data as
303 a sequence of bytes, starting with the first byte at the
304 *right* margin and proceeding to the *left*, with the most-
305 significant bit of each byte on the left as usual, one would be
306 able to parse the result from right to left, with fixed-width
307 elements in the correct MSB-to-LSB order and Huffman codes in
308 bit-reversed order (i.e., with the first bit of the code in the
309 relative LSB position).
310
311 3.2. Compressed block format
312
313 3.2.1. Synopsis of prefix and Huffman coding
314
315 Prefix coding represents symbols from an a priori known
316 alphabet by bit sequences (codes), one code for each symbol, in
317 a manner such that different symbols may be represented by bit
318 sequences of different lengths, but a parser can always parse
319 an encoded string unambiguously symbol-by-symbol.
320
321 We define a prefix code in terms of a binary tree in which the
322 two edges descending from each non-leaf node are labeled 0 and
323 1 and in which the leaf nodes correspond one-for-one with (are
324 labeled with) the symbols of the alphabet; then the code for a
325 symbol is the sequence of 0's and 1's on the edges leading from
326 the root to the leaf labeled with that symbol. For example:
327
328
329
330
331
332
333
334
335
336
337
338Deutsch Informational [Page 6]
339
340RFC 1951 DEFLATE Compressed Data Format Specification May 1996
341
342
343 /\ Symbol Code
344 0 1 ------ ----
345 / \ A 00
346 /\ B B 1
347 0 1 C 011
348 / \ D 010
349 A /\
350 0 1
351 / \
352 D C
353
354 A parser can decode the next symbol from an encoded input
355 stream by walking down the tree from the root, at each step
356 choosing the edge corresponding to the next input bit.
357
358 Given an alphabet with known symbol frequencies, the Huffman
359 algorithm allows the construction of an optimal prefix code
360 (one which represents strings with those symbol frequencies
361 using the fewest bits of any possible prefix codes for that
362 alphabet). Such a code is called a Huffman code. (See
363 reference [1] in Chapter 5, references for additional
364 information on Huffman codes.)
365
366 Note that in the "deflate" format, the Huffman codes for the
367 various alphabets must not exceed certain maximum code lengths.
368 This constraint complicates the algorithm for computing code
369 lengths from symbol frequencies. Again, see Chapter 5,
370 references for details.
371
372 3.2.2. Use of Huffman coding in the "deflate" format
373
374 The Huffman codes used for each alphabet in the "deflate"
375 format have two additional rules:
376
377 * All codes of a given bit length have lexicographically
378 consecutive values, in the same order as the symbols
379 they represent;
380
381 * Shorter codes lexicographically precede longer codes.
382
383
384
385
386
387
388
389
390
391
392
393
394Deutsch Informational [Page 7]
395
396RFC 1951 DEFLATE Compressed Data Format Specification May 1996
397
398
399 We could recode the example above to follow this rule as
400 follows, assuming that the order of the alphabet is ABCD:
401
402 Symbol Code
403 ------ ----
404 A 10
405 B 0
406 C 110
407 D 111
408
409 I.e., 0 precedes 10 which precedes 11x, and 110 and 111 are
410 lexicographically consecutive.
411
412 Given this rule, we can define the Huffman code for an alphabet
413 just by giving the bit lengths of the codes for each symbol of
414 the alphabet in order; this is sufficient to determine the
415 actual codes. In our example, the code is completely defined
416 by the sequence of bit lengths (2, 1, 3, 3). The following
417 algorithm generates the codes as integers, intended to be read
418 from most- to least-significant bit. The code lengths are
419 initially in tree[I].Len; the codes are produced in
420 tree[I].Code.
421
422 1) Count the number of codes for each code length. Let
423 bl_count[N] be the number of codes of length N, N >= 1.
424
425 2) Find the numerical value of the smallest code for each
426 code length:
427
428 code = 0;
429 bl_count[0] = 0;
430 for (bits = 1; bits <= MAX_BITS; bits++) {
431 code = (code + bl_count[bits-1]) << 1;
432 next_code[bits] = code;
433 }
434
435 3) Assign numerical values to all codes, using consecutive
436 values for all codes of the same length with the base
437 values determined at step 2. Codes that are never used
438 (which have a bit length of zero) must not be assigned a
439 value.
440
441 for (n = 0; n <= max_code; n++) {
442 len = tree[n].Len;
443 if (len != 0) {
444 tree[n].Code = next_code[len];
445 next_code[len]++;
446 }
447
448
449
450Deutsch Informational [Page 8]
451
452RFC 1951 DEFLATE Compressed Data Format Specification May 1996
453
454
455 }
456
457 Example:
458
459 Consider the alphabet ABCDEFGH, with bit lengths (3, 3, 3, 3,
460 3, 2, 4, 4). After step 1, we have:
461
462 N bl_count[N]
463 - -----------
464 2 1
465 3 5
466 4 2
467
468 Step 2 computes the following next_code values:
469
470 N next_code[N]
471 - ------------
472 1 0
473 2 0
474 3 2
475 4 14
476
477 Step 3 produces the following code values:
478
479 Symbol Length Code
480 ------ ------ ----
481 A 3 010
482 B 3 011
483 C 3 100
484 D 3 101
485 E 3 110
486 F 2 00
487 G 4 1110
488 H 4 1111
489
490 3.2.3. Details of block format
491
492 Each block of compressed data begins with 3 header bits
493 containing the following data:
494
495 first bit BFINAL
496 next 2 bits BTYPE
497
498 Note that the header bits do not necessarily begin on a byte
499 boundary, since a block does not necessarily occupy an integral
500 number of bytes.
501
502
503
504
505
506Deutsch Informational [Page 9]
507
508RFC 1951 DEFLATE Compressed Data Format Specification May 1996
509
510
511 BFINAL is set if and only if this is the last block of the data
512 set.
513
514 BTYPE specifies how the data are compressed, as follows:
515
516 00 - no compression
517 01 - compressed with fixed Huffman codes
518 10 - compressed with dynamic Huffman codes
519 11 - reserved (error)
520
521 The only difference between the two compressed cases is how the
522 Huffman codes for the literal/length and distance alphabets are
523 defined.
524
525 In all cases, the decoding algorithm for the actual data is as
526 follows:
527
528 do
529 read block header from input stream.
530 if stored with no compression
531 skip any remaining bits in current partially
532 processed byte
533 read LEN and NLEN (see next section)
534 copy LEN bytes of data to output
535 otherwise
536 if compressed with dynamic Huffman codes
537 read representation of code trees (see
538 subsection below)
539 loop (until end of block code recognized)
540 decode literal/length value from input stream
541 if value < 256
542 copy value (literal byte) to output stream
543 otherwise
544 if value = end of block (256)
545 break from loop
546 otherwise (value = 257..285)
547 decode distance from input stream
548
549 move backwards distance bytes in the output
550 stream, and copy length bytes from this
551 position to the output stream.
552 end loop
553 while not last block
554
555 Note that a duplicated string reference may refer to a string
556 in a previous block; i.e., the backward distance may cross one
557 or more block boundaries. However a distance cannot refer past
558 the beginning of the output stream. (An application using a
559
560
561
562Deutsch Informational [Page 10]
563
564RFC 1951 DEFLATE Compressed Data Format Specification May 1996
565
566
567 preset dictionary might discard part of the output stream; a
568 distance can refer to that part of the output stream anyway)
569 Note also that the referenced string may overlap the current
570 position; for example, if the last 2 bytes decoded have values
571 X and Y, a string reference with <length = 5, distance = 2>
572 adds X,Y,X,Y,X to the output stream.
573
574 We now specify each compression method in turn.
575
576 3.2.4. Non-compressed blocks (BTYPE=00)
577
578 Any bits of input up to the next byte boundary are ignored.
579 The rest of the block consists of the following information:
580
581 0 1 2 3 4...
582 +---+---+---+---+================================+
583 | LEN | NLEN |... LEN bytes of literal data...|
584 +---+---+---+---+================================+
585
586 LEN is the number of data bytes in the block. NLEN is the
587 one's complement of LEN.
588
589 3.2.5. Compressed blocks (length and distance codes)
590
591 As noted above, encoded data blocks in the "deflate" format
592 consist of sequences of symbols drawn from three conceptually
593 distinct alphabets: either literal bytes, from the alphabet of
594 byte values (0..255), or <length, backward distance> pairs,
595 where the length is drawn from (3..258) and the distance is
596 drawn from (1..32,768). In fact, the literal and length
597 alphabets are merged into a single alphabet (0..285), where
598 values 0..255 represent literal bytes, the value 256 indicates
599 end-of-block, and values 257..285 represent length codes
600 (possibly in conjunction with extra bits following the symbol
601 code) as follows:
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618Deutsch Informational [Page 11]
619
620RFC 1951 DEFLATE Compressed Data Format Specification May 1996
621
622
623 Extra Extra Extra
624 Code Bits Length(s) Code Bits Lengths Code Bits Length(s)
625 ---- ---- ------ ---- ---- ------- ---- ---- -------
626 257 0 3 267 1 15,16 277 4 67-82
627 258 0 4 268 1 17,18 278 4 83-98
628 259 0 5 269 2 19-22 279 4 99-114
629 260 0 6 270 2 23-26 280 4 115-130
630 261 0 7 271 2 27-30 281 5 131-162
631 262 0 8 272 2 31-34 282 5 163-194
632 263 0 9 273 3 35-42 283 5 195-226
633 264 0 10 274 3 43-50 284 5 227-257
634 265 1 11,12 275 3 51-58 285 0 258
635 266 1 13,14 276 3 59-66
636
637 The extra bits should be interpreted as a machine integer
638 stored with the most-significant bit first, e.g., bits 1110
639 represent the value 14.
640
641 Extra Extra Extra
642 Code Bits Dist Code Bits Dist Code Bits Distance
643 ---- ---- ---- ---- ---- ------ ---- ---- --------
644 0 0 1 10 4 33-48 20 9 1025-1536
645 1 0 2 11 4 49-64 21 9 1537-2048
646 2 0 3 12 5 65-96 22 10 2049-3072
647 3 0 4 13 5 97-128 23 10 3073-4096
648 4 1 5,6 14 6 129-192 24 11 4097-6144
649 5 1 7,8 15 6 193-256 25 11 6145-8192
650 6 2 9-12 16 7 257-384 26 12 8193-12288
651 7 2 13-16 17 7 385-512 27 12 12289-16384
652 8 3 17-24 18 8 513-768 28 13 16385-24576
653 9 3 25-32 19 8 769-1024 29 13 24577-32768
654
655 3.2.6. Compression with fixed Huffman codes (BTYPE=01)
656
657 The Huffman codes for the two alphabets are fixed, and are not
658 represented explicitly in the data. The Huffman code lengths
659 for the literal/length alphabet are:
660
661 Lit Value Bits Codes
662 --------- ---- -----
663 0 - 143 8 00110000 through
664 10111111
665 144 - 255 9 110010000 through
666 111111111
667 256 - 279 7 0000000 through
668 0010111
669 280 - 287 8 11000000 through
670 11000111
671
672
673
674Deutsch Informational [Page 12]
675
676RFC 1951 DEFLATE Compressed Data Format Specification May 1996
677
678
679 The code lengths are sufficient to generate the actual codes,
680 as described above; we show the codes in the table for added
681 clarity. Literal/length values 286-287 will never actually
682 occur in the compressed data, but participate in the code
683 construction.
684
685 Distance codes 0-31 are represented by (fixed-length) 5-bit
686 codes, with possible additional bits as shown in the table
687 shown in Paragraph 3.2.5, above. Note that distance codes 30-
688 31 will never actually occur in the compressed data.
689
690 3.2.7. Compression with dynamic Huffman codes (BTYPE=10)
691
692 The Huffman codes for the two alphabets appear in the block
693 immediately after the header bits and before the actual
694 compressed data, first the literal/length code and then the
695 distance code. Each code is defined by a sequence of code
696 lengths, as discussed in Paragraph 3.2.2, above. For even
697 greater compactness, the code length sequences themselves are
698 compressed using a Huffman code. The alphabet for code lengths
699 is as follows:
700
701 0 - 15: Represent code lengths of 0 - 15
702 16: Copy the previous code length 3 - 6 times.
703 The next 2 bits indicate repeat length
704 (0 = 3, ... , 3 = 6)
705 Example: Codes 8, 16 (+2 bits 11),
706 16 (+2 bits 10) will expand to
707 12 code lengths of 8 (1 + 6 + 5)
708 17: Repeat a code length of 0 for 3 - 10 times.
709 (3 bits of length)
710 18: Repeat a code length of 0 for 11 - 138 times
711 (7 bits of length)
712
713 A code length of 0 indicates that the corresponding symbol in
714 the literal/length or distance alphabet will not occur in the
715 block, and should not participate in the Huffman code
716 construction algorithm given earlier. If only one distance
717 code is used, it is encoded using one bit, not zero bits; in
718 this case there is a single code length of one, with one unused
719 code. One distance code of zero bits means that there are no
720 distance codes used at all (the data is all literals).
721
722 We can now define the format of the block:
723
724 5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286)
725 5 Bits: HDIST, # of Distance codes - 1 (1 - 32)
726 4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19)
727
728
729
730Deutsch Informational [Page 13]
731
732RFC 1951 DEFLATE Compressed Data Format Specification May 1996
733
734
735 (HCLEN + 4) x 3 bits: code lengths for the code length
736 alphabet given just above, in the order: 16, 17, 18,
737 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15
738
739 These code lengths are interpreted as 3-bit integers
740 (0-7); as above, a code length of 0 means the
741 corresponding symbol (literal/length or distance code
742 length) is not used.
743
744 HLIT + 257 code lengths for the literal/length alphabet,
745 encoded using the code length Huffman code
746
747 HDIST + 1 code lengths for the distance alphabet,
748 encoded using the code length Huffman code
749
750 The actual compressed data of the block,
751 encoded using the literal/length and distance Huffman
752 codes
753
754 The literal/length symbol 256 (end of data),
755 encoded using the literal/length Huffman code
756
757 The code length repeat codes can cross from HLIT + 257 to the
758 HDIST + 1 code lengths. In other words, all code lengths form
759 a single sequence of HLIT + HDIST + 258 values.
760
761 3.3. Compliance
762
763 A compressor may limit further the ranges of values specified in
764 the previous section and still be compliant; for example, it may
765 limit the range of backward pointers to some value smaller than
766 32K. Similarly, a compressor may limit the size of blocks so that
767 a compressible block fits in memory.
768
769 A compliant decompressor must accept the full range of possible
770 values defined in the previous section, and must accept blocks of
771 arbitrary size.
772
7734. Compression algorithm details
774
775 While it is the intent of this document to define the "deflate"
776 compressed data format without reference to any particular
777 compression algorithm, the format is related to the compressed
778 formats produced by LZ77 (Lempel-Ziv 1977, see reference [2] below);
779 since many variations of LZ77 are patented, it is strongly
780 recommended that the implementor of a compressor follow the general
781 algorithm presented here, which is known not to be patented per se.
782 The material in this section is not part of the definition of the
783
784
785
786Deutsch Informational [Page 14]
787
788RFC 1951 DEFLATE Compressed Data Format Specification May 1996
789
790
791 specification per se, and a compressor need not follow it in order to
792 be compliant.
793
794 The compressor terminates a block when it determines that starting a
795 new block with fresh trees would be useful, or when the block size
796 fills up the compressor's block buffer.
797
798 The compressor uses a chained hash table to find duplicated strings,
799 using a hash function that operates on 3-byte sequences. At any
800 given point during compression, let XYZ be the next 3 input bytes to
801 be examined (not necessarily all different, of course). First, the
802 compressor examines the hash chain for XYZ. If the chain is empty,
803 the compressor simply writes out X as a literal byte and advances one
804 byte in the input. If the hash chain is not empty, indicating that
805 the sequence XYZ (or, if we are unlucky, some other 3 bytes with the
806 same hash function value) has occurred recently, the compressor
807 compares all strings on the XYZ hash chain with the actual input data
808 sequence starting at the current point, and selects the longest
809 match.
810
811 The compressor searches the hash chains starting with the most recent
812 strings, to favor small distances and thus take advantage of the
813 Huffman encoding. The hash chains are singly linked. There are no
814 deletions from the hash chains; the algorithm simply discards matches
815 that are too old. To avoid a worst-case situation, very long hash
816 chains are arbitrarily truncated at a certain length, determined by a
817 run-time parameter.
818
819 To improve overall compression, the compressor optionally defers the
820 selection of matches ("lazy matching"): after a match of length N has
821 been found, the compressor searches for a longer match starting at
822 the next input byte. If it finds a longer match, it truncates the
823 previous match to a length of one (thus producing a single literal
824 byte) and then emits the longer match. Otherwise, it emits the
825 original match, and, as described above, advances N bytes before
826 continuing.
827
828 Run-time parameters also control this "lazy match" procedure. If
829 compression ratio is most important, the compressor attempts a
830 complete second search regardless of the length of the first match.
831 In the normal case, if the current match is "long enough", the
832 compressor reduces the search for a longer match, thus speeding up
833 the process. If speed is most important, the compressor inserts new
834 strings in the hash table only when no match was found, or when the
835 match is not "too long". This degrades the compression ratio but
836 saves time since there are both fewer insertions and fewer searches.
837
838
839
840
841
842Deutsch Informational [Page 15]
843
844RFC 1951 DEFLATE Compressed Data Format Specification May 1996
845
846
8475. References
848
849 [1] Huffman, D. A., "A Method for the Construction of Minimum
850 Redundancy Codes", Proceedings of the Institute of Radio
851 Engineers, September 1952, Volume 40, Number 9, pp. 1098-1101.
852
853 [2] Ziv J., Lempel A., "A Universal Algorithm for Sequential Data
854 Compression", IEEE Transactions on Information Theory, Vol. 23,
855 No. 3, pp. 337-343.
856
857 [3] Gailly, J.-L., and Adler, M., ZLIB documentation and sources,
858 available in ftp://ftp.uu.net/pub/archiving/zip/doc/
859
860 [4] Gailly, J.-L., and Adler, M., GZIP documentation and sources,
861 available as gzip-*.tar in ftp://prep.ai.mit.edu/pub/gnu/
862
863 [5] Schwartz, E. S., and Kallick, B. "Generating a canonical prefix
864 encoding." Comm. ACM, 7,3 (Mar. 1964), pp. 166-169.
865
866 [6] Hirschberg and Lelewer, "Efficient decoding of prefix codes,"
867 Comm. ACM, 33,4, April 1990, pp. 449-459.
868
8696. Security Considerations
870
871 Any data compression method involves the reduction of redundancy in
872 the data. Consequently, any corruption of the data is likely to have
873 severe effects and be difficult to correct. Uncompressed text, on
874 the other hand, will probably still be readable despite the presence
875 of some corrupted bytes.
876
877 It is recommended that systems using this data format provide some
878 means of validating the integrity of the compressed data. See
879 reference [3], for example.
880
8817. Source code
882
883 Source code for a C language implementation of a "deflate" compliant
884 compressor and decompressor is available within the zlib package at
885 ftp://ftp.uu.net/pub/archiving/zip/zlib/.
886
8878. Acknowledgements
888
889 Trademarks cited in this document are the property of their
890 respective owners.
891
892 Phil Katz designed the deflate format. Jean-Loup Gailly and Mark
893 Adler wrote the related software described in this specification.
894 Glenn Randers-Pehrson converted this document to RFC and HTML format.
895
896
897
898Deutsch Informational [Page 16]
899
900RFC 1951 DEFLATE Compressed Data Format Specification May 1996
901
902
9039. Author's Address
904
905 L. Peter Deutsch
906 Aladdin Enterprises
907 203 Santa Margarita Ave.
908 Menlo Park, CA 94025
909
910 Phone: (415) 322-0103 (AM only)
911 FAX: (415) 322-1734
912 EMail: <ghost@aladdin.com>
913
914 Questions about the technical content of this specification can be
915 sent by email to:
916
917 Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
918 Mark Adler <madler@alumni.caltech.edu>
919
920 Editorial comments on this specification can be sent by email to:
921
922 L. Peter Deutsch <ghost@aladdin.com> and
923 Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954Deutsch Informational [Page 17]
955
lib/std/compress/testdata/rfc1951.txt.fixed.z.9 deleted
Binary files a/lib/std/compress/testdata/rfc1951.txt.fixed.z.9 and /dev/null differ
lib/std/compress/testdata/rfc1951.txt.z.0 deleted
Binary files a/lib/std/compress/testdata/rfc1951.txt.z.0 and /dev/null differ
lib/std/compress/testdata/rfc1951.txt.z.9 deleted
Binary files a/lib/std/compress/testdata/rfc1951.txt.z.9 and /dev/null differ
lib/std/compress/testdata/rfc1952.txt deleted-675
......@@ -1,675 +0,0 @@
1
2
3
4
5
6
7Network Working Group P. Deutsch
8Request for Comments: 1952 Aladdin Enterprises
9Category: Informational May 1996
10
11
12 GZIP file format specification version 4.3
13
14Status of This Memo
15
16 This memo provides information for the Internet community. This memo
17 does not specify an Internet standard of any kind. Distribution of
18 this memo is unlimited.
19
20IESG Note:
21
22 The IESG takes no position on the validity of any Intellectual
23 Property Rights statements contained in this document.
24
25Notices
26
27 Copyright (c) 1996 L. Peter Deutsch
28
29 Permission is granted to copy and distribute this document for any
30 purpose and without charge, including translations into other
31 languages and incorporation into compilations, provided that the
32 copyright notice and this notice are preserved, and that any
33 substantive changes or deletions from the original are clearly
34 marked.
35
36 A pointer to the latest version of this and related documentation in
37 HTML format can be found at the URL
38 <ftp://ftp.uu.net/graphics/png/documents/zlib/zdoc-index.html>.
39
40Abstract
41
42 This specification defines a lossless compressed data format that is
43 compatible with the widely used GZIP utility. The format includes a
44 cyclic redundancy check value for detecting data corruption. The
45 format presently uses the DEFLATE method of compression but can be
46 easily extended to use other compression methods. The format can be
47 implemented readily in a manner not covered by patents.
48
49
50
51
52
53
54
55
56
57
58Deutsch Informational [Page 1]
59
60RFC 1952 GZIP File Format Specification May 1996
61
62
63Table of Contents
64
65 1. Introduction ................................................... 2
66 1.1. Purpose ................................................... 2
67 1.2. Intended audience ......................................... 3
68 1.3. Scope ..................................................... 3
69 1.4. Compliance ................................................ 3
70 1.5. Definitions of terms and conventions used ................. 3
71 1.6. Changes from previous versions ............................ 3
72 2. Detailed specification ......................................... 4
73 2.1. Overall conventions ....................................... 4
74 2.2. File format ............................................... 5
75 2.3. Member format ............................................. 5
76 2.3.1. Member header and trailer ........................... 6
77 2.3.1.1. Extra field ................................... 8
78 2.3.1.2. Compliance .................................... 9
79 3. References .................................................. 9
80 4. Security Considerations .................................... 10
81 5. Acknowledgements ........................................... 10
82 6. Author's Address ........................................... 10
83 7. Appendix: Jean-Loup Gailly's gzip utility .................. 11
84 8. Appendix: Sample CRC Code .................................. 11
85
861. Introduction
87
88 1.1. Purpose
89
90 The purpose of this specification is to define a lossless
91 compressed data format that:
92
93 * Is independent of CPU type, operating system, file system,
94 and character set, and hence can be used for interchange;
95 * Can compress or decompress a data stream (as opposed to a
96 randomly accessible file) to produce another data stream,
97 using only an a priori bounded amount of intermediate
98 storage, and hence can be used in data communications or
99 similar structures such as Unix filters;
100 * Compresses data with efficiency comparable to the best
101 currently available general-purpose compression methods,
102 and in particular considerably better than the "compress"
103 program;
104 * Can be implemented readily in a manner not covered by
105 patents, and hence can be practiced freely;
106 * Is compatible with the file format produced by the current
107 widely used gzip utility, in that conforming decompressors
108 will be able to read data produced by the existing gzip
109 compressor.
110
111
112
113
114Deutsch Informational [Page 2]
115
116RFC 1952 GZIP File Format Specification May 1996
117
118
119 The data format defined by this specification does not attempt to:
120
121 * Provide random access to compressed data;
122 * Compress specialized data (e.g., raster graphics) as well as
123 the best currently available specialized algorithms.
124
125 1.2. Intended audience
126
127 This specification is intended for use by implementors of software
128 to compress data into gzip format and/or decompress data from gzip
129 format.
130
131 The text of the specification assumes a basic background in
132 programming at the level of bits and other primitive data
133 representations.
134
135 1.3. Scope
136
137 The specification specifies a compression method and a file format
138 (the latter assuming only that a file can store a sequence of
139 arbitrary bytes). It does not specify any particular interface to
140 a file system or anything about character sets or encodings
141 (except for file names and comments, which are optional).
142
143 1.4. Compliance
144
145 Unless otherwise indicated below, a compliant decompressor must be
146 able to accept and decompress any file that conforms to all the
147 specifications presented here; a compliant compressor must produce
148 files that conform to all the specifications presented here. The
149 material in the appendices is not part of the specification per se
150 and is not relevant to compliance.
151
152 1.5. Definitions of terms and conventions used
153
154 byte: 8 bits stored or transmitted as a unit (same as an octet).
155 (For this specification, a byte is exactly 8 bits, even on
156 machines which store a character on a number of bits different
157 from 8.) See below for the numbering of bits within a byte.
158
159 1.6. Changes from previous versions
160
161 There have been no technical changes to the gzip format since
162 version 4.1 of this specification. In version 4.2, some
163 terminology was changed, and the sample CRC code was rewritten for
164 clarity and to eliminate the requirement for the caller to do pre-
165 and post-conditioning. Version 4.3 is a conversion of the
166 specification to RFC style.
167
168
169
170Deutsch Informational [Page 3]
171
172RFC 1952 GZIP File Format Specification May 1996
173
174
1752. Detailed specification
176
177 2.1. Overall conventions
178
179 In the diagrams below, a box like this:
180
181 +---+
182 | | <-- the vertical bars might be missing
183 +---+
184
185 represents one byte; a box like this:
186
187 +==============+
188 | |
189 +==============+
190
191 represents a variable number of bytes.
192
193 Bytes stored within a computer do not have a "bit order", since
194 they are always treated as a unit. However, a byte considered as
195 an integer between 0 and 255 does have a most- and least-
196 significant bit, and since we write numbers with the most-
197 significant digit on the left, we also write bytes with the most-
198 significant bit on the left. In the diagrams below, we number the
199 bits of a byte so that bit 0 is the least-significant bit, i.e.,
200 the bits are numbered:
201
202 +--------+
203 |76543210|
204 +--------+
205
206 This document does not address the issue of the order in which
207 bits of a byte are transmitted on a bit-sequential medium, since
208 the data format described here is byte- rather than bit-oriented.
209
210 Within a computer, a number may occupy multiple bytes. All
211 multi-byte numbers in the format described here are stored with
212 the least-significant byte first (at the lower memory address).
213 For example, the decimal number 520 is stored as:
214
215 0 1
216 +--------+--------+
217 |00001000|00000010|
218 +--------+--------+
219 ^ ^
220 | |
221 | + more significant byte = 2 x 256
222 + less significant byte = 8
223
224
225
226Deutsch Informational [Page 4]
227
228RFC 1952 GZIP File Format Specification May 1996
229
230
231 2.2. File format
232
233 A gzip file consists of a series of "members" (compressed data
234 sets). The format of each member is specified in the following
235 section. The members simply appear one after another in the file,
236 with no additional information before, between, or after them.
237
238 2.3. Member format
239
240 Each member has the following structure:
241
242 +---+---+---+---+---+---+---+---+---+---+
243 |ID1|ID2|CM |FLG| MTIME |XFL|OS | (more-->)
244 +---+---+---+---+---+---+---+---+---+---+
245
246 (if FLG.FEXTRA set)
247
248 +---+---+=================================+
249 | XLEN |...XLEN bytes of "extra field"...| (more-->)
250 +---+---+=================================+
251
252 (if FLG.FNAME set)
253
254 +=========================================+
255 |...original file name, zero-terminated...| (more-->)
256 +=========================================+
257
258 (if FLG.FCOMMENT set)
259
260 +===================================+
261 |...file comment, zero-terminated...| (more-->)
262 +===================================+
263
264 (if FLG.FHCRC set)
265
266 +---+---+
267 | CRC16 |
268 +---+---+
269
270 +=======================+
271 |...compressed blocks...| (more-->)
272 +=======================+
273
274 0 1 2 3 4 5 6 7
275 +---+---+---+---+---+---+---+---+
276 | CRC32 | ISIZE |
277 +---+---+---+---+---+---+---+---+
278
279
280
281
282Deutsch Informational [Page 5]
283
284RFC 1952 GZIP File Format Specification May 1996
285
286
287 2.3.1. Member header and trailer
288
289 ID1 (IDentification 1)
290 ID2 (IDentification 2)
291 These have the fixed values ID1 = 31 (0x1f, \037), ID2 = 139
292 (0x8b, \213), to identify the file as being in gzip format.
293
294 CM (Compression Method)
295 This identifies the compression method used in the file. CM
296 = 0-7 are reserved. CM = 8 denotes the "deflate"
297 compression method, which is the one customarily used by
298 gzip and which is documented elsewhere.
299
300 FLG (FLaGs)
301 This flag byte is divided into individual bits as follows:
302
303 bit 0 FTEXT
304 bit 1 FHCRC
305 bit 2 FEXTRA
306 bit 3 FNAME
307 bit 4 FCOMMENT
308 bit 5 reserved
309 bit 6 reserved
310 bit 7 reserved
311
312 If FTEXT is set, the file is probably ASCII text. This is
313 an optional indication, which the compressor may set by
314 checking a small amount of the input data to see whether any
315 non-ASCII characters are present. In case of doubt, FTEXT
316 is cleared, indicating binary data. For systems which have
317 different file formats for ascii text and binary data, the
318 decompressor can use FTEXT to choose the appropriate format.
319 We deliberately do not specify the algorithm used to set
320 this bit, since a compressor always has the option of
321 leaving it cleared and a decompressor always has the option
322 of ignoring it and letting some other program handle issues
323 of data conversion.
324
325 If FHCRC is set, a CRC16 for the gzip header is present,
326 immediately before the compressed data. The CRC16 consists
327 of the two least significant bytes of the CRC32 for all
328 bytes of the gzip header up to and not including the CRC16.
329 [The FHCRC bit was never set by versions of gzip up to
330 1.2.4, even though it was documented with a different
331 meaning in gzip 1.2.4.]
332
333 If FEXTRA is set, optional extra fields are present, as
334 described in a following section.
335
336
337
338Deutsch Informational [Page 6]
339
340RFC 1952 GZIP File Format Specification May 1996
341
342
343 If FNAME is set, an original file name is present,
344 terminated by a zero byte. The name must consist of ISO
345 8859-1 (LATIN-1) characters; on operating systems using
346 EBCDIC or any other character set for file names, the name
347 must be translated to the ISO LATIN-1 character set. This
348 is the original name of the file being compressed, with any
349 directory components removed, and, if the file being
350 compressed is on a file system with case insensitive names,
351 forced to lower case. There is no original file name if the
352 data was compressed from a source other than a named file;
353 for example, if the source was stdin on a Unix system, there
354 is no file name.
355
356 If FCOMMENT is set, a zero-terminated file comment is
357 present. This comment is not interpreted; it is only
358 intended for human consumption. The comment must consist of
359 ISO 8859-1 (LATIN-1) characters. Line breaks should be
360 denoted by a single line feed character (10 decimal).
361
362 Reserved FLG bits must be zero.
363
364 MTIME (Modification TIME)
365 This gives the most recent modification time of the original
366 file being compressed. The time is in Unix format, i.e.,
367 seconds since 00:00:00 GMT, Jan. 1, 1970. (Note that this
368 may cause problems for MS-DOS and other systems that use
369 local rather than Universal time.) If the compressed data
370 did not come from a file, MTIME is set to the time at which
371 compression started. MTIME = 0 means no time stamp is
372 available.
373
374 XFL (eXtra FLags)
375 These flags are available for use by specific compression
376 methods. The "deflate" method (CM = 8) sets these flags as
377 follows:
378
379 XFL = 2 - compressor used maximum compression,
380 slowest algorithm
381 XFL = 4 - compressor used fastest algorithm
382
383 OS (Operating System)
384 This identifies the type of file system on which compression
385 took place. This may be useful in determining end-of-line
386 convention for text files. The currently defined values are
387 as follows:
388
389
390
391
392
393
394Deutsch Informational [Page 7]
395
396RFC 1952 GZIP File Format Specification May 1996
397
398
399 0 - FAT filesystem (MS-DOS, OS/2, NT/Win32)
400 1 - Amiga
401 2 - VMS (or OpenVMS)
402 3 - Unix
403 4 - VM/CMS
404 5 - Atari TOS
405 6 - HPFS filesystem (OS/2, NT)
406 7 - Macintosh
407 8 - Z-System
408 9 - CP/M
409 10 - TOPS-20
410 11 - NTFS filesystem (NT)
411 12 - QDOS
412 13 - Acorn RISCOS
413 255 - unknown
414
415 XLEN (eXtra LENgth)
416 If FLG.FEXTRA is set, this gives the length of the optional
417 extra field. See below for details.
418
419 CRC32 (CRC-32)
420 This contains a Cyclic Redundancy Check value of the
421 uncompressed data computed according to CRC-32 algorithm
422 used in the ISO 3309 standard and in section 8.1.1.6.2 of
423 ITU-T recommendation V.42. (See http://www.iso.ch for
424 ordering ISO documents. See gopher://info.itu.ch for an
425 online version of ITU-T V.42.)
426
427 ISIZE (Input SIZE)
428 This contains the size of the original (uncompressed) input
429 data modulo 2^32.
430
431 2.3.1.1. Extra field
432
433 If the FLG.FEXTRA bit is set, an "extra field" is present in
434 the header, with total length XLEN bytes. It consists of a
435 series of subfields, each of the form:
436
437 +---+---+---+---+==================================+
438 |SI1|SI2| LEN |... LEN bytes of subfield data ...|
439 +---+---+---+---+==================================+
440
441 SI1 and SI2 provide a subfield ID, typically two ASCII letters
442 with some mnemonic value. Jean-Loup Gailly
443 <gzip@prep.ai.mit.edu> is maintaining a registry of subfield
444 IDs; please send him any subfield ID you wish to use. Subfield
445 IDs with SI2 = 0 are reserved for future use. The following
446 IDs are currently defined:
447
448
449
450Deutsch Informational [Page 8]
451
452RFC 1952 GZIP File Format Specification May 1996
453
454
455 SI1 SI2 Data
456 ---------- ---------- ----
457 0x41 ('A') 0x70 ('P') Apollo file type information
458
459 LEN gives the length of the subfield data, excluding the 4
460 initial bytes.
461
462 2.3.1.2. Compliance
463
464 A compliant compressor must produce files with correct ID1,
465 ID2, CM, CRC32, and ISIZE, but may set all the other fields in
466 the fixed-length part of the header to default values (255 for
467 OS, 0 for all others). The compressor must set all reserved
468 bits to zero.
469
470 A compliant decompressor must check ID1, ID2, and CM, and
471 provide an error indication if any of these have incorrect
472 values. It must examine FEXTRA/XLEN, FNAME, FCOMMENT and FHCRC
473 at least so it can skip over the optional fields if they are
474 present. It need not examine any other part of the header or
475 trailer; in particular, a decompressor may ignore FTEXT and OS
476 and always produce binary output, and still be compliant. A
477 compliant decompressor must give an error indication if any
478 reserved bit is non-zero, since such a bit could indicate the
479 presence of a new field that would cause subsequent data to be
480 interpreted incorrectly.
481
4823. References
483
484 [1] "Information Processing - 8-bit single-byte coded graphic
485 character sets - Part 1: Latin alphabet No.1" (ISO 8859-1:1987).
486 The ISO 8859-1 (Latin-1) character set is a superset of 7-bit
487 ASCII. Files defining this character set are available as
488 iso_8859-1.* in ftp://ftp.uu.net/graphics/png/documents/
489
490 [2] ISO 3309
491
492 [3] ITU-T recommendation V.42
493
494 [4] Deutsch, L.P.,"DEFLATE Compressed Data Format Specification",
495 available in ftp://ftp.uu.net/pub/archiving/zip/doc/
496
497 [5] Gailly, J.-L., GZIP documentation, available as gzip-*.tar in
498 ftp://prep.ai.mit.edu/pub/gnu/
499
500 [6] Sarwate, D.V., "Computation of Cyclic Redundancy Checks via Table
501 Look-Up", Communications of the ACM, 31(8), pp.1008-1013.
502
503
504
505
506Deutsch Informational [Page 9]
507
508RFC 1952 GZIP File Format Specification May 1996
509
510
511 [7] Schwaderer, W.D., "CRC Calculation", April 85 PC Tech Journal,
512 pp.118-133.
513
514 [8] ftp://ftp.adelaide.edu.au/pub/rocksoft/papers/crc_v3.txt,
515 describing the CRC concept.
516
5174. Security Considerations
518
519 Any data compression method involves the reduction of redundancy in
520 the data. Consequently, any corruption of the data is likely to have
521 severe effects and be difficult to correct. Uncompressed text, on
522 the other hand, will probably still be readable despite the presence
523 of some corrupted bytes.
524
525 It is recommended that systems using this data format provide some
526 means of validating the integrity of the compressed data, such as by
527 setting and checking the CRC-32 check value.
528
5295. Acknowledgements
530
531 Trademarks cited in this document are the property of their
532 respective owners.
533
534 Jean-Loup Gailly designed the gzip format and wrote, with Mark Adler,
535 the related software described in this specification. Glenn
536 Randers-Pehrson converted this document to RFC and HTML format.
537
5386. Author's Address
539
540 L. Peter Deutsch
541 Aladdin Enterprises
542 203 Santa Margarita Ave.
543 Menlo Park, CA 94025
544
545 Phone: (415) 322-0103 (AM only)
546 FAX: (415) 322-1734
547 EMail: <ghost@aladdin.com>
548
549 Questions about the technical content of this specification can be
550 sent by email to:
551
552 Jean-Loup Gailly <gzip@prep.ai.mit.edu> and
553 Mark Adler <madler@alumni.caltech.edu>
554
555 Editorial comments on this specification can be sent by email to:
556
557 L. Peter Deutsch <ghost@aladdin.com> and
558 Glenn Randers-Pehrson <randeg@alumni.rpi.edu>
559
560
561
562Deutsch Informational [Page 10]
563
564RFC 1952 GZIP File Format Specification May 1996
565
566
5677. Appendix: Jean-Loup Gailly's gzip utility
568
569 The most widely used implementation of gzip compression, and the
570 original documentation on which this specification is based, were
571 created by Jean-Loup Gailly <gzip@prep.ai.mit.edu>. Since this
572 implementation is a de facto standard, we mention some more of its
573 features here. Again, the material in this section is not part of
574 the specification per se, and implementations need not follow it to
575 be compliant.
576
577 When compressing or decompressing a file, gzip preserves the
578 protection, ownership, and modification time attributes on the local
579 file system, since there is no provision for representing protection
580 attributes in the gzip file format itself. Since the file format
581 includes a modification time, the gzip decompressor provides a
582 command line switch that assigns the modification time from the file,
583 rather than the local modification time of the compressed input, to
584 the decompressed output.
585
5868. Appendix: Sample CRC Code
587
588 The following sample code represents a practical implementation of
589 the CRC (Cyclic Redundancy Check). (See also ISO 3309 and ITU-T V.42
590 for a formal specification.)
591
592 The sample code is in the ANSI C programming language. Non C users
593 may find it easier to read with these hints:
594
595 & Bitwise AND operator.
596 ^ Bitwise exclusive-OR operator.
597 >> Bitwise right shift operator. When applied to an
598 unsigned quantity, as here, right shift inserts zero
599 bit(s) at the left.
600 ! Logical NOT operator.
601 ++ "n++" increments the variable n.
602 0xNNN 0x introduces a hexadecimal (base 16) constant.
603 Suffix L indicates a long value (at least 32 bits).
604
605 /* Table of CRCs of all 8-bit messages. */
606 unsigned long crc_table[256];
607
608 /* Flag: has the table been computed? Initially false. */
609 int crc_table_computed = 0;
610
611 /* Make the table for a fast CRC. */
612 void make_crc_table(void)
613 {
614 unsigned long c;
615
616
617
618Deutsch Informational [Page 11]
619
620RFC 1952 GZIP File Format Specification May 1996
621
622
623 int n, k;
624 for (n = 0; n < 256; n++) {
625 c = (unsigned long) n;
626 for (k = 0; k < 8; k++) {
627 if (c & 1) {
628 c = 0xedb88320L ^ (c >> 1);
629 } else {
630 c = c >> 1;
631 }
632 }
633 crc_table[n] = c;
634 }
635 crc_table_computed = 1;
636 }
637
638 /*
639 Update a running crc with the bytes buf[0..len-1] and return
640 the updated crc. The crc should be initialized to zero. Pre- and
641 post-conditioning (one's complement) is performed within this
642 function so it shouldn't be done by the caller. Usage example:
643
644 unsigned long crc = 0L;
645
646 while (read_buffer(buffer, length) != EOF) {
647 crc = update_crc(crc, buffer, length);
648 }
649 if (crc != original_crc) error();
650 */
651 unsigned long update_crc(unsigned long crc,
652 unsigned char *buf, int len)
653 {
654 unsigned long c = crc ^ 0xffffffffL;
655 int n;
656
657 if (!crc_table_computed)
658 make_crc_table();
659 for (n = 0; n < len; n++) {
660 c = crc_table[(c ^ buf[n]) & 0xff] ^ (c >> 8);
661 }
662 return c ^ 0xffffffffL;
663 }
664
665 /* Return the CRC of the bytes buf[0..len-1]. */
666 unsigned long crc(unsigned char *buf, int len)
667 {
668 return update_crc(0L, buf, len);
669 }
670
671
672
673
674Deutsch Informational [Page 12]
675
lib/std/compress/testdata/rfc1952.txt.gz deleted
Binary files a/lib/std/compress/testdata/rfc1952.txt.gz and /dev/null differ
lib/std/compress/zlib.zig deleted-282
......@@ -1,282 +0,0 @@
1//
2// Compressor/Decompressor for ZLIB data streams (RFC1950)
3
4const std = @import("std");
5const io = std.io;
6const fs = std.fs;
7const testing = std.testing;
8const mem = std.mem;
9const deflate = @import("deflate.zig");
10
11// Zlib header format as specified in RFC1950
12const ZLibHeader = packed struct {
13 checksum: u5,
14 preset_dict: u1,
15 compression_level: u2,
16 compression_method: u4,
17 compression_info: u4,
18
19 const DEFLATE = 8;
20 const WINDOW_32K = 7;
21};
22
23pub fn DecompressStream(comptime ReaderType: type) type {
24 return struct {
25 const Self = @This();
26
27 pub const Error = ReaderType.Error ||
28 deflate.Decompressor(ReaderType).Error ||
29 error{ WrongChecksum, Unsupported };
30 pub const Reader = io.Reader(*Self, Error, read);
31
32 allocator: mem.Allocator,
33 inflater: deflate.Decompressor(ReaderType),
34 in_reader: ReaderType,
35 hasher: std.hash.Adler32,
36
37 fn init(allocator: mem.Allocator, source: ReaderType) !Self {
38 // Zlib header format is specified in RFC1950
39 const header_u16 = try source.readInt(u16, .big);
40
41 // verify the header checksum
42 if (header_u16 % 31 != 0)
43 return error.BadHeader;
44 const header = @as(ZLibHeader, @bitCast(header_u16));
45
46 // The CM field must be 8 to indicate the use of DEFLATE
47 if (header.compression_method != ZLibHeader.DEFLATE)
48 return error.InvalidCompression;
49 // CINFO is the base-2 logarithm of the LZ77 window size, minus 8.
50 // Values above 7 are unspecified and therefore rejected.
51 if (header.compression_info > ZLibHeader.WINDOW_32K)
52 return error.InvalidWindowSize;
53
54 const dictionary = null;
55 // TODO: Support this case
56 if (header.preset_dict != 0)
57 return error.Unsupported;
58
59 return Self{
60 .allocator = allocator,
61 .inflater = try deflate.decompressor(allocator, source, dictionary),
62 .in_reader = source,
63 .hasher = std.hash.Adler32.init(),
64 };
65 }
66
67 pub fn deinit(self: *Self) void {
68 self.inflater.deinit();
69 }
70
71 // Implements the io.Reader interface
72 pub fn read(self: *Self, buffer: []u8) Error!usize {
73 if (buffer.len == 0)
74 return 0;
75
76 // Read from the compressed stream and update the computed checksum
77 const r = try self.inflater.read(buffer);
78 if (r != 0) {
79 self.hasher.update(buffer[0..r]);
80 return r;
81 }
82
83 // We've reached the end of stream, check if the checksum matches
84 const hash = try self.in_reader.readInt(u32, .big);
85 if (hash != self.hasher.final())
86 return error.WrongChecksum;
87
88 return 0;
89 }
90
91 pub fn reader(self: *Self) Reader {
92 return .{ .context = self };
93 }
94 };
95}
96
97pub fn decompressStream(allocator: mem.Allocator, reader: anytype) !DecompressStream(@TypeOf(reader)) {
98 return DecompressStream(@TypeOf(reader)).init(allocator, reader);
99}
100
101pub const CompressionLevel = enum(u2) {
102 no_compression = 0,
103 fastest = 1,
104 default = 2,
105 maximum = 3,
106};
107
108pub const CompressStreamOptions = struct {
109 level: CompressionLevel = .default,
110};
111
112pub fn CompressStream(comptime WriterType: type) type {
113 return struct {
114 const Self = @This();
115
116 const Error = WriterType.Error ||
117 deflate.Compressor(WriterType).Error;
118 pub const Writer = io.Writer(*Self, Error, write);
119
120 allocator: mem.Allocator,
121 deflator: deflate.Compressor(WriterType),
122 in_writer: WriterType,
123 hasher: std.hash.Adler32,
124
125 fn init(allocator: mem.Allocator, dest: WriterType, options: CompressStreamOptions) !Self {
126 var header = ZLibHeader{
127 .compression_info = ZLibHeader.WINDOW_32K,
128 .compression_method = ZLibHeader.DEFLATE,
129 .compression_level = @intFromEnum(options.level),
130 .preset_dict = 0,
131 .checksum = 0,
132 };
133 header.checksum = @as(u5, @truncate(31 - @as(u16, @bitCast(header)) % 31));
134
135 try dest.writeInt(u16, @as(u16, @bitCast(header)), .big);
136
137 const compression_level: deflate.Compression = switch (options.level) {
138 .no_compression => .no_compression,
139 .fastest => .best_speed,
140 .default => .default_compression,
141 .maximum => .best_compression,
142 };
143
144 return Self{
145 .allocator = allocator,
146 .deflator = try deflate.compressor(allocator, dest, .{ .level = compression_level }),
147 .in_writer = dest,
148 .hasher = std.hash.Adler32.init(),
149 };
150 }
151
152 pub fn write(self: *Self, bytes: []const u8) Error!usize {
153 if (bytes.len == 0) {
154 return 0;
155 }
156
157 const w = try self.deflator.write(bytes);
158
159 self.hasher.update(bytes[0..w]);
160 return w;
161 }
162
163 pub fn writer(self: *Self) Writer {
164 return .{ .context = self };
165 }
166
167 pub fn deinit(self: *Self) void {
168 self.deflator.deinit();
169 }
170
171 pub fn finish(self: *Self) !void {
172 const hash = self.hasher.final();
173 try self.deflator.close();
174 try self.in_writer.writeInt(u32, hash, .big);
175 }
176 };
177}
178
179pub fn compressStream(allocator: mem.Allocator, writer: anytype, options: CompressStreamOptions) !CompressStream(@TypeOf(writer)) {
180 return CompressStream(@TypeOf(writer)).init(allocator, writer, options);
181}
182
183fn testDecompress(data: []const u8, expected: []const u8) !void {
184 var in_stream = io.fixedBufferStream(data);
185
186 var zlib_stream = try decompressStream(testing.allocator, in_stream.reader());
187 defer zlib_stream.deinit();
188
189 // Read and decompress the whole file
190 const buf = try zlib_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
191 defer testing.allocator.free(buf);
192
193 // Check against the reference
194 try testing.expectEqualSlices(u8, expected, buf);
195}
196
197// All the test cases are obtained by compressing the RFC1951 text
198//
199// https://tools.ietf.org/rfc/rfc1951.txt length=36944 bytes
200// SHA256=5ebf4b5b7fe1c3a0c0ab9aa3ac8c0f3853a7dc484905e76e03b0b0f301350009
201test "compressed data" {
202 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
203
204 // Compressed with compression level = 0
205 try testDecompress(
206 @embedFile("testdata/rfc1951.txt.z.0"),
207 rfc1951_txt,
208 );
209 // Compressed with compression level = 9
210 try testDecompress(
211 @embedFile("testdata/rfc1951.txt.z.9"),
212 rfc1951_txt,
213 );
214 // Compressed with compression level = 9 and fixed Huffman codes
215 try testDecompress(
216 @embedFile("testdata/rfc1951.txt.fixed.z.9"),
217 rfc1951_txt,
218 );
219}
220
221test "don't read past deflate stream's end" {
222 try testDecompress(&[_]u8{
223 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff,
224 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01,
225 0x83, 0x95, 0x0b, 0xf5,
226 }, &[_]u8{
227 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff,
228 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00,
229 0x00, 0x00, 0xff, 0xff, 0xff,
230 });
231}
232
233test "sanity checks" {
234 // Truncated header
235 try testing.expectError(
236 error.EndOfStream,
237 testDecompress(&[_]u8{0x78}, ""),
238 );
239 // Failed FCHECK check
240 try testing.expectError(
241 error.BadHeader,
242 testDecompress(&[_]u8{ 0x78, 0x9D }, ""),
243 );
244 // Wrong CM
245 try testing.expectError(
246 error.InvalidCompression,
247 testDecompress(&[_]u8{ 0x79, 0x94 }, ""),
248 );
249 // Wrong CINFO
250 try testing.expectError(
251 error.InvalidWindowSize,
252 testDecompress(&[_]u8{ 0x88, 0x98 }, ""),
253 );
254 // Wrong checksum
255 try testing.expectError(
256 error.WrongChecksum,
257 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""),
258 );
259 // Truncated checksum
260 try testing.expectError(
261 error.EndOfStream,
262 testDecompress(&[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""),
263 );
264}
265
266test "compress data" {
267 const allocator = testing.allocator;
268 const rfc1951_txt = @embedFile("testdata/rfc1951.txt");
269
270 for (std.meta.tags(CompressionLevel)) |level| {
271 var compressed_data = std.ArrayList(u8).init(allocator);
272 defer compressed_data.deinit();
273
274 var compressor = try compressStream(allocator, compressed_data.writer(), .{ .level = level });
275 defer compressor.deinit();
276
277 try compressor.writer().writeAll(rfc1951_txt);
278 try compressor.finish();
279
280 try testDecompress(compressed_data.items, rfc1951_txt);
281 }
282}