authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-27 10:25:46-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-31 22:10:11-07:00
loga4f05a4588100c5e7f311dd5319e97e394f109a4
treeae80f91e5dc391ede8a598e3d204b7b78057984f
parent83513ade3591de673e9ac4824fe974cd8f90c847

delete flate implementation

58 files changed, 783 insertions(+), 1775 deletions(-)

lib/std/compress/flate.zig+99-119
......@@ -1,7 +1,7 @@
11const builtin = @import("builtin");
22const std = @import("../std.zig");
33const testing = std.testing;
4const Writer = std.io.Writer;
4const Writer = std.Io.Writer;
55
66/// Container of the deflate bit stream body. Container adds header before
77/// deflate bit stream and footer after. It can bi gzip, zlib or raw (no header,
......@@ -77,7 +77,7 @@ pub const Container = enum {
7777 raw: void,
7878 gzip: struct {
7979 crc: std.hash.Crc32 = .init(),
80 count: usize = 0,
80 count: u32 = 0,
8181 },
8282 zlib: std.hash.Adler32,
8383
......@@ -98,7 +98,7 @@ pub const Container = enum {
9898 .raw => {},
9999 .gzip => |*gzip| {
100100 gzip.update(buf);
101 gzip.count += buf.len;
101 gzip.count +%= buf.len;
102102 },
103103 .zlib => |*zlib| {
104104 zlib.update(buf);
......@@ -148,35 +148,9 @@ pub const Compress = @import("flate/Compress.zig");
148148/// decompression and correctly produces the original full-size data or file.
149149pub const Decompress = @import("flate/Decompress.zig");
150150
151/// Huffman only compression. Without Lempel-Ziv match searching. Faster
152/// compression, less memory requirements but bigger compressed sizes.
153pub const huffman = struct {
154 // The odd order in which the codegen code sizes are written.
155 pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
156 // The number of codegen codes.
157 pub const codegen_code_count = 19;
158
159 // The largest distance code.
160 pub const distance_code_count = 30;
161
162 // Maximum number of literals.
163 pub const max_num_lit = 286;
164
165 // Max number of frequencies used for a Huffman Code
166 // Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
167 // The largest of these is max_num_lit.
168 pub const max_num_frequencies = max_num_lit;
169
170 // Biggest block size for uncompressed block.
171 pub const max_store_block_size = 65535;
172 // The special code used to mark the end of a block.
173 pub const end_block_marker = 256;
174};
175
176test {
177 _ = Compress;
178 _ = Decompress;
179}
151/// Compression without Lempel-Ziv match searching. Faster compression, less
152/// memory requirements but bigger compressed sizes.
153pub const HuffmanEncoder = @import("flate/HuffmanEncoder.zig");
180154
181155test "compress/decompress" {
182156 const print = std.debug.print;
......@@ -217,12 +191,11 @@ test "compress/decompress" {
217191 },
218192 };
219193
220 for (cases, 0..) |case, case_no| { // for each case
194 for (cases, 0..) |case, case_no| {
221195 const data = case.data;
222196
223 for (levels, 0..) |level, i| { // for each compression level
224
225 inline for (Container.list) |container| { // for each wrapping
197 for (levels, 0..) |level, i| {
198 for (Container.list) |container| {
226199 var compressed_size: usize = if (case.gzip_sizes[i] > 0)
227200 case.gzip_sizes[i] - Container.gzip.size() + container.size()
228201 else
......@@ -230,21 +203,21 @@ test "compress/decompress" {
230203
231204 // compress original stream to compressed stream
232205 {
233 var original: std.io.Reader = .fixed(data);
234206 var compressed: Writer = .fixed(&cmp_buf);
235 var compress: Compress = .init(&original, &.{}, .{ .container = .raw, .level = level });
236 const n = try compress.reader.streamRemaining(&compressed);
207 var compress: Compress = .init(&compressed, &.{}, .{ .container = .raw, .level = level });
208 try compress.writer.writeAll(data);
209 try compress.end();
210
237211 if (compressed_size == 0) {
238212 if (container == .gzip)
239213 print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
240214 compressed_size = compressed.end;
241215 }
242 try testing.expectEqual(compressed_size, n);
243216 try testing.expectEqual(compressed_size, compressed.end);
244217 }
245218 // decompress compressed stream to decompressed stream
246219 {
247 var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
220 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
248221 var decompressed: Writer = .fixed(&dcm_buf);
249222 var decompress: Decompress = .init(&compressed, container, &.{});
250223 _ = try decompress.reader.streamRemaining(&decompressed);
......@@ -266,7 +239,7 @@ test "compress/decompress" {
266239 }
267240 // decompressor reader interface
268241 {
269 var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
242 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
270243 var decompress: Decompress = .init(&compressed, container, &.{});
271244 const n = try decompress.reader.readSliceShort(&dcm_buf);
272245 try testing.expectEqual(data.len, n);
......@@ -276,7 +249,7 @@ test "compress/decompress" {
276249 }
277250 // huffman only compression
278251 {
279 inline for (Container.list) |container| { // for each wrapping
252 for (Container.list) |container| {
280253 var compressed_size: usize = if (case.huffman_only_size > 0)
281254 case.huffman_only_size - Container.gzip.size() + container.size()
282255 else
......@@ -284,7 +257,7 @@ test "compress/decompress" {
284257
285258 // compress original stream to compressed stream
286259 {
287 var original: std.io.Reader = .fixed(data);
260 var original: std.Io.Reader = .fixed(data);
288261 var compressed: Writer = .fixed(&cmp_buf);
289262 var cmp = try Compress.Huffman.init(container, &compressed);
290263 try cmp.compress(original.reader());
......@@ -298,7 +271,7 @@ test "compress/decompress" {
298271 }
299272 // decompress compressed stream to decompressed stream
300273 {
301 var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
274 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
302275 var decompress: Decompress = .init(&compressed, container, &.{});
303276 var decompressed: Writer = .fixed(&dcm_buf);
304277 _ = try decompress.reader.streamRemaining(&decompressed);
......@@ -309,7 +282,7 @@ test "compress/decompress" {
309282
310283 // store only
311284 {
312 inline for (Container.list) |container| { // for each wrapping
285 for (Container.list) |container| {
313286 var compressed_size: usize = if (case.store_size > 0)
314287 case.store_size - Container.gzip.size() + container.size()
315288 else
......@@ -317,7 +290,7 @@ test "compress/decompress" {
317290
318291 // compress original stream to compressed stream
319292 {
320 var original: std.io.Reader = .fixed(data);
293 var original: std.Io.Reader = .fixed(data);
321294 var compressed: Writer = .fixed(&cmp_buf);
322295 var cmp = try Compress.SimpleCompressor(.store, container).init(&compressed);
323296 try cmp.compress(original.reader());
......@@ -332,7 +305,7 @@ test "compress/decompress" {
332305 }
333306 // decompress compressed stream to decompressed stream
334307 {
335 var compressed: std.io.Reader = .fixed(cmp_buf[0..compressed_size]);
308 var compressed: std.Io.Reader = .fixed(cmp_buf[0..compressed_size]);
336309 var decompress: Decompress = .init(&compressed, container, &.{});
337310 var decompressed: Writer = .fixed(&dcm_buf);
338311 _ = try decompress.reader.streamRemaining(&decompressed);
......@@ -344,13 +317,13 @@ test "compress/decompress" {
344317}
345318
346319fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
347 var in: std.io.Reader = .fixed(compressed);
348 var aw: std.io.Writer.Allocating = .init(testing.allocator);
320 var in: std.Io.Reader = .fixed(compressed);
321 var aw: std.Io.Writer.Allocating = .init(testing.allocator);
349322 defer aw.deinit();
350323
351324 var decompress: Decompress = .init(&in, container, &.{});
352325 _ = try decompress.reader.streamRemaining(&aw.writer);
353 try testing.expectEqualSlices(u8, expected_plain, aw.items);
326 try testing.expectEqualSlices(u8, expected_plain, aw.getWritten());
354327}
355328
356329test "don't read past deflate stream's end" {
......@@ -483,17 +456,12 @@ test "public interface" {
483456 var buffer1: [64]u8 = undefined;
484457 var buffer2: [64]u8 = undefined;
485458
486 // TODO These used to be functions, need to migrate the tests
487 const decompress = void;
488 const compress = void;
489 const store = void;
490
491459 // decompress
492460 {
493461 var plain: Writer = .fixed(&buffer2);
494
495 var in: std.io.Reader = .fixed(gzip_data);
496 try decompress(&in, &plain);
462 var in: std.Io.Reader = .fixed(gzip_data);
463 var d: Decompress = .init(&in, .raw, &.{});
464 _ = try d.reader.streamRemaining(&plain);
497465 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
498466 }
499467
......@@ -502,11 +470,13 @@ test "public interface" {
502470 var plain: Writer = .fixed(&buffer2);
503471 var compressed: Writer = .fixed(&buffer1);
504472
505 var in: std.io.Reader = .fixed(plain_data);
506 try compress(&in, &compressed, .{});
473 var cmp: Compress = .init(&compressed, &.{}, .{});
474 try cmp.writer.writeAll(plain_data);
475 try cmp.end();
507476
508 var r: std.io.Reader = .fixed(&buffer1);
509 try decompress(&r, &plain);
477 var r: std.Io.Reader = .fixed(&buffer1);
478 var d: Decompress = .init(&r, .raw, &.{});
479 _ = try d.reader.streamRemaining(&plain);
510480 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
511481 }
512482
......@@ -515,12 +485,11 @@ test "public interface" {
515485 var plain: Writer = .fixed(&buffer2);
516486 var compressed: Writer = .fixed(&buffer1);
517487
518 var in: std.io.Reader = .fixed(plain_data);
519 var cmp = try Compress(&compressed, .{});
520 try cmp.compress(&in);
521 try cmp.finish();
488 var cmp: Compress = .init(&compressed, &.{}, .{});
489 try cmp.writer.writeAll(plain_data);
490 try cmp.end();
522491
523 var r: std.io.Reader = .fixed(&buffer1);
492 var r: std.Io.Reader = .fixed(&buffer1);
524493 var dcp = Decompress(&r);
525494 try dcp.decompress(&plain);
526495 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
......@@ -533,11 +502,12 @@ test "public interface" {
533502 var plain: Writer = .fixed(&buffer2);
534503 var compressed: Writer = .fixed(&buffer1);
535504
536 var in: std.io.Reader = .fixed(plain_data);
537 try huffman.compress(&in, &compressed);
505 var in: std.Io.Reader = .fixed(plain_data);
506 try HuffmanEncoder.compress(&in, &compressed);
538507
539 var r: std.io.Reader = .fixed(&buffer1);
540 try decompress(&r, &plain);
508 var r: std.Io.Reader = .fixed(&buffer1);
509 var d: Decompress = .init(&r, .raw, &.{});
510 _ = try d.reader.streamRemaining(&plain);
541511 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
542512 }
543513
......@@ -546,47 +516,50 @@ test "public interface" {
546516 var plain: Writer = .fixed(&buffer2);
547517 var compressed: Writer = .fixed(&buffer1);
548518
549 var in: std.io.Reader = .fixed(plain_data);
550 var cmp = try huffman.Compressor(&compressed);
519 var in: std.Io.Reader = .fixed(plain_data);
520 var cmp = try HuffmanEncoder.Compressor(&compressed);
551521 try cmp.compress(&in);
552522 try cmp.finish();
553523
554 var r: std.io.Reader = .fixed(&buffer1);
555 try decompress(&r, &plain);
524 var r: std.Io.Reader = .fixed(&buffer1);
525 var d: Decompress = .init(&r, .raw, &.{});
526 _ = try d.reader.streamRemaining(&plain);
556527 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
557528 }
558529 }
559530
560 // store
561 {
562 // store compress/decompress
563 {
564 var plain: Writer = .fixed(&buffer2);
565 var compressed: Writer = .fixed(&buffer1);
566
567 var in: std.io.Reader = .fixed(plain_data);
568 try store.compress(&in, &compressed);
569
570 var r: std.io.Reader = .fixed(&buffer1);
571 try decompress(&r, &plain);
572 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
573 }
574
575 // store compressor/decompressor
576 {
577 var plain: Writer = .fixed(&buffer2);
578 var compressed: Writer = .fixed(&buffer1);
579
580 var in: std.io.Reader = .fixed(plain_data);
581 var cmp = try store.compressor(&compressed);
582 try cmp.compress(&in);
583 try cmp.finish();
584
585 var r: std.io.Reader = .fixed(&buffer1);
586 try decompress(&r, &plain);
587 try testing.expectEqualSlices(u8, plain_data, plain.buffered());
588 }
589 }
531 // TODO
532 //{
533 // // store compress/decompress
534 // {
535 // var plain: Writer = .fixed(&buffer2);
536 // var compressed: Writer = .fixed(&buffer1);
537
538 // var in: std.Io.Reader = .fixed(plain_data);
539 // try store.compress(&in, &compressed);
540
541 // var r: std.Io.Reader = .fixed(&buffer1);
542 // var d: Decompress = .init(&r, .raw, &.{});
543 // _ = try d.reader.streamRemaining(&plain);
544 // try testing.expectEqualSlices(u8, plain_data, plain.buffered());
545 // }
546
547 // // store compressor/decompressor
548 // {
549 // var plain: Writer = .fixed(&buffer2);
550 // var compressed: Writer = .fixed(&buffer1);
551
552 // var in: std.Io.Reader = .fixed(plain_data);
553 // var cmp = try store.compressor(&compressed);
554 // try cmp.compress(&in);
555 // try cmp.finish();
556
557 // var r: std.Io.Reader = .fixed(&buffer1);
558 // var d: Decompress = .init(&r, .raw, &.{});
559 // _ = try d.reader.streamRemaining(&plain);
560 // try testing.expectEqualSlices(u8, plain_data, plain.buffered());
561 // }
562 //}
590563}
591564
592565pub const match = struct {
......@@ -615,26 +588,33 @@ test "zlib should not overshoot" {
615588 0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
616589 };
617590
618 var stream: std.io.Reader = .fixed(&data);
619 const reader = stream.reader();
591 var reader: std.Io.Reader = .fixed(&data);
620592
621 var dcp = Decompress.init(reader);
593 var decompress: Decompress = .init(&reader, .zlib, &.{});
622594 var out: [128]u8 = undefined;
623595
624 // Decompress
625 var n = try dcp.reader().readAll(out[0..]);
596 {
597 const n = try decompress.reader.readSliceShort(out[0..]);
626598
627 // Expected decompressed data
628 try std.testing.expectEqual(46, n);
629 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
599 // Expected decompressed data
600 try std.testing.expectEqual(46, n);
601 try std.testing.expectEqualStrings("Copyright Willem van Schaik, Singapore 1995-96", out[0..n]);
630602
631 // Decompressor don't overshoot underlying reader.
632 // It is leaving it at the end of compressed data chunk.
633 try std.testing.expectEqual(data.len - 4, stream.getPos());
634 try std.testing.expectEqual(0, dcp.unreadBytes());
603 // Decompressor don't overshoot underlying reader.
604 // It is leaving it at the end of compressed data chunk.
605 try std.testing.expectEqual(data.len - 4, reader.seek);
606 // TODO what was this testing, exactly?
607 //try std.testing.expectEqual(0, decompress.unreadBytes());
608 }
635609
636610 // 4 bytes after compressed chunk are available in reader.
637 n = try reader.readAll(out[0..]);
611 const n = try reader.readSliceShort(out[0..]);
638612 try std.testing.expectEqual(n, 4);
639613 try std.testing.expectEqualSlices(u8, data[data.len - 4 .. data.len], out[0..n]);
640614}
615
616test {
617 _ = HuffmanEncoder;
618 _ = Compress;
619 _ = Decompress;
620}
lib/std/compress/flate/BlockWriter.zig+69-192
......@@ -8,32 +8,33 @@ const Writer = std.io.Writer;
88const BlockWriter = @This();
99const flate = @import("../flate.zig");
1010const Compress = flate.Compress;
11const huffman = flate.huffman;
11const HuffmanEncoder = flate.HuffmanEncoder;
1212const Token = @import("Token.zig");
1313
14const codegen_order = huffman.codegen_order;
14const codegen_order = HuffmanEncoder.codegen_order;
1515const end_code_mark = 255;
1616
1717output: *Writer,
1818
19codegen_freq: [huffman.codegen_code_count]u16 = undefined,
20literal_freq: [huffman.max_num_lit]u16 = undefined,
21distance_freq: [huffman.distance_code_count]u16 = undefined,
22codegen: [huffman.max_num_lit + huffman.distance_code_count + 1]u8 = undefined,
23literal_encoding: Compress.LiteralEncoder = .{},
24distance_encoding: Compress.DistanceEncoder = .{},
25codegen_encoding: Compress.CodegenEncoder = .{},
26fixed_literal_encoding: Compress.LiteralEncoder,
27fixed_distance_encoding: Compress.DistanceEncoder,
28huff_distance: Compress.DistanceEncoder,
29
30pub fn init(output: *Writer) BlockWriter {
31 return .{
32 .output = output,
33 .fixed_literal_encoding = Compress.fixedLiteralEncoder(),
34 .fixed_distance_encoding = Compress.fixedDistanceEncoder(),
35 .huff_distance = Compress.huffmanDistanceEncoder(),
36 };
19codegen_freq: [HuffmanEncoder.codegen_code_count]u16,
20literal_freq: [HuffmanEncoder.max_num_lit]u16,
21distance_freq: [HuffmanEncoder.distance_code_count]u16,
22codegen: [HuffmanEncoder.max_num_lit + HuffmanEncoder.distance_code_count + 1]u8,
23literal_encoding: HuffmanEncoder,
24distance_encoding: HuffmanEncoder,
25codegen_encoding: HuffmanEncoder,
26fixed_literal_encoding: HuffmanEncoder,
27fixed_distance_encoding: HuffmanEncoder,
28huff_distance: HuffmanEncoder,
29
30fixed_literal_codes: [HuffmanEncoder.max_num_frequencies]HuffmanEncoder.Code,
31fixed_distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
32distance_codes: [HuffmanEncoder.distance_code_count]HuffmanEncoder.Code,
33
34pub fn init(bw: *BlockWriter) void {
35 bw.fixed_literal_encoding = .fixedLiteralEncoder(&bw.fixed_literal_codes);
36 bw.fixed_distance_encoding = .fixedDistanceEncoder(&bw.fixed_distance_codes);
37 bw.huff_distance = .huffmanDistanceEncoder(&bw.distance_codes);
3738}
3839
3940/// Flush intrenal bit buffer to the writer.
......@@ -46,27 +47,23 @@ pub fn flush(self: *BlockWriter) Writer.Error!void {
4647 try self.bit_writer.flush();
4748}
4849
49pub fn setWriter(self: *BlockWriter, new_writer: *Writer) void {
50 self.bit_writer.setWriter(new_writer);
51}
52
5350fn writeCode(self: *BlockWriter, c: Compress.HuffCode) Writer.Error!void {
5451 try self.bit_writer.writeBits(c.code, c.len);
5552}
5653
57// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
58// the literal and distance lengths arrays (which are concatenated into a single
59// array). This method generates that run-length encoding.
60//
61// The result is written into the codegen array, and the frequencies
62// of each code is written into the codegen_freq array.
63// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
64// information. Code bad_code is an end marker
65//
66// num_literals: The number of literals in literal_encoding
67// num_distances: The number of distances in distance_encoding
68// lit_enc: The literal encoder to use
69// dist_enc: The distance encoder to use
54/// RFC 1951 3.2.7 specifies a special run-length encoding for specifying
55/// the literal and distance lengths arrays (which are concatenated into a single
56/// array). This method generates that run-length encoding.
57///
58/// The result is written into the codegen array, and the frequencies
59/// of each code is written into the codegen_freq array.
60/// Codes 0-15 are single byte codes. Codes 16-18 are followed by additional
61/// information. Code bad_code is an end marker
62///
63/// num_literals: The number of literals in literal_encoding
64/// num_distances: The number of distances in distance_encoding
65/// lit_enc: The literal encoder to use
66/// dist_enc: The distance encoder to use
7067fn generateCodegen(
7168 self: *BlockWriter,
7269 num_literals: u32,
......@@ -167,7 +164,7 @@ const DynamicSize = struct {
167164 num_codegens: u32,
168165};
169166
170// dynamicSize returns the size of dynamically encoded data in bits.
167/// dynamicSize returns the size of dynamically encoded data in bits.
171168fn dynamicSize(
172169 self: *BlockWriter,
173170 lit_enc: *Compress.LiteralEncoder, // literal encoder
......@@ -194,7 +191,7 @@ fn dynamicSize(
194191 };
195192}
196193
197// fixedSize returns the size of dynamically encoded data in bits.
194/// fixedSize returns the size of dynamically encoded data in bits.
198195fn fixedSize(self: *BlockWriter, extra_bits: u32) u32 {
199196 return 3 +
200197 self.fixed_literal_encoding.bitLength(&self.literal_freq) +
......@@ -207,25 +204,25 @@ const StoredSize = struct {
207204 storable: bool,
208205};
209206
210// storedSizeFits calculates the stored size, including header.
211// The function returns the size in bits and whether the block
212// fits inside a single block.
207/// storedSizeFits calculates the stored size, including header.
208/// The function returns the size in bits and whether the block
209/// fits inside a single block.
213210fn storedSizeFits(in: ?[]const u8) StoredSize {
214211 if (in == null) {
215212 return .{ .size = 0, .storable = false };
216213 }
217 if (in.?.len <= huffman.max_store_block_size) {
214 if (in.?.len <= HuffmanEncoder.max_store_block_size) {
218215 return .{ .size = @as(u32, @intCast((in.?.len + 5) * 8)), .storable = true };
219216 }
220217 return .{ .size = 0, .storable = false };
221218}
222219
223// Write the header of a dynamic Huffman block to the output stream.
224//
225// num_literals: The number of literals specified in codegen
226// num_distances: The number of distances specified in codegen
227// num_codegens: The number of codegens used in codegen
228// eof: Is it the end-of-file? (end of stream)
220/// Write the header of a dynamic Huffman block to the output stream.
221///
222/// num_literals: The number of literals specified in codegen
223/// num_distances: The number of distances specified in codegen
224/// num_codegens: The number of codegens used in codegen
225/// eof: Is it the end-of-file? (end of stream)
229226fn dynamicHeader(
230227 self: *BlockWriter,
231228 num_literals: u32,
......@@ -291,11 +288,11 @@ fn fixedHeader(self: *BlockWriter, eof: bool) Writer.Error!void {
291288 try self.bit_writer.writeBits(value, 3);
292289}
293290
294// Write a block of tokens with the smallest encoding. Will choose block type.
295// The original input can be supplied, and if the huffman encoded data
296// is larger than the original bytes, the data will be written as a
297// stored block.
298// If the input is null, the tokens will always be Huffman encoded.
291/// Write a block of tokens with the smallest encoding. Will choose block type.
292/// The original input can be supplied, and if the huffman encoded data
293/// is larger than the original bytes, the data will be written as a
294/// stored block.
295/// If the input is null, the tokens will always be Huffman encoded.
299296pub fn write(self: *BlockWriter, tokens: []const Token, eof: bool, input: ?[]const u8) Writer.Error!void {
300297 const lit_and_dist = self.indexTokens(tokens);
301298 const num_literals = lit_and_dist.num_literals;
......@@ -379,11 +376,11 @@ pub fn storedBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Erro
379376 try self.bit_writer.writeBytes(input);
380377}
381378
382// writeBlockDynamic encodes a block using a dynamic Huffman table.
383// This should be used if the symbols used have a disproportionate
384// histogram distribution.
385// If input is supplied and the compression savings are below 1/16th of the
386// input size the block is stored.
379/// writeBlockDynamic encodes a block using a dynamic Huffman table.
380/// This should be used if the symbols used have a disproportionate
381/// histogram distribution.
382/// If input is supplied and the compression savings are below 1/16th of the
383/// input size the block is stored.
387384fn dynamicBlock(
388385 self: *BlockWriter,
389386 tokens: []const Token,
......@@ -429,10 +426,10 @@ const TotalIndexedTokens = struct {
429426 num_distances: u32,
430427};
431428
432// Indexes a slice of tokens followed by an end_block_marker, and updates
433// literal_freq and distance_freq, and generates literal_encoding
434// and distance_encoding.
435// The number of literal and distance tokens is returned.
429/// Indexes a slice of tokens followed by an end_block_marker, and updates
430/// literal_freq and distance_freq, and generates literal_encoding
431/// and distance_encoding.
432/// The number of literal and distance tokens is returned.
436433fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens {
437434 var num_literals: u32 = 0;
438435 var num_distances: u32 = 0;
......@@ -453,7 +450,7 @@ fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens {
453450 self.distance_freq[t.distanceCode()] += 1;
454451 }
455452 // add end_block_marker token at the end
456 self.literal_freq[huffman.end_block_marker] += 1;
453 self.literal_freq[HuffmanEncoder.end_block_marker] += 1;
457454
458455 // get the number of literals
459456 num_literals = @as(u32, @intCast(self.literal_freq.len));
......@@ -479,8 +476,8 @@ fn indexTokens(self: *BlockWriter, tokens: []const Token) TotalIndexedTokens {
479476 };
480477}
481478
482// Writes a slice of tokens to the output followed by and end_block_marker.
483// codes for literal and distance encoding must be supplied.
479/// Writes a slice of tokens to the output followed by and end_block_marker.
480/// codes for literal and distance encoding must be supplied.
484481fn writeTokens(
485482 self: *BlockWriter,
486483 tokens: []const Token,
......@@ -508,18 +505,18 @@ fn writeTokens(
508505 }
509506 }
510507 // add end_block_marker at the end
511 try self.writeCode(le_codes[huffman.end_block_marker]);
508 try self.writeCode(le_codes[HuffmanEncoder.end_block_marker]);
512509}
513510
514// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
515// if the results only gains very little from compression.
511/// Encodes a block of bytes as either Huffman encoded literals or uncompressed bytes
512/// if the results only gains very little from compression.
516513pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Error!void {
517514 // Add everything as literals
518515 histogram(input, &self.literal_freq);
519516
520 self.literal_freq[huffman.end_block_marker] = 1;
517 self.literal_freq[HuffmanEncoder.end_block_marker] = 1;
521518
522 const num_literals = huffman.end_block_marker + 1;
519 const num_literals = HuffmanEncoder.end_block_marker + 1;
523520 self.distance_freq[0] = 1;
524521 const num_distances = 1;
525522
......@@ -560,10 +557,9 @@ pub fn huffmanBlock(self: *BlockWriter, input: []const u8, eof: bool) Writer.Err
560557 const c = encoding[t];
561558 try self.bit_writer.writeBits(c.code, c.len);
562559 }
563 try self.writeCode(encoding[huffman.end_block_marker]);
560 try self.writeCode(encoding[HuffmanEncoder.end_block_marker]);
564561}
565562
566// histogram accumulates a histogram of b in h.
567563fn histogram(b: []const u8, h: *[286]u16) void {
568564 // Clear histogram
569565 for (h, 0..) |_, i| {
......@@ -575,122 +571,3 @@ fn histogram(b: []const u8, h: *[286]u16) void {
575571 lh[t] += 1;
576572 }
577573}
578
579// tests
580const expect = std.testing.expect;
581const fmt = std.fmt;
582const testing = std.testing;
583const ArrayList = std.ArrayList;
584
585const TestCase = @import("testdata/block_writer.zig").TestCase;
586const testCases = @import("testdata/block_writer.zig").testCases;
587
588// tests if the writeBlock encoding has changed.
589test "write" {
590 inline for (0..testCases.len) |i| {
591 try testBlock(testCases[i], .write_block);
592 }
593}
594
595// tests if the writeBlockDynamic encoding has changed.
596test "dynamicBlock" {
597 inline for (0..testCases.len) |i| {
598 try testBlock(testCases[i], .write_dyn_block);
599 }
600}
601
602test "huffmanBlock" {
603 inline for (0..testCases.len) |i| {
604 try testBlock(testCases[i], .write_huffman_block);
605 }
606 try testBlock(.{
607 .tokens = &[_]Token{},
608 .input = "huffman-rand-max.input",
609 .want = "huffman-rand-max.{s}.expect",
610 }, .write_huffman_block);
611}
612
613const TestFn = enum {
614 write_block,
615 write_dyn_block, // write dynamic block
616 write_huffman_block,
617
618 fn to_s(self: TestFn) []const u8 {
619 return switch (self) {
620 .write_block => "wb",
621 .write_dyn_block => "dyn",
622 .write_huffman_block => "huff",
623 };
624 }
625
626 fn write(
627 comptime self: TestFn,
628 bw: anytype,
629 tok: []const Token,
630 input: ?[]const u8,
631 final: bool,
632 ) !void {
633 switch (self) {
634 .write_block => try bw.write(tok, final, input),
635 .write_dyn_block => try bw.dynamicBlock(tok, final, input),
636 .write_huffman_block => try bw.huffmanBlock(input.?, final),
637 }
638 try bw.flush();
639 }
640};
641
642// testBlock tests a block against its references
643//
644// size
645// 64K [file-name].input - input non compressed file
646// 8.1K [file-name].golden -
647// 78 [file-name].dyn.expect - output with writeBlockDynamic
648// 78 [file-name].wb.expect - output with writeBlock
649// 8.1K [file-name].huff.expect - output with writeBlockHuff
650// 78 [file-name].dyn.expect-noinput - output with writeBlockDynamic when input is null
651// 78 [file-name].wb.expect-noinput - output with writeBlock when input is null
652//
653// wb - writeBlock
654// dyn - writeBlockDynamic
655// huff - writeBlockHuff
656//
657fn testBlock(comptime tc: TestCase, comptime tfn: TestFn) !void {
658 if (tc.input.len != 0 and tc.want.len != 0) {
659 const want_name = comptime fmt.comptimePrint(tc.want, .{tfn.to_s()});
660 const input = @embedFile("testdata/block_writer/" ++ tc.input);
661 const want = @embedFile("testdata/block_writer/" ++ want_name);
662 try testWriteBlock(tfn, input, want, tc.tokens);
663 }
664
665 if (tfn == .write_huffman_block) {
666 return;
667 }
668
669 const want_name_no_input = comptime fmt.comptimePrint(tc.want_no_input, .{tfn.to_s()});
670 const want = @embedFile("testdata/block_writer/" ++ want_name_no_input);
671 try testWriteBlock(tfn, null, want, tc.tokens);
672}
673
674// Uses writer function `tfn` to write `tokens`, tests that we got `want` as output.
675fn testWriteBlock(comptime tfn: TestFn, input: ?[]const u8, want: []const u8, tokens: []const Token) !void {
676 var buf = ArrayList(u8).init(testing.allocator);
677 var bw: BlockWriter = .init(buf.writer());
678 try tfn.write(&bw, tokens, input, false);
679 var got = buf.items;
680 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
681 try expect(got[0] & 0b0000_0001 == 0); // bfinal is not set
682 //
683 // Test if the writer produces the same output after reset.
684 buf.deinit();
685 buf = ArrayList(u8).init(testing.allocator);
686 defer buf.deinit();
687 bw.setWriter(buf.writer());
688
689 try tfn.write(&bw, tokens, input, true);
690 try bw.flush();
691 got = buf.items;
692
693 try expect(got[0] & 1 == 1); // bfinal is set
694 buf.items[0] &= 0b1111_1110; // remove bfinal bit, so we can run test slices
695 try testing.expectEqualSlices(u8, want, got); // expect writeBlock to yield expected result
696}
lib/std/compress/flate/Compress.zig+138-819
......@@ -39,6 +39,7 @@
3939//!
4040//!
4141//! Allocates statically ~400K (192K lookup, 128K tokens, 64K window).
42
4243const builtin = @import("builtin");
4344const std = @import("std");
4445const assert = std.debug.assert;
......@@ -47,7 +48,6 @@ const expect = testing.expect;
4748const mem = std.mem;
4849const math = std.math;
4950const Writer = std.Io.Writer;
50const Reader = std.Io.Reader;
5151
5252const Compress = @This();
5353const Token = @import("Token.zig");
......@@ -55,22 +55,24 @@ const BlockWriter = @import("BlockWriter.zig");
5555const flate = @import("../flate.zig");
5656const Container = flate.Container;
5757const Lookup = @import("Lookup.zig");
58const huffman = flate.huffman;
58const HuffmanEncoder = flate.HuffmanEncoder;
59const LiteralNode = HuffmanEncoder.LiteralNode;
5960
6061lookup: Lookup = .{},
6162tokens: Tokens = .{},
62/// Asserted to have a buffer capacity of at least `flate.max_window_len`.
63input: *Reader,
6463block_writer: BlockWriter,
6564level: LevelArgs,
6665hasher: Container.Hasher,
67reader: Reader,
66writer: Writer,
67state: State,
6868
6969// Match and literal at the previous position.
7070// Used for lazy match finding in processWindow.
7171prev_match: ?Token = null,
7272prev_literal: ?u8 = null,
7373
74pub const State = enum { header, middle, ended };
75
7476/// Trades between speed and compression size.
7577/// Starts with level 4: in [zlib](https://github.com/madler/zlib/blob/abd3d1a28930f89375d4b41408b39f6c1be157b2/deflate.c#L115C1-L117C43)
7678/// levels 1-3 are using different algorithm to perform faster but with less
......@@ -118,188 +120,34 @@ pub const Options = struct {
118120 container: Container = .raw,
119121};
120122
121pub fn init(input: *Reader, buffer: []u8, options: Options) Compress {
123pub fn init(output: *Writer, buffer: []u8, options: Options) Compress {
122124 return .{
123 .input = input,
124 .block_writer = undefined,
125 .block_writer = .{
126 .output = output,
127 .codegen_freq = undefined,
128 .literal_freq = undefined,
129 .distance_freq = undefined,
130 .codegen = undefined,
131 .literal_encoding = undefined,
132 .distance_encoding = undefined,
133 .codegen_encoding = undefined,
134 .fixed_literal_encoding = undefined,
135 .fixed_distance_encoding = undefined,
136 .huff_distance = undefined,
137 .fixed_literal_codes = undefined,
138 .fixed_distance_codes = undefined,
139 .distance_codes = undefined,
140 },
125141 .level = .get(options.level),
126142 .hasher = .init(options.container),
127143 .state = .header,
128 .reader = .{
144 .writer = .{
129145 .buffer = buffer,
130 .stream = stream,
146 .vtable = &.{ .drain = drain },
131147 },
132148 };
133149}
134150
135const FlushOption = enum { none, flush, final };
136
137/// Process data in window and create tokens. If token buffer is full
138/// flush tokens to the token writer.
139///
140/// Returns number of bytes consumed from `lh`.
141fn tokenizeSlice(c: *Compress, bw: *Writer, limit: std.Io.Limit, lh: []const u8) !usize {
142 _ = bw;
143 _ = limit;
144 if (true) @panic("TODO");
145 var step: u16 = 1; // 1 in the case of literal, match length otherwise
146 const pos: u16 = c.win.pos();
147 const literal = lh[0]; // literal at current position
148 const min_len: u16 = if (c.prev_match) |m| m.length() else 0;
149
150 // Try to find match at least min_len long.
151 if (c.findMatch(pos, lh, min_len)) |match| {
152 // Found better match than previous.
153 try c.addPrevLiteral();
154
155 // Is found match length good enough?
156 if (match.length() >= c.level.lazy) {
157 // Don't try to lazy find better match, use this.
158 step = try c.addMatch(match);
159 } else {
160 // Store this match.
161 c.prev_literal = literal;
162 c.prev_match = match;
163 }
164 } else {
165 // There is no better match at current pos then it was previous.
166 // Write previous match or literal.
167 if (c.prev_match) |m| {
168 // Write match from previous position.
169 step = try c.addMatch(m) - 1; // we already advanced 1 from previous position
170 } else {
171 // No match at previous position.
172 // Write previous literal if any, and remember this literal.
173 try c.addPrevLiteral();
174 c.prev_literal = literal;
175 }
176 }
177 // Advance window and add hashes.
178 c.windowAdvance(step, lh, pos);
179}
180
181fn windowAdvance(self: *Compress, step: u16, lh: []const u8, pos: u16) void {
182 // current position is already added in findMatch
183 self.lookup.bulkAdd(lh[1..], step - 1, pos + 1);
184 self.win.advance(step);
185}
186
187// Add previous literal (if any) to the tokens list.
188fn addPrevLiteral(self: *Compress) !void {
189 if (self.prev_literal) |l| try self.addToken(Token.initLiteral(l));
190}
191
192// Add match to the tokens list, reset prev pointers.
193// Returns length of the added match.
194fn addMatch(self: *Compress, m: Token) !u16 {
195 try self.addToken(m);
196 self.prev_literal = null;
197 self.prev_match = null;
198 return m.length();
199}
200
201fn addToken(self: *Compress, token: Token) !void {
202 self.tokens.add(token);
203 if (self.tokens.full()) try self.flushTokens(.none);
204}
205
206// Finds largest match in the history window with the data at current pos.
207fn findMatch(self: *Compress, pos: u16, lh: []const u8, min_len: u16) ?Token {
208 var len: u16 = min_len;
209 // Previous location with the same hash (same 4 bytes).
210 var prev_pos = self.lookup.add(lh, pos);
211 // Last found match.
212 var match: ?Token = null;
213
214 // How much back-references to try, performance knob.
215 var chain: usize = self.level.chain;
216 if (len >= self.level.good) {
217 // If we've got a match that's good enough, only look in 1/4 the chain.
218 chain >>= 2;
219 }
220
221 // Hot path loop!
222 while (prev_pos > 0 and chain > 0) : (chain -= 1) {
223 const distance = pos - prev_pos;
224 if (distance > flate.match.max_distance)
225 break;
226
227 const new_len = self.win.match(prev_pos, pos, len);
228 if (new_len > len) {
229 match = Token.initMatch(@intCast(distance), new_len);
230 if (new_len >= self.level.nice) {
231 // The match is good enough that we don't try to find a better one.
232 return match;
233 }
234 len = new_len;
235 }
236 prev_pos = self.lookup.prev(prev_pos);
237 }
238
239 return match;
240}
241
242fn flushTokens(self: *Compress, flush_opt: FlushOption) !void {
243 // Pass tokens to the token writer
244 try self.block_writer.write(self.tokens.tokens(), flush_opt == .final, self.win.tokensBuffer());
245 // Stored block ensures byte alignment.
246 // It has 3 bits (final, block_type) and then padding until byte boundary.
247 // After that everything is aligned to the boundary in the stored block.
248 // Empty stored block is Ob000 + (0-7) bits of padding + 0x00 0x00 0xFF 0xFF.
249 // Last 4 bytes are byte aligned.
250 if (flush_opt == .flush) {
251 try self.block_writer.storedBlock("", false);
252 }
253 if (flush_opt != .none) {
254 // Safe to call only when byte aligned or it is OK to add
255 // padding bits (on last byte of the final block).
256 try self.block_writer.flush();
257 }
258 // Reset internal tokens store.
259 self.tokens.reset();
260 // Notify win that tokens are flushed.
261 self.win.flush();
262}
263
264// Slide win and if needed lookup tables.
265fn slide(self: *Compress) void {
266 const n = self.win.slide();
267 self.lookup.slide(n);
268}
269
270/// Flushes internal buffers to the output writer. Outputs empty stored
271/// block to sync bit stream to the byte boundary, so that the
272/// decompressor can get all input data available so far.
273///
274/// It is useful mainly in compressed network protocols, to ensure that
275/// deflate bit stream can be used as byte stream. May degrade
276/// compression so it should be used only when necessary.
277///
278/// Completes the current deflate block and follows it with an empty
279/// stored block that is three zero bits plus filler bits to the next
280/// byte, followed by four bytes (00 00 ff ff).
281///
282pub fn flush(c: *Compress) !void {
283 try c.tokenize(.flush);
284}
285
286/// Completes deflate bit stream by writing any pending data as deflate
287/// final deflate block. HAS to be called once all data are written to
288/// the compressor as a signal that next block has to have final bit
289/// set.
290///
291pub fn finish(c: *Compress) !void {
292 _ = c;
293 @panic("TODO");
294}
295
296/// Use another writer while preserving history. Most probably flush
297/// should be called on old writer before setting new.
298pub fn setWriter(self: *Compress, new_writer: *Writer) void {
299 self.block_writer.setWriter(new_writer);
300 self.output = new_writer;
301}
302
303151// Tokens store
304152const Tokens = struct {
305153 list: [n_tokens]Token = undefined,
......@@ -323,527 +171,110 @@ const Tokens = struct {
323171 }
324172};
325173
326/// Creates huffman only deflate blocks. Disables Lempel-Ziv match searching and
327/// only performs Huffman entropy encoding. Results in faster compression, much
328/// less memory requirements during compression but bigger compressed sizes.
329pub const Huffman = SimpleCompressor(.huffman, .raw);
330
331/// Creates store blocks only. Data are not compressed only packed into deflate
332/// store blocks. That adds 9 bytes of header for each block. Max stored block
333/// size is 64K. Block is emitted when flush is called on on finish.
334pub const store = struct {
335 pub fn Compressor(comptime container: Container, comptime WriterType: type) type {
336 return SimpleCompressor(.store, container, WriterType);
337 }
338
339 pub fn compressor(comptime container: Container, writer: anytype) !store.Compressor(container, @TypeOf(writer)) {
340 return try store.Compressor(container, @TypeOf(writer)).init(writer);
174fn drain(me: *Writer, data: []const []const u8, splat: usize) Writer.Error!usize {
175 _ = data;
176 _ = splat;
177 const c: *Compress = @fieldParentPtr("writer", me);
178 const out = c.block_writer.output;
179 switch (c.state) {
180 .header => {
181 c.state = .middle;
182 const header = c.hasher.container().header();
183 try out.writeAll(header);
184 return header.len;
185 },
186 .middle => {},
187 .ended => unreachable,
341188 }
342};
343189
344const SimpleCompressorKind = enum {
345 huffman,
346 store,
347};
190 const buffered = me.buffered();
191 const min_lookahead = flate.match.min_length + flate.match.max_length;
192 const history_plus_lookahead_len = flate.history_len + min_lookahead;
193 if (buffered.len < history_plus_lookahead_len) return 0;
194 const lookahead = buffered[flate.history_len..];
348195
349fn simpleCompressor(
350 comptime kind: SimpleCompressorKind,
351 comptime container: Container,
352 writer: anytype,
353) !SimpleCompressor(kind, container, @TypeOf(writer)) {
354 return try SimpleCompressor(kind, container, @TypeOf(writer)).init(writer);
196 _ = lookahead;
197 // TODO tokenize
198 //c.hasher.update(lookahead[0..n]);
199 @panic("TODO");
355200}
356201
357fn SimpleCompressor(
358 comptime kind: SimpleCompressorKind,
359 comptime container: Container,
360 comptime WriterType: type,
361) type {
362 const BlockWriterType = BlockWriter(WriterType);
363 return struct {
364 buffer: [65535]u8 = undefined, // because store blocks are limited to 65535 bytes
365 wp: usize = 0,
366
367 output: WriterType,
368 block_writer: BlockWriterType,
369 hasher: container.Hasher() = .{},
370
371 const Self = @This();
372
373 pub fn init(output: WriterType) !Self {
374 const self = Self{
375 .output = output,
376 .block_writer = BlockWriterType.init(output),
377 };
378 try container.writeHeader(self.output);
379 return self;
380 }
381
382 pub fn flush(self: *Self) !void {
383 try self.flushBuffer(false);
384 try self.block_writer.storedBlock("", false);
385 try self.block_writer.flush();
386 }
387
388 pub fn finish(self: *Self) !void {
389 try self.flushBuffer(true);
390 try self.block_writer.flush();
391 try container.writeFooter(&self.hasher, self.output);
392 }
393
394 fn flushBuffer(self: *Self, final: bool) !void {
395 const buf = self.buffer[0..self.wp];
396 switch (kind) {
397 .huffman => try self.block_writer.huffmanBlock(buf, final),
398 .store => try self.block_writer.storedBlock(buf, final),
399 }
400 self.wp = 0;
401 }
402 };
202pub fn end(c: *Compress) !void {
203 try endUnflushed(c);
204 try c.output.flush();
403205}
404206
405const LiteralNode = struct {
406 literal: u16,
407 freq: u16,
408};
409
410// Describes the state of the constructed tree for a given depth.
411const LevelInfo = struct {
412 // Our level. for better printing
413 level: u32,
414
415 // The frequency of the last node at this level
416 last_freq: u32,
207pub fn endUnflushed(c: *Compress) !void {
208 while (c.writer.end != 0) _ = try drain(&c.writer, &.{""}, 1);
209 c.state = .ended;
417210
418 // The frequency of the next character to add to this level
419 next_char_freq: u32,
211 const out = c.block_writer.output;
420212
421 // The frequency of the next pair (from level below) to add to this level.
422 // Only valid if the "needed" value of the next lower level is 0.
423 next_pair_freq: u32,
424
425 // The number of chains remaining to generate for this level before moving
426 // up to the next level
427 needed: u32,
428};
213 // TODO flush tokens
429214
430// hcode is a huffman code with a bit code and bit length.
431pub const HuffCode = struct {
432 code: u16 = 0,
433 len: u16 = 0,
434
435 // set sets the code and length of an hcode.
436 fn set(self: *HuffCode, code: u16, length: u16) void {
437 self.len = length;
438 self.code = code;
215 switch (c.hasher) {
216 .gzip => |*gzip| {
217 // GZIP 8 bytes footer
218 // - 4 bytes, CRC32 (CRC-32)
219 // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
220 const footer = try out.writableArray(8);
221 std.mem.writeInt(u32, footer[0..4], gzip.crc.final(), .little);
222 std.mem.writeInt(u32, footer[4..8], @truncate(gzip.count), .little);
223 },
224 .zlib => |*zlib| {
225 // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
226 // 4 bytes of ADLER32 (Adler-32 checksum)
227 // Checksum value of the uncompressed data (excluding any
228 // dictionary data) computed according to Adler-32
229 // algorithm.
230 std.mem.writeInt(u32, try out.writableArray(4), zlib.final, .big);
231 },
232 .raw => {},
439233 }
440};
441
442pub fn HuffmanEncoder(comptime size: usize) type {
443 return struct {
444 codes: [size]HuffCode = undefined,
445 // Reusable buffer with the longest possible frequency table.
446 freq_cache: [huffman.max_num_frequencies + 1]LiteralNode = undefined,
447 bit_count: [17]u32 = undefined,
448 lns: []LiteralNode = undefined, // sorted by literal, stored to avoid repeated allocation in generate
449 lfs: []LiteralNode = undefined, // sorted by frequency, stored to avoid repeated allocation in generate
450
451 const Self = @This();
452
453 // Update this Huffman Code object to be the minimum code for the specified frequency count.
454 //
455 // freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
456 // max_bits The maximum number of bits to use for any literal.
457 pub fn generate(self: *Self, freq: []u16, max_bits: u32) void {
458 var list = self.freq_cache[0 .. freq.len + 1];
459 // Number of non-zero literals
460 var count: u32 = 0;
461 // Set list to be the set of all non-zero literals and their frequencies
462 for (freq, 0..) |f, i| {
463 if (f != 0) {
464 list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
465 count += 1;
466 } else {
467 list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
468 self.codes[i].len = 0;
469 }
470 }
471 list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
472
473 list = list[0..count];
474 if (count <= 2) {
475 // Handle the small cases here, because they are awkward for the general case code. With
476 // two or fewer literals, everything has bit length 1.
477 for (list, 0..) |node, i| {
478 // "list" is in order of increasing literal value.
479 self.codes[node.literal].set(@as(u16, @intCast(i)), 1);
480 }
481 return;
482 }
483 self.lfs = list;
484 mem.sort(LiteralNode, self.lfs, {}, byFreq);
485
486 // Get the number of literals for each bit count
487 const bit_count = self.bitCounts(list, max_bits);
488 // And do the assignment
489 self.assignEncodingAndSize(bit_count, list);
490 }
491
492 pub fn bitLength(self: *Self, freq: []u16) u32 {
493 var total: u32 = 0;
494 for (freq, 0..) |f, i| {
495 if (f != 0) {
496 total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
497 }
498 }
499 return total;
500 }
501
502 // Return the number of literals assigned to each bit size in the Huffman encoding
503 //
504 // This method is only called when list.len >= 3
505 // The cases of 0, 1, and 2 literals are handled by special case code.
506 //
507 // list: An array of the literals with non-zero frequencies
508 // and their associated frequencies. The array is in order of increasing
509 // frequency, and has as its last element a special element with frequency
510 // `math.maxInt(i32)`
511 //
512 // max_bits: The maximum number of bits that should be used to encode any literal.
513 // Must be less than 16.
514 //
515 // Returns an integer array in which array[i] indicates the number of literals
516 // that should be encoded in i bits.
517 fn bitCounts(self: *Self, list: []LiteralNode, max_bits_to_use: usize) []u32 {
518 var max_bits = max_bits_to_use;
519 const n = list.len;
520 const max_bits_limit = 16;
521
522 assert(max_bits < max_bits_limit);
523
524 // The tree can't have greater depth than n - 1, no matter what. This
525 // saves a little bit of work in some small cases
526 max_bits = @min(max_bits, n - 1);
527
528 // Create information about each of the levels.
529 // A bogus "Level 0" whose sole purpose is so that
530 // level1.prev.needed == 0. This makes level1.next_pair_freq
531 // be a legitimate value that never gets chosen.
532 var levels: [max_bits_limit]LevelInfo = mem.zeroes([max_bits_limit]LevelInfo);
533 // leaf_counts[i] counts the number of literals at the left
534 // of ancestors of the rightmost node at level i.
535 // leaf_counts[i][j] is the number of literals at the left
536 // of the level j ancestor.
537 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = mem.zeroes([max_bits_limit][max_bits_limit]u32);
538
539 {
540 var level = @as(u32, 1);
541 while (level <= max_bits) : (level += 1) {
542 // For every level, the first two items are the first two characters.
543 // We initialize the levels as if we had already figured this out.
544 levels[level] = LevelInfo{
545 .level = level,
546 .last_freq = list[1].freq,
547 .next_char_freq = list[2].freq,
548 .next_pair_freq = list[0].freq + list[1].freq,
549 .needed = 0,
550 };
551 leaf_counts[level][level] = 2;
552 if (level == 1) {
553 levels[level].next_pair_freq = math.maxInt(i32);
554 }
555 }
556 }
557
558 // We need a total of 2*n - 2 items at top level and have already generated 2.
559 levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
560
561 {
562 var level = max_bits;
563 while (true) {
564 var l = &levels[level];
565 if (l.next_pair_freq == math.maxInt(i32) and l.next_char_freq == math.maxInt(i32)) {
566 // We've run out of both leaves and pairs.
567 // End all calculations for this level.
568 // To make sure we never come back to this level or any lower level,
569 // set next_pair_freq impossibly large.
570 l.needed = 0;
571 levels[level + 1].next_pair_freq = math.maxInt(i32);
572 level += 1;
573 continue;
574 }
575
576 const prev_freq = l.last_freq;
577 if (l.next_char_freq < l.next_pair_freq) {
578 // The next item on this row is a leaf node.
579 const next = leaf_counts[level][level] + 1;
580 l.last_freq = l.next_char_freq;
581 // Lower leaf_counts are the same of the previous node.
582 leaf_counts[level][level] = next;
583 if (next >= list.len) {
584 l.next_char_freq = maxNode().freq;
585 } else {
586 l.next_char_freq = list[next].freq;
587 }
588 } else {
589 // The next item on this row is a pair from the previous row.
590 // next_pair_freq isn't valid until we generate two
591 // more values in the level below
592 l.last_freq = l.next_pair_freq;
593 // Take leaf counts from the lower level, except counts[level] remains the same.
594 @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
595 levels[l.level - 1].needed = 2;
596 }
597
598 l.needed -= 1;
599 if (l.needed == 0) {
600 // We've done everything we need to do for this level.
601 // Continue calculating one level up. Fill in next_pair_freq
602 // of that level with the sum of the two nodes we've just calculated on
603 // this level.
604 if (l.level == max_bits) {
605 // All done!
606 break;
607 }
608 levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
609 level += 1;
610 } else {
611 // If we stole from below, move down temporarily to replenish it.
612 while (levels[level - 1].needed > 0) {
613 level -= 1;
614 if (level == 0) {
615 break;
616 }
617 }
618 }
619 }
620 }
621
622 // Somethings is wrong if at the end, the top level is null or hasn't used
623 // all of the leaves.
624 assert(leaf_counts[max_bits][max_bits] == n);
625
626 var bit_count = self.bit_count[0 .. max_bits + 1];
627 var bits: u32 = 1;
628 const counts = &leaf_counts[max_bits];
629 {
630 var level = max_bits;
631 while (level > 0) : (level -= 1) {
632 // counts[level] gives the number of literals requiring at least "bits"
633 // bits to encode.
634 bit_count[bits] = counts[level] - counts[level - 1];
635 bits += 1;
636 if (level == 0) {
637 break;
638 }
639 }
640 }
641 return bit_count;
642 }
643
644 // Look at the leaves and assign them a bit count and an encoding as specified
645 // in RFC 1951 3.2.2
646 fn assignEncodingAndSize(self: *Self, bit_count: []u32, list_arg: []LiteralNode) void {
647 var code = @as(u16, 0);
648 var list = list_arg;
649
650 for (bit_count, 0..) |bits, n| {
651 code <<= 1;
652 if (n == 0 or bits == 0) {
653 continue;
654 }
655 // The literals list[list.len-bits] .. list[list.len-bits]
656 // are encoded using "bits" bits, and get the values
657 // code, code + 1, .... The code values are
658 // assigned in literal order (not frequency order).
659 const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
660
661 self.lns = chunk;
662 mem.sort(LiteralNode, self.lns, {}, byLiteral);
663
664 for (chunk) |node| {
665 self.codes[node.literal] = HuffCode{
666 .code = bitReverse(u16, code, @as(u5, @intCast(n))),
667 .len = @as(u16, @intCast(n)),
668 };
669 code += 1;
670 }
671 list = list[0 .. list.len - @as(u32, @intCast(bits))];
672 }
673 }
674 };
675234}
676235
677fn maxNode() LiteralNode {
678 return LiteralNode{
679 .literal = math.maxInt(u16),
680 .freq = math.maxInt(u16),
681 };
682}
236pub const Simple = struct {
237 /// Note that store blocks are limited to 65535 bytes.
238 buffer: []u8,
239 wp: usize,
240 block_writer: BlockWriter,
241 hasher: Container.Hasher,
242 strategy: Strategy,
683243
684pub fn huffmanEncoder(comptime size: u32) HuffmanEncoder(size) {
685 return .{};
686}
244 pub const Strategy = enum { huffman, store };
687245
688pub const LiteralEncoder = HuffmanEncoder(huffman.max_num_frequencies);
689pub const DistanceEncoder = HuffmanEncoder(huffman.distance_code_count);
690pub const CodegenEncoder = HuffmanEncoder(19);
691
692// Generates a HuffmanCode corresponding to the fixed literal table
693pub fn fixedLiteralEncoder() LiteralEncoder {
694 var h: LiteralEncoder = undefined;
695 var ch: u16 = 0;
696
697 while (ch < huffman.max_num_frequencies) : (ch += 1) {
698 var bits: u16 = undefined;
699 var size: u16 = undefined;
700 switch (ch) {
701 0...143 => {
702 // size 8, 000110000 .. 10111111
703 bits = ch + 48;
704 size = 8;
705 },
706 144...255 => {
707 // size 9, 110010000 .. 111111111
708 bits = ch + 400 - 144;
709 size = 9;
710 },
711 256...279 => {
712 // size 7, 0000000 .. 0010111
713 bits = ch - 256;
714 size = 7;
715 },
716 else => {
717 // size 8, 11000000 .. 11000111
718 bits = ch + 192 - 280;
719 size = 8;
720 },
721 }
722 h.codes[ch] = HuffCode{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
246 pub fn init(out: *Writer, buffer: []u8, container: Container) !Simple {
247 const self: Simple = .{
248 .buffer = buffer,
249 .wp = 0,
250 .block_writer = .init(out),
251 .hasher = .init(container),
252 };
253 try container.writeHeader(self.out);
254 return self;
723255 }
724 return h;
725}
726256
727pub fn fixedDistanceEncoder() DistanceEncoder {
728 var h: DistanceEncoder = undefined;
729 for (h.codes, 0..) |_, ch| {
730 h.codes[ch] = HuffCode{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
257 pub fn flush(self: *Simple) !void {
258 try self.flushBuffer(false);
259 try self.block_writer.storedBlock("", false);
260 try self.block_writer.flush();
731261 }
732 return h;
733}
734262
735pub fn huffmanDistanceEncoder() DistanceEncoder {
736 var distance_freq = [1]u16{0} ** huffman.distance_code_count;
737 distance_freq[0] = 1;
738 // huff_distance is a static distance encoder used for huffman only encoding.
739 // It can be reused since we will not be encoding distance values.
740 var h: DistanceEncoder = .{};
741 h.generate(distance_freq[0..], 15);
742 return h;
743}
744
745fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
746 _ = context;
747 return a.literal < b.literal;
748}
749
750fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
751 _ = context;
752 if (a.freq == b.freq) {
753 return a.literal < b.literal;
263 pub fn finish(self: *Simple) !void {
264 try self.flushBuffer(true);
265 try self.block_writer.flush();
266 try self.hasher.container().writeFooter(&self.hasher, self.out);
754267 }
755 return a.freq < b.freq;
756}
757268
758fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!usize {
759 const c: *Compress = @fieldParentPtr("reader", r);
760 switch (c.state) {
761 .header => |i| {
762 const header = c.hasher.container().header();
763 const n = try w.write(header[i..]);
764 if (header.len - i - n == 0) {
765 c.state = .middle;
766 } else {
767 c.state.header += n;
768 }
769 return n;
770 },
771 .middle => {
772 c.input.fillMore() catch |err| switch (err) {
773 error.EndOfStream => {
774 c.state = .final;
775 return 0;
776 },
777 else => |e| return e,
778 };
779 const buffer_contents = c.input.buffered();
780 const min_lookahead = flate.match.min_length + flate.match.max_length;
781 const history_plus_lookahead_len = flate.history_len + min_lookahead;
782 if (buffer_contents.len < history_plus_lookahead_len) return 0;
783 const lookahead = buffer_contents[flate.history_len..];
784 const start = w.count;
785 const n = try c.tokenizeSlice(w, limit, lookahead) catch |err| switch (err) {
786 error.WriteFailed => return error.WriteFailed,
787 };
788 c.hasher.update(lookahead[0..n]);
789 c.input.toss(n);
790 return w.count - start;
791 },
792 .final => {
793 const buffer_contents = c.input.buffered();
794 const start = w.count;
795 const n = c.tokenizeSlice(w, limit, buffer_contents) catch |err| switch (err) {
796 error.WriteFailed => return error.WriteFailed,
797 };
798 if (buffer_contents.len - n == 0) {
799 c.hasher.update(buffer_contents);
800 c.input.tossAll();
801 {
802 // In the case of flushing, last few lookahead buffers were
803 // smaller than min match len, so only last literal can be
804 // unwritten.
805 assert(c.prev_match == null);
806 try c.addPrevLiteral();
807 c.prev_literal = null;
808
809 try c.flushTokens(.final);
810 }
811 switch (c.hasher) {
812 .gzip => |*gzip| {
813 // GZIP 8 bytes footer
814 // - 4 bytes, CRC32 (CRC-32)
815 // - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
816 comptime assert(c.footer_buffer.len == 8);
817 std.mem.writeInt(u32, c.footer_buffer[0..4], gzip.final(), .little);
818 std.mem.writeInt(u32, c.footer_buffer[4..8], gzip.bytes_read, .little);
819 c.state = .{ .footer = 0 };
820 },
821 .zlib => |*zlib| {
822 // ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
823 // 4 bytes of ADLER32 (Adler-32 checksum)
824 // Checksum value of the uncompressed data (excluding any
825 // dictionary data) computed according to Adler-32
826 // algorithm.
827 comptime assert(c.footer_buffer.len == 8);
828 std.mem.writeInt(u32, c.footer_buffer[4..8], zlib.final, .big);
829 c.state = .{ .footer = 4 };
830 },
831 .raw => {
832 c.state = .ended;
833 },
834 }
835 }
836 return w.count - start;
837 },
838 .ended => return error.EndOfStream,
839 .footer => |i| {
840 const remaining = c.footer_buffer[i..];
841 const n = try w.write(limit.slice(remaining));
842 c.state = if (n == remaining) .ended else .{ .footer = i - n };
843 return n;
844 },
269 fn flushBuffer(self: *Simple, final: bool) !void {
270 const buf = self.buffer[0..self.wp];
271 switch (self.strategy) {
272 .huffman => try self.block_writer.huffmanBlock(buf, final),
273 .store => try self.block_writer.storedBlock(buf, final),
274 }
275 self.wp = 0;
845276 }
846}
277};
847278
848279test "generate a Huffman code from an array of frequencies" {
849280 var freqs: [19]u16 = [_]u16{
......@@ -868,7 +299,8 @@ test "generate a Huffman code from an array of frequencies" {
868299 5, // 18
869300 };
870301
871 var enc = huffmanEncoder(19);
302 var codes: [19]HuffmanEncoder.Code = undefined;
303 var enc: HuffmanEncoder = .{ .codes = &codes };
872304 enc.generate(freqs[0..], 7);
873305
874306 try testing.expectEqual(@as(u32, 141), enc.bitLength(freqs[0..]));
......@@ -906,120 +338,6 @@ test "generate a Huffman code from an array of frequencies" {
906338 try testing.expectEqual(@as(u16, 0x3f), enc.codes[16].code);
907339}
908340
909test "generate a Huffman code for the fixed literal table specific to Deflate" {
910 const enc = fixedLiteralEncoder();
911 for (enc.codes) |c| {
912 switch (c.len) {
913 7 => {
914 const v = @bitReverse(@as(u7, @intCast(c.code)));
915 try testing.expect(v <= 0b0010111);
916 },
917 8 => {
918 const v = @bitReverse(@as(u8, @intCast(c.code)));
919 try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
920 (v >= 0b11000000 and v <= 11000111));
921 },
922 9 => {
923 const v = @bitReverse(@as(u9, @intCast(c.code)));
924 try testing.expect(v >= 0b110010000 and v <= 0b111111111);
925 },
926 else => unreachable,
927 }
928 }
929}
930
931test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
932 const enc = fixedDistanceEncoder();
933 for (enc.codes) |c| {
934 const v = @bitReverse(@as(u5, @intCast(c.code)));
935 try testing.expect(v <= 29);
936 try testing.expect(c.len == 5);
937 }
938}
939
940// Reverse bit-by-bit a N-bit code.
941fn bitReverse(comptime T: type, value: T, n: usize) T {
942 const r = @bitReverse(value);
943 return r >> @as(math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
944}
945
946test bitReverse {
947 const ReverseBitsTest = struct {
948 in: u16,
949 bit_count: u5,
950 out: u16,
951 };
952
953 const reverse_bits_tests = [_]ReverseBitsTest{
954 .{ .in = 1, .bit_count = 1, .out = 1 },
955 .{ .in = 1, .bit_count = 2, .out = 2 },
956 .{ .in = 1, .bit_count = 3, .out = 4 },
957 .{ .in = 1, .bit_count = 4, .out = 8 },
958 .{ .in = 1, .bit_count = 5, .out = 16 },
959 .{ .in = 17, .bit_count = 5, .out = 17 },
960 .{ .in = 257, .bit_count = 9, .out = 257 },
961 .{ .in = 29, .bit_count = 5, .out = 23 },
962 };
963
964 for (reverse_bits_tests) |h| {
965 const v = bitReverse(u16, h.in, h.bit_count);
966 try std.testing.expectEqual(h.out, v);
967 }
968}
969
970test "fixedLiteralEncoder codes" {
971 var al = std.ArrayList(u8).init(testing.allocator);
972 defer al.deinit();
973 var bw = std.Io.bitWriter(.little, al.writer());
974
975 const f = fixedLiteralEncoder();
976 for (f.codes) |c| {
977 try bw.writeBits(c.code, c.len);
978 }
979 try testing.expectEqualSlices(u8, &fixed_codes, al.items);
980}
981
982pub const fixed_codes = [_]u8{
983 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
984 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
985 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
986 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
987 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
988 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
989 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
990 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
991 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
992 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
993 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
994 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
995 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
996 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
997 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
998 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
999 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
1000 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
1001 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
1002 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
1003 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
1004 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
1005 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
1006 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
1007 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
1008 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
1009 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
1010 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
1011 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
1012 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
1013 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
1014 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
1015 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
1016 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
1017 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
1018 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
1019 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
1020 0b10100011,
1021};
1022
1023341test "tokenization" {
1024342 const L = Token.initLiteral;
1025343 const M = Token.initMatch;
......@@ -1133,7 +451,7 @@ test "file tokenization" {
1133451 const data = case.data;
1134452
1135453 for (levels, 0..) |level, i| { // for each compression level
1136 var original: Reader = .fixed(data);
454 var original: std.Io.Reader = .fixed(data);
1137455
1138456 // buffer for decompressed data
1139457 var al = std.ArrayList(u8).init(testing.allocator);
......@@ -1198,32 +516,33 @@ const TokenDecoder = struct {
1198516};
1199517
1200518test "store simple compressor" {
1201 const data = "Hello world!";
1202 const expected = [_]u8{
1203 0x1, // block type 0, final bit set
1204 0xc, 0x0, // len = 12
1205 0xf3, 0xff, // ~len
1206 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
1207 //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
1208 };
1209
1210 var fbs: Reader = .fixed(data);
1211 var al = std.ArrayList(u8).init(testing.allocator);
1212 defer al.deinit();
1213
1214 var cmp = try store.compressor(.raw, al.writer());
1215 try cmp.compress(&fbs);
1216 try cmp.finish();
1217 try testing.expectEqualSlices(u8, &expected, al.items);
1218
1219 fbs = .fixed(data);
1220 try al.resize(0);
1221
1222 // huffman only compresoor will also emit store block for this small sample
1223 var hc = try huffman.compressor(.raw, al.writer());
1224 try hc.compress(&fbs);
1225 try hc.finish();
1226 try testing.expectEqualSlices(u8, &expected, al.items);
519 if (true) return error.SkipZigTest;
520 //const data = "Hello world!";
521 //const expected = [_]u8{
522 // 0x1, // block type 0, final bit set
523 // 0xc, 0x0, // len = 12
524 // 0xf3, 0xff, // ~len
525 // 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', '!', //
526 // //0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
527 //};
528
529 //var fbs: std.Io.Reader = .fixed(data);
530 //var al = std.ArrayList(u8).init(testing.allocator);
531 //defer al.deinit();
532
533 //var cmp = try store.compressor(.raw, al.writer());
534 //try cmp.compress(&fbs);
535 //try cmp.finish();
536 //try testing.expectEqualSlices(u8, &expected, al.items);
537
538 //fbs = .fixed(data);
539 //try al.resize(0);
540
541 //// huffman only compresoor will also emit store block for this small sample
542 //var hc = try huffman.compressor(.raw, al.writer());
543 //try hc.compress(&fbs);
544 //try hc.finish();
545 //try testing.expectEqualSlices(u8, &expected, al.items);
1227546}
1228547
1229548test "sliding window match" {
lib/std/compress/flate/Decompress.zig+2-3
......@@ -620,10 +620,9 @@ test "init/find" {
620620}
621621
622622test "encode/decode literals" {
623 const LiteralEncoder = std.compress.flate.Compress.LiteralEncoder;
624
623 var codes: [flate.HuffmanEncoder.max_num_frequencies]flate.HuffmanEncoder.Code = undefined;
625624 for (1..286) |j| { // for all different number of codes
626 var enc: LiteralEncoder = .{};
625 var enc: flate.HuffmanEncoder = .{ .codes = &codes };
627626 // create frequencies
628627 var freq = [_]u16{0} ** 286;
629628 freq[256] = 1; // ensure we have end of block code
lib/std/compress/flate/HuffmanEncoder.zig created+475
......@@ -0,0 +1,475 @@
1const HuffmanEncoder = @This();
2const std = @import("std");
3const assert = std.debug.assert;
4const testing = std.testing;
5
6codes: []Code,
7// Reusable buffer with the longest possible frequency table.
8freq_cache: [max_num_frequencies + 1]LiteralNode,
9bit_count: [17]u32,
10lns: []LiteralNode, // sorted by literal, stored to avoid repeated allocation in generate
11lfs: []LiteralNode, // sorted by frequency, stored to avoid repeated allocation in generate
12
13pub const LiteralNode = struct {
14 literal: u16,
15 freq: u16,
16
17 pub fn max() LiteralNode {
18 return .{
19 .literal = std.math.maxInt(u16),
20 .freq = std.math.maxInt(u16),
21 };
22 }
23};
24
25pub const Code = struct {
26 code: u16 = 0,
27 len: u16 = 0,
28};
29
30/// The odd order in which the codegen code sizes are written.
31pub const codegen_order = [_]u32{ 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 };
32/// The number of codegen codes.
33pub const codegen_code_count = 19;
34
35/// The largest distance code.
36pub const distance_code_count = 30;
37
38/// Maximum number of literals.
39pub const max_num_lit = 286;
40
41/// Max number of frequencies used for a Huffman Code
42/// Possible lengths are codegen_code_count (19), distance_code_count (30) and max_num_lit (286).
43/// The largest of these is max_num_lit.
44pub const max_num_frequencies = max_num_lit;
45
46/// Biggest block size for uncompressed block.
47pub const max_store_block_size = 65535;
48/// The special code used to mark the end of a block.
49pub const end_block_marker = 256;
50
51/// Update this Huffman Code object to be the minimum code for the specified frequency count.
52///
53/// freq An array of frequencies, in which frequency[i] gives the frequency of literal i.
54/// max_bits The maximum number of bits to use for any literal.
55pub fn generate(self: *HuffmanEncoder, freq: []u16, max_bits: u32) void {
56 var list = self.freq_cache[0 .. freq.len + 1];
57 // Number of non-zero literals
58 var count: u32 = 0;
59 // Set list to be the set of all non-zero literals and their frequencies
60 for (freq, 0..) |f, i| {
61 if (f != 0) {
62 list[count] = LiteralNode{ .literal = @as(u16, @intCast(i)), .freq = f };
63 count += 1;
64 } else {
65 list[count] = LiteralNode{ .literal = 0x00, .freq = 0 };
66 self.codes[i].len = 0;
67 }
68 }
69 list[freq.len] = LiteralNode{ .literal = 0x00, .freq = 0 };
70
71 list = list[0..count];
72 if (count <= 2) {
73 // Handle the small cases here, because they are awkward for the general case code. With
74 // two or fewer literals, everything has bit length 1.
75 for (list, 0..) |node, i| {
76 // "list" is in order of increasing literal value.
77 self.codes[node.literal] = .{
78 .code = @intCast(i),
79 .len = 1,
80 };
81 }
82 return;
83 }
84 self.lfs = list;
85 std.mem.sort(LiteralNode, self.lfs, {}, byFreq);
86
87 // Get the number of literals for each bit count
88 const bit_count = self.bitCounts(list, max_bits);
89 // And do the assignment
90 self.assignEncodingAndSize(bit_count, list);
91}
92
93pub fn bitLength(self: *HuffmanEncoder, freq: []u16) u32 {
94 var total: u32 = 0;
95 for (freq, 0..) |f, i| {
96 if (f != 0) {
97 total += @as(u32, @intCast(f)) * @as(u32, @intCast(self.codes[i].len));
98 }
99 }
100 return total;
101}
102
103/// Return the number of literals assigned to each bit size in the Huffman encoding
104///
105/// This method is only called when list.len >= 3
106/// The cases of 0, 1, and 2 literals are handled by special case code.
107///
108/// list: An array of the literals with non-zero frequencies
109/// and their associated frequencies. The array is in order of increasing
110/// frequency, and has as its last element a special element with frequency
111/// `math.maxInt(i32)`
112///
113/// max_bits: The maximum number of bits that should be used to encode any literal.
114/// Must be less than 16.
115///
116/// Returns an integer array in which array[i] indicates the number of literals
117/// that should be encoded in i bits.
118fn bitCounts(self: *HuffmanEncoder, list: []LiteralNode, max_bits_to_use: usize) []u32 {
119 var max_bits = max_bits_to_use;
120 const n = list.len;
121 const max_bits_limit = 16;
122
123 assert(max_bits < max_bits_limit);
124
125 // The tree can't have greater depth than n - 1, no matter what. This
126 // saves a little bit of work in some small cases
127 max_bits = @min(max_bits, n - 1);
128
129 // Create information about each of the levels.
130 // A bogus "Level 0" whose sole purpose is so that
131 // level1.prev.needed == 0. This makes level1.next_pair_freq
132 // be a legitimate value that never gets chosen.
133 var levels: [max_bits_limit]LevelInfo = std.mem.zeroes([max_bits_limit]LevelInfo);
134 // leaf_counts[i] counts the number of literals at the left
135 // of ancestors of the rightmost node at level i.
136 // leaf_counts[i][j] is the number of literals at the left
137 // of the level j ancestor.
138 var leaf_counts: [max_bits_limit][max_bits_limit]u32 = @splat(0);
139
140 {
141 var level = @as(u32, 1);
142 while (level <= max_bits) : (level += 1) {
143 // For every level, the first two items are the first two characters.
144 // We initialize the levels as if we had already figured this out.
145 levels[level] = LevelInfo{
146 .level = level,
147 .last_freq = list[1].freq,
148 .next_char_freq = list[2].freq,
149 .next_pair_freq = list[0].freq + list[1].freq,
150 .needed = 0,
151 };
152 leaf_counts[level][level] = 2;
153 if (level == 1) {
154 levels[level].next_pair_freq = std.math.maxInt(i32);
155 }
156 }
157 }
158
159 // We need a total of 2*n - 2 items at top level and have already generated 2.
160 levels[max_bits].needed = 2 * @as(u32, @intCast(n)) - 4;
161
162 {
163 var level = max_bits;
164 while (true) {
165 var l = &levels[level];
166 if (l.next_pair_freq == std.math.maxInt(i32) and l.next_char_freq == std.math.maxInt(i32)) {
167 // We've run out of both leaves and pairs.
168 // End all calculations for this level.
169 // To make sure we never come back to this level or any lower level,
170 // set next_pair_freq impossibly large.
171 l.needed = 0;
172 levels[level + 1].next_pair_freq = std.math.maxInt(i32);
173 level += 1;
174 continue;
175 }
176
177 const prev_freq = l.last_freq;
178 if (l.next_char_freq < l.next_pair_freq) {
179 // The next item on this row is a leaf node.
180 const next = leaf_counts[level][level] + 1;
181 l.last_freq = l.next_char_freq;
182 // Lower leaf_counts are the same of the previous node.
183 leaf_counts[level][level] = next;
184 if (next >= list.len) {
185 l.next_char_freq = LiteralNode.max().freq;
186 } else {
187 l.next_char_freq = list[next].freq;
188 }
189 } else {
190 // The next item on this row is a pair from the previous row.
191 // next_pair_freq isn't valid until we generate two
192 // more values in the level below
193 l.last_freq = l.next_pair_freq;
194 // Take leaf counts from the lower level, except counts[level] remains the same.
195 @memcpy(leaf_counts[level][0..level], leaf_counts[level - 1][0..level]);
196 levels[l.level - 1].needed = 2;
197 }
198
199 l.needed -= 1;
200 if (l.needed == 0) {
201 // We've done everything we need to do for this level.
202 // Continue calculating one level up. Fill in next_pair_freq
203 // of that level with the sum of the two nodes we've just calculated on
204 // this level.
205 if (l.level == max_bits) {
206 // All done!
207 break;
208 }
209 levels[l.level + 1].next_pair_freq = prev_freq + l.last_freq;
210 level += 1;
211 } else {
212 // If we stole from below, move down temporarily to replenish it.
213 while (levels[level - 1].needed > 0) {
214 level -= 1;
215 if (level == 0) {
216 break;
217 }
218 }
219 }
220 }
221 }
222
223 // Somethings is wrong if at the end, the top level is null or hasn't used
224 // all of the leaves.
225 assert(leaf_counts[max_bits][max_bits] == n);
226
227 var bit_count = self.bit_count[0 .. max_bits + 1];
228 var bits: u32 = 1;
229 const counts = &leaf_counts[max_bits];
230 {
231 var level = max_bits;
232 while (level > 0) : (level -= 1) {
233 // counts[level] gives the number of literals requiring at least "bits"
234 // bits to encode.
235 bit_count[bits] = counts[level] - counts[level - 1];
236 bits += 1;
237 if (level == 0) {
238 break;
239 }
240 }
241 }
242 return bit_count;
243}
244
245/// Look at the leaves and assign them a bit count and an encoding as specified
246/// in RFC 1951 3.2.2
247fn assignEncodingAndSize(self: *HuffmanEncoder, bit_count: []u32, list_arg: []LiteralNode) void {
248 var code = @as(u16, 0);
249 var list = list_arg;
250
251 for (bit_count, 0..) |bits, n| {
252 code <<= 1;
253 if (n == 0 or bits == 0) {
254 continue;
255 }
256 // The literals list[list.len-bits] .. list[list.len-bits]
257 // are encoded using "bits" bits, and get the values
258 // code, code + 1, .... The code values are
259 // assigned in literal order (not frequency order).
260 const chunk = list[list.len - @as(u32, @intCast(bits)) ..];
261
262 self.lns = chunk;
263 std.mem.sort(LiteralNode, self.lns, {}, byLiteral);
264
265 for (chunk) |node| {
266 self.codes[node.literal] = .{
267 .code = bitReverse(u16, code, @as(u5, @intCast(n))),
268 .len = @as(u16, @intCast(n)),
269 };
270 code += 1;
271 }
272 list = list[0 .. list.len - @as(u32, @intCast(bits))];
273 }
274}
275
276fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
277 _ = context;
278 if (a.freq == b.freq) {
279 return a.literal < b.literal;
280 }
281 return a.freq < b.freq;
282}
283
284/// Describes the state of the constructed tree for a given depth.
285const LevelInfo = struct {
286 /// Our level. for better printing
287 level: u32,
288 /// The frequency of the last node at this level
289 last_freq: u32,
290 /// The frequency of the next character to add to this level
291 next_char_freq: u32,
292 /// The frequency of the next pair (from level below) to add to this level.
293 /// Only valid if the "needed" value of the next lower level is 0.
294 next_pair_freq: u32,
295 /// The number of chains remaining to generate for this level before moving
296 /// up to the next level
297 needed: u32,
298};
299
300fn byLiteral(context: void, a: LiteralNode, b: LiteralNode) bool {
301 _ = context;
302 return a.literal < b.literal;
303}
304
305/// Reverse bit-by-bit a N-bit code.
306fn bitReverse(comptime T: type, value: T, n: usize) T {
307 const r = @bitReverse(value);
308 return r >> @as(std.math.Log2Int(T), @intCast(@typeInfo(T).int.bits - n));
309}
310
311test bitReverse {
312 const ReverseBitsTest = struct {
313 in: u16,
314 bit_count: u5,
315 out: u16,
316 };
317
318 const reverse_bits_tests = [_]ReverseBitsTest{
319 .{ .in = 1, .bit_count = 1, .out = 1 },
320 .{ .in = 1, .bit_count = 2, .out = 2 },
321 .{ .in = 1, .bit_count = 3, .out = 4 },
322 .{ .in = 1, .bit_count = 4, .out = 8 },
323 .{ .in = 1, .bit_count = 5, .out = 16 },
324 .{ .in = 17, .bit_count = 5, .out = 17 },
325 .{ .in = 257, .bit_count = 9, .out = 257 },
326 .{ .in = 29, .bit_count = 5, .out = 23 },
327 };
328
329 for (reverse_bits_tests) |h| {
330 const v = bitReverse(u16, h.in, h.bit_count);
331 try std.testing.expectEqual(h.out, v);
332 }
333}
334
335/// Generates a HuffmanCode corresponding to the fixed literal table
336pub fn fixedLiteralEncoder(codes: *[max_num_frequencies]Code) HuffmanEncoder {
337 var h: HuffmanEncoder = undefined;
338 h.codes = codes;
339 var ch: u16 = 0;
340
341 while (ch < max_num_frequencies) : (ch += 1) {
342 var bits: u16 = undefined;
343 var size: u16 = undefined;
344 switch (ch) {
345 0...143 => {
346 // size 8, 000110000 .. 10111111
347 bits = ch + 48;
348 size = 8;
349 },
350 144...255 => {
351 // size 9, 110010000 .. 111111111
352 bits = ch + 400 - 144;
353 size = 9;
354 },
355 256...279 => {
356 // size 7, 0000000 .. 0010111
357 bits = ch - 256;
358 size = 7;
359 },
360 else => {
361 // size 8, 11000000 .. 11000111
362 bits = ch + 192 - 280;
363 size = 8;
364 },
365 }
366 h.codes[ch] = .{ .code = bitReverse(u16, bits, @as(u5, @intCast(size))), .len = size };
367 }
368 return h;
369}
370
371pub fn fixedDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder {
372 var h: HuffmanEncoder = undefined;
373 h.codes = codes;
374 for (h.codes, 0..) |_, ch| {
375 h.codes[ch] = .{ .code = bitReverse(u16, @as(u16, @intCast(ch)), 5), .len = 5 };
376 }
377 return h;
378}
379
380pub fn huffmanDistanceEncoder(codes: *[distance_code_count]Code) HuffmanEncoder {
381 var distance_freq: [distance_code_count]u16 = @splat(0);
382 distance_freq[0] = 1;
383 // huff_distance is a static distance encoder used for huffman only encoding.
384 // It can be reused since we will not be encoding distance values.
385 var h: HuffmanEncoder = .{};
386 h.codes = codes;
387 h.generate(distance_freq[0..], 15);
388 return h;
389}
390
391test "generate a Huffman code for the fixed literal table specific to Deflate" {
392 const enc = fixedLiteralEncoder();
393 for (enc.codes) |c| {
394 switch (c.len) {
395 7 => {
396 const v = @bitReverse(@as(u7, @intCast(c.code)));
397 try testing.expect(v <= 0b0010111);
398 },
399 8 => {
400 const v = @bitReverse(@as(u8, @intCast(c.code)));
401 try testing.expect((v >= 0b000110000 and v <= 0b10111111) or
402 (v >= 0b11000000 and v <= 11000111));
403 },
404 9 => {
405 const v = @bitReverse(@as(u9, @intCast(c.code)));
406 try testing.expect(v >= 0b110010000 and v <= 0b111111111);
407 },
408 else => unreachable,
409 }
410 }
411}
412
413test "generate a Huffman code for the 30 possible relative distances (LZ77 distances) of Deflate" {
414 var codes: [distance_code_count]Code = undefined;
415 const enc = fixedDistanceEncoder(&codes);
416 for (enc.codes) |c| {
417 const v = @bitReverse(@as(u5, @intCast(c.code)));
418 try testing.expect(v <= 29);
419 try testing.expect(c.len == 5);
420 }
421}
422
423test "fixedLiteralEncoder codes" {
424 var al = std.ArrayList(u8).init(testing.allocator);
425 defer al.deinit();
426 var bw = std.Io.bitWriter(.little, al.writer());
427
428 var codes: [max_num_frequencies]Code = undefined;
429 const f = fixedLiteralEncoder(&codes);
430 for (f.codes) |c| {
431 try bw.writeBits(c.code, c.len);
432 }
433 try testing.expectEqualSlices(u8, &fixed_codes, al.items);
434}
435
436pub const fixed_codes = [_]u8{
437 0b00001100, 0b10001100, 0b01001100, 0b11001100, 0b00101100, 0b10101100, 0b01101100, 0b11101100,
438 0b00011100, 0b10011100, 0b01011100, 0b11011100, 0b00111100, 0b10111100, 0b01111100, 0b11111100,
439 0b00000010, 0b10000010, 0b01000010, 0b11000010, 0b00100010, 0b10100010, 0b01100010, 0b11100010,
440 0b00010010, 0b10010010, 0b01010010, 0b11010010, 0b00110010, 0b10110010, 0b01110010, 0b11110010,
441 0b00001010, 0b10001010, 0b01001010, 0b11001010, 0b00101010, 0b10101010, 0b01101010, 0b11101010,
442 0b00011010, 0b10011010, 0b01011010, 0b11011010, 0b00111010, 0b10111010, 0b01111010, 0b11111010,
443 0b00000110, 0b10000110, 0b01000110, 0b11000110, 0b00100110, 0b10100110, 0b01100110, 0b11100110,
444 0b00010110, 0b10010110, 0b01010110, 0b11010110, 0b00110110, 0b10110110, 0b01110110, 0b11110110,
445 0b00001110, 0b10001110, 0b01001110, 0b11001110, 0b00101110, 0b10101110, 0b01101110, 0b11101110,
446 0b00011110, 0b10011110, 0b01011110, 0b11011110, 0b00111110, 0b10111110, 0b01111110, 0b11111110,
447 0b00000001, 0b10000001, 0b01000001, 0b11000001, 0b00100001, 0b10100001, 0b01100001, 0b11100001,
448 0b00010001, 0b10010001, 0b01010001, 0b11010001, 0b00110001, 0b10110001, 0b01110001, 0b11110001,
449 0b00001001, 0b10001001, 0b01001001, 0b11001001, 0b00101001, 0b10101001, 0b01101001, 0b11101001,
450 0b00011001, 0b10011001, 0b01011001, 0b11011001, 0b00111001, 0b10111001, 0b01111001, 0b11111001,
451 0b00000101, 0b10000101, 0b01000101, 0b11000101, 0b00100101, 0b10100101, 0b01100101, 0b11100101,
452 0b00010101, 0b10010101, 0b01010101, 0b11010101, 0b00110101, 0b10110101, 0b01110101, 0b11110101,
453 0b00001101, 0b10001101, 0b01001101, 0b11001101, 0b00101101, 0b10101101, 0b01101101, 0b11101101,
454 0b00011101, 0b10011101, 0b01011101, 0b11011101, 0b00111101, 0b10111101, 0b01111101, 0b11111101,
455 0b00010011, 0b00100110, 0b01001110, 0b10011010, 0b00111100, 0b01100101, 0b11101010, 0b10110100,
456 0b11101001, 0b00110011, 0b01100110, 0b11001110, 0b10011010, 0b00111101, 0b01100111, 0b11101110,
457 0b10111100, 0b11111001, 0b00001011, 0b00010110, 0b00101110, 0b01011010, 0b10111100, 0b01100100,
458 0b11101001, 0b10110010, 0b11100101, 0b00101011, 0b01010110, 0b10101110, 0b01011010, 0b10111101,
459 0b01100110, 0b11101101, 0b10111010, 0b11110101, 0b00011011, 0b00110110, 0b01101110, 0b11011010,
460 0b10111100, 0b01100101, 0b11101011, 0b10110110, 0b11101101, 0b00111011, 0b01110110, 0b11101110,
461 0b11011010, 0b10111101, 0b01100111, 0b11101111, 0b10111110, 0b11111101, 0b00000111, 0b00001110,
462 0b00011110, 0b00111010, 0b01111100, 0b11100100, 0b11101000, 0b10110001, 0b11100011, 0b00100111,
463 0b01001110, 0b10011110, 0b00111010, 0b01111101, 0b11100110, 0b11101100, 0b10111001, 0b11110011,
464 0b00010111, 0b00101110, 0b01011110, 0b10111010, 0b01111100, 0b11100101, 0b11101010, 0b10110101,
465 0b11101011, 0b00110111, 0b01101110, 0b11011110, 0b10111010, 0b01111101, 0b11100111, 0b11101110,
466 0b10111101, 0b11111011, 0b00001111, 0b00011110, 0b00111110, 0b01111010, 0b11111100, 0b11100100,
467 0b11101001, 0b10110011, 0b11100111, 0b00101111, 0b01011110, 0b10111110, 0b01111010, 0b11111101,
468 0b11100110, 0b11101101, 0b10111011, 0b11110111, 0b00011111, 0b00111110, 0b01111110, 0b11111010,
469 0b11111100, 0b11100101, 0b11101011, 0b10110111, 0b11101111, 0b00111111, 0b01111110, 0b11111110,
470 0b11111010, 0b11111101, 0b11100111, 0b11101111, 0b10111111, 0b11111111, 0b00000000, 0b00100000,
471 0b00001000, 0b00001100, 0b10000001, 0b11000010, 0b11100000, 0b00001000, 0b00100100, 0b00001010,
472 0b10001101, 0b11000001, 0b11100010, 0b11110000, 0b00000100, 0b00100010, 0b10001001, 0b01001100,
473 0b10100001, 0b11010010, 0b11101000, 0b00000011, 0b10000011, 0b01000011, 0b11000011, 0b00100011,
474 0b10100011,
475};
lib/std/compress/flate/testdata/block_writer.zig deleted-606
......@@ -1,606 +0,0 @@
1const Token = @import("../Token.zig");
2
3pub const TestCase = struct {
4 tokens: []const Token,
5 input: []const u8 = "", // File name of input data matching the tokens.
6 want: []const u8 = "", // File name of data with the expected output with input available.
7 want_no_input: []const u8 = "", // File name of the expected output when no input is available.
8};
9
10pub const testCases = blk: {
11 @setEvalBranchQuota(4096 * 2);
12
13 const L = Token.initLiteral;
14 const M = Token.initMatch;
15 const ml = M(1, 258); // Maximum length token. Used to reduce the size of writeBlockTests
16
17 break :blk &[_]TestCase{
18 TestCase{
19 .input = "huffman-null-max.input",
20 .want = "huffman-null-max.{s}.expect",
21 .want_no_input = "huffman-null-max.{s}.expect-noinput",
22 .tokens = &[_]Token{
23 L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
24 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
25 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
26 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
27 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
28 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
29 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
30 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
31 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
32 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
33 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
34 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
35 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, L(0x0), L(0x0),
36 },
37 },
38 TestCase{
39 .input = "huffman-pi.input",
40 .want = "huffman-pi.{s}.expect",
41 .want_no_input = "huffman-pi.{s}.expect-noinput",
42 .tokens = &[_]Token{
43 L('3'), L('.'), L('1'), L('4'), L('1'), L('5'), L('9'), L('2'),
44 L('6'), L('5'), L('3'), L('5'), L('8'), L('9'), L('7'), L('9'),
45 L('3'), L('2'), L('3'), L('8'), L('4'), L('6'), L('2'), L('6'),
46 L('4'), L('3'), L('3'), L('8'), L('3'), L('2'), L('7'), L('9'),
47 L('5'), L('0'), L('2'), L('8'), L('8'), L('4'), L('1'), L('9'),
48 L('7'), L('1'), L('6'), L('9'), L('3'), L('9'), L('9'), L('3'),
49 L('7'), L('5'), L('1'), L('0'), L('5'), L('8'), L('2'), L('0'),
50 L('9'), L('7'), L('4'), L('9'), L('4'), L('4'), L('5'), L('9'),
51 L('2'), L('3'), L('0'), L('7'), L('8'), L('1'), L('6'), L('4'),
52 L('0'), L('6'), L('2'), L('8'), L('6'), L('2'), L('0'), L('8'),
53 L('9'), L('9'), L('8'), L('6'), L('2'), L('8'), L('0'), L('3'),
54 L('4'), L('8'), L('2'), L('5'), L('3'), L('4'), L('2'), L('1'),
55 L('1'), L('7'), L('0'), L('6'), L('7'), L('9'), L('8'), L('2'),
56 L('1'), L('4'), L('8'), L('0'), L('8'), L('6'), L('5'), L('1'),
57 L('3'), L('2'), L('8'), L('2'), L('3'), L('0'), L('6'), L('6'),
58 L('4'), L('7'), L('0'), L('9'), L('3'), L('8'), L('4'), L('4'),
59 L('6'), L('0'), L('9'), L('5'), L('5'), L('0'), L('5'), L('8'),
60 L('2'), L('2'), L('3'), L('1'), L('7'), L('2'), L('5'), L('3'),
61 L('5'), L('9'), L('4'), L('0'), L('8'), L('1'), L('2'), L('8'),
62 L('4'), L('8'), L('1'), L('1'), L('1'), L('7'), L('4'), M(127, 4),
63 L('4'), L('1'), L('0'), L('2'), L('7'), L('0'), L('1'), L('9'),
64 L('3'), L('8'), L('5'), L('2'), L('1'), L('1'), L('0'), L('5'),
65 L('5'), L('5'), L('9'), L('6'), L('4'), L('4'), L('6'), L('2'),
66 L('2'), L('9'), L('4'), L('8'), L('9'), L('5'), L('4'), L('9'),
67 L('3'), L('0'), L('3'), L('8'), L('1'), M(19, 4), L('2'), L('8'),
68 L('8'), L('1'), L('0'), L('9'), L('7'), L('5'), L('6'), L('6'),
69 L('5'), L('9'), L('3'), L('3'), L('4'), L('4'), L('6'), M(72, 4),
70 L('7'), L('5'), L('6'), L('4'), L('8'), L('2'), L('3'), L('3'),
71 L('7'), L('8'), L('6'), L('7'), L('8'), L('3'), L('1'), L('6'),
72 L('5'), L('2'), L('7'), L('1'), L('2'), L('0'), L('1'), L('9'),
73 L('0'), L('9'), L('1'), L('4'), M(27, 4), L('5'), L('6'), L('6'),
74 L('9'), L('2'), L('3'), L('4'), L('6'), M(179, 4), L('6'), L('1'),
75 L('0'), L('4'), L('5'), L('4'), L('3'), L('2'), L('6'), M(51, 4),
76 L('1'), L('3'), L('3'), L('9'), L('3'), L('6'), L('0'), L('7'),
77 L('2'), L('6'), L('0'), L('2'), L('4'), L('9'), L('1'), L('4'),
78 L('1'), L('2'), L('7'), L('3'), L('7'), L('2'), L('4'), L('5'),
79 L('8'), L('7'), L('0'), L('0'), L('6'), L('6'), L('0'), L('6'),
80 L('3'), L('1'), L('5'), L('5'), L('8'), L('8'), L('1'), L('7'),
81 L('4'), L('8'), L('8'), L('1'), L('5'), L('2'), L('0'), L('9'),
82 L('2'), L('0'), L('9'), L('6'), L('2'), L('8'), L('2'), L('9'),
83 L('2'), L('5'), L('4'), L('0'), L('9'), L('1'), L('7'), L('1'),
84 L('5'), L('3'), L('6'), L('4'), L('3'), L('6'), L('7'), L('8'),
85 L('9'), L('2'), L('5'), L('9'), L('0'), L('3'), L('6'), L('0'),
86 L('0'), L('1'), L('1'), L('3'), L('3'), L('0'), L('5'), L('3'),
87 L('0'), L('5'), L('4'), L('8'), L('8'), L('2'), L('0'), L('4'),
88 L('6'), L('6'), L('5'), L('2'), L('1'), L('3'), L('8'), L('4'),
89 L('1'), L('4'), L('6'), L('9'), L('5'), L('1'), L('9'), L('4'),
90 L('1'), L('5'), L('1'), L('1'), L('6'), L('0'), L('9'), L('4'),
91 L('3'), L('3'), L('0'), L('5'), L('7'), L('2'), L('7'), L('0'),
92 L('3'), L('6'), L('5'), L('7'), L('5'), L('9'), L('5'), L('9'),
93 L('1'), L('9'), L('5'), L('3'), L('0'), L('9'), L('2'), L('1'),
94 L('8'), L('6'), L('1'), L('1'), L('7'), M(234, 4), L('3'), L('2'),
95 M(10, 4), L('9'), L('3'), L('1'), L('0'), L('5'), L('1'), L('1'),
96 L('8'), L('5'), L('4'), L('8'), L('0'), L('7'), M(271, 4), L('3'),
97 L('7'), L('9'), L('9'), L('6'), L('2'), L('7'), L('4'), L('9'),
98 L('5'), L('6'), L('7'), L('3'), L('5'), L('1'), L('8'), L('8'),
99 L('5'), L('7'), L('5'), L('2'), L('7'), L('2'), L('4'), L('8'),
100 L('9'), L('1'), L('2'), L('2'), L('7'), L('9'), L('3'), L('8'),
101 L('1'), L('8'), L('3'), L('0'), L('1'), L('1'), L('9'), L('4'),
102 L('9'), L('1'), L('2'), L('9'), L('8'), L('3'), L('3'), L('6'),
103 L('7'), L('3'), L('3'), L('6'), L('2'), L('4'), L('4'), L('0'),
104 L('6'), L('5'), L('6'), L('6'), L('4'), L('3'), L('0'), L('8'),
105 L('6'), L('0'), L('2'), L('1'), L('3'), L('9'), L('4'), L('9'),
106 L('4'), L('6'), L('3'), L('9'), L('5'), L('2'), L('2'), L('4'),
107 L('7'), L('3'), L('7'), L('1'), L('9'), L('0'), L('7'), L('0'),
108 L('2'), L('1'), L('7'), L('9'), L('8'), M(154, 5), L('7'), L('0'),
109 L('2'), L('7'), L('7'), L('0'), L('5'), L('3'), L('9'), L('2'),
110 L('1'), L('7'), L('1'), L('7'), L('6'), L('2'), L('9'), L('3'),
111 L('1'), L('7'), L('6'), L('7'), L('5'), M(563, 5), L('7'), L('4'),
112 L('8'), L('1'), M(7, 4), L('6'), L('6'), L('9'), L('4'), L('0'),
113 M(488, 4), L('0'), L('0'), L('0'), L('5'), L('6'), L('8'), L('1'),
114 L('2'), L('7'), L('1'), L('4'), L('5'), L('2'), L('6'), L('3'),
115 L('5'), L('6'), L('0'), L('8'), L('2'), L('7'), L('7'), L('8'),
116 L('5'), L('7'), L('7'), L('1'), L('3'), L('4'), L('2'), L('7'),
117 L('5'), L('7'), L('7'), L('8'), L('9'), L('6'), M(298, 4), L('3'),
118 L('6'), L('3'), L('7'), L('1'), L('7'), L('8'), L('7'), L('2'),
119 L('1'), L('4'), L('6'), L('8'), L('4'), L('4'), L('0'), L('9'),
120 L('0'), L('1'), L('2'), L('2'), L('4'), L('9'), L('5'), L('3'),
121 L('4'), L('3'), L('0'), L('1'), L('4'), L('6'), L('5'), L('4'),
122 L('9'), L('5'), L('8'), L('5'), L('3'), L('7'), L('1'), L('0'),
123 L('5'), L('0'), L('7'), L('9'), M(203, 4), L('6'), M(340, 4), L('8'),
124 L('9'), L('2'), L('3'), L('5'), L('4'), M(458, 4), L('9'), L('5'),
125 L('6'), L('1'), L('1'), L('2'), L('1'), L('2'), L('9'), L('0'),
126 L('2'), L('1'), L('9'), L('6'), L('0'), L('8'), L('6'), L('4'),
127 L('0'), L('3'), L('4'), L('4'), L('1'), L('8'), L('1'), L('5'),
128 L('9'), L('8'), L('1'), L('3'), L('6'), L('2'), L('9'), L('7'),
129 L('7'), L('4'), M(117, 4), L('0'), L('9'), L('9'), L('6'), L('0'),
130 L('5'), L('1'), L('8'), L('7'), L('0'), L('7'), L('2'), L('1'),
131 L('1'), L('3'), L('4'), L('9'), M(1, 5), L('8'), L('3'), L('7'),
132 L('2'), L('9'), L('7'), L('8'), L('0'), L('4'), L('9'), L('9'),
133 M(731, 4), L('9'), L('7'), L('3'), L('1'), L('7'), L('3'), L('2'),
134 L('8'), M(395, 4), L('6'), L('3'), L('1'), L('8'), L('5'), M(770, 4),
135 M(745, 4), L('4'), L('5'), L('5'), L('3'), L('4'), L('6'), L('9'),
136 L('0'), L('8'), L('3'), L('0'), L('2'), L('6'), L('4'), L('2'),
137 L('5'), L('2'), L('2'), L('3'), L('0'), M(740, 4), M(616, 4), L('8'),
138 L('5'), L('0'), L('3'), L('5'), L('2'), L('6'), L('1'), L('9'),
139 L('3'), L('1'), L('1'), M(531, 4), L('1'), L('0'), L('1'), L('0'),
140 L('0'), L('0'), L('3'), L('1'), L('3'), L('7'), L('8'), L('3'),
141 L('8'), L('7'), L('5'), L('2'), L('8'), L('8'), L('6'), L('5'),
142 L('8'), L('7'), L('5'), L('3'), L('3'), L('2'), L('0'), L('8'),
143 L('3'), L('8'), L('1'), L('4'), L('2'), L('0'), L('6'), M(321, 4),
144 M(300, 4), L('1'), L('4'), L('7'), L('3'), L('0'), L('3'), L('5'),
145 L('9'), M(815, 5), L('9'), L('0'), L('4'), L('2'), L('8'), L('7'),
146 L('5'), L('5'), L('4'), L('6'), L('8'), L('7'), L('3'), L('1'),
147 L('1'), L('5'), L('9'), L('5'), M(854, 4), L('3'), L('8'), L('8'),
148 L('2'), L('3'), L('5'), L('3'), L('7'), L('8'), L('7'), L('5'),
149 M(896, 5), L('9'), M(315, 4), L('1'), M(329, 4), L('8'), L('0'), L('5'),
150 L('3'), M(395, 4), L('2'), L('2'), L('6'), L('8'), L('0'), L('6'),
151 L('6'), L('1'), L('3'), L('0'), L('0'), L('1'), L('9'), L('2'),
152 L('7'), L('8'), L('7'), L('6'), L('6'), L('1'), L('1'), L('1'),
153 L('9'), L('5'), L('9'), M(568, 4), L('6'), M(293, 5), L('8'), L('9'),
154 L('3'), L('8'), L('0'), L('9'), L('5'), L('2'), L('5'), L('7'),
155 L('2'), L('0'), L('1'), L('0'), L('6'), L('5'), L('4'), L('8'),
156 L('5'), L('8'), L('6'), L('3'), L('2'), L('7'), M(155, 4), L('9'),
157 L('3'), L('6'), L('1'), L('5'), L('3'), M(545, 4), M(349, 5), L('2'),
158 L('3'), L('0'), L('3'), L('0'), L('1'), L('9'), L('5'), L('2'),
159 L('0'), L('3'), L('5'), L('3'), L('0'), L('1'), L('8'), L('5'),
160 L('2'), M(370, 4), M(118, 4), L('3'), L('6'), L('2'), L('2'), L('5'),
161 L('9'), L('9'), L('4'), L('1'), L('3'), M(597, 4), L('4'), L('9'),
162 L('7'), L('2'), L('1'), L('7'), M(223, 4), L('3'), L('4'), L('7'),
163 L('9'), L('1'), L('3'), L('1'), L('5'), L('1'), L('5'), L('5'),
164 L('7'), L('4'), L('8'), L('5'), L('7'), L('2'), L('4'), L('2'),
165 L('4'), L('5'), L('4'), L('1'), L('5'), L('0'), L('6'), L('9'),
166 M(320, 4), L('8'), L('2'), L('9'), L('5'), L('3'), L('3'), L('1'),
167 L('1'), L('6'), L('8'), L('6'), L('1'), L('7'), L('2'), L('7'),
168 L('8'), M(824, 4), L('9'), L('0'), L('7'), L('5'), L('0'), L('9'),
169 M(270, 4), L('7'), L('5'), L('4'), L('6'), L('3'), L('7'), L('4'),
170 L('6'), L('4'), L('9'), L('3'), L('9'), L('3'), L('1'), L('9'),
171 L('2'), L('5'), L('5'), L('0'), L('6'), L('0'), L('4'), L('0'),
172 L('0'), L('9'), M(620, 4), L('1'), L('6'), L('7'), L('1'), L('1'),
173 L('3'), L('9'), L('0'), L('0'), L('9'), L('8'), M(822, 4), L('4'),
174 L('0'), L('1'), L('2'), L('8'), L('5'), L('8'), L('3'), L('6'),
175 L('1'), L('6'), L('0'), L('3'), L('5'), L('6'), L('3'), L('7'),
176 L('0'), L('7'), L('6'), L('6'), L('0'), L('1'), L('0'), L('4'),
177 M(371, 4), L('8'), L('1'), L('9'), L('4'), L('2'), L('9'), M(1055, 5),
178 M(240, 4), M(652, 4), L('7'), L('8'), L('3'), L('7'), L('4'), M(1193, 4),
179 L('8'), L('2'), L('5'), L('5'), L('3'), L('7'), M(522, 5), L('2'),
180 L('6'), L('8'), M(47, 4), L('4'), L('0'), L('4'), L('7'), M(466, 4),
181 L('4'), M(1206, 4), M(910, 4), L('8'), L('4'), M(937, 4), L('6'), M(800, 6),
182 L('3'), L('3'), L('1'), L('3'), L('6'), L('7'), L('7'), L('0'),
183 L('2'), L('8'), L('9'), L('8'), L('9'), L('1'), L('5'), L('2'),
184 M(99, 4), L('5'), L('2'), L('1'), L('6'), L('2'), L('0'), L('5'),
185 L('6'), L('9'), L('6'), M(1042, 4), L('0'), L('5'), L('8'), M(1144, 4),
186 L('5'), M(1177, 4), L('5'), L('1'), L('1'), M(522, 4), L('8'), L('2'),
187 L('4'), L('3'), L('0'), L('0'), L('3'), L('5'), L('5'), L('8'),
188 L('7'), L('6'), L('4'), L('0'), L('2'), L('4'), L('7'), L('4'),
189 L('9'), L('6'), L('4'), L('7'), L('3'), L('2'), L('6'), L('3'),
190 M(1087, 4), L('9'), L('9'), L('2'), M(1100, 4), L('4'), L('2'), L('6'),
191 L('9'), M(710, 6), L('7'), M(471, 4), L('4'), M(1342, 4), M(1054, 4), L('9'),
192 L('3'), L('4'), L('1'), L('7'), M(430, 4), L('1'), L('2'), M(43, 4),
193 L('4'), M(415, 4), L('1'), L('5'), L('0'), L('3'), L('0'), L('2'),
194 L('8'), L('6'), L('1'), L('8'), L('2'), L('9'), L('7'), L('4'),
195 L('5'), L('5'), L('5'), L('7'), L('0'), L('6'), L('7'), L('4'),
196 M(310, 4), L('5'), L('0'), L('5'), L('4'), L('9'), L('4'), L('5'),
197 L('8'), M(454, 4), L('9'), M(82, 4), L('5'), L('6'), M(493, 4), L('7'),
198 L('2'), L('1'), L('0'), L('7'), L('9'), M(346, 4), L('3'), L('0'),
199 M(267, 4), L('3'), L('2'), L('1'), L('1'), L('6'), L('5'), L('3'),
200 L('4'), L('4'), L('9'), L('8'), L('7'), L('2'), L('0'), L('2'),
201 L('7'), M(284, 4), L('0'), L('2'), L('3'), L('6'), L('4'), M(559, 4),
202 L('5'), L('4'), L('9'), L('9'), L('1'), L('1'), L('9'), L('8'),
203 M(1049, 4), L('4'), M(284, 4), L('5'), L('3'), L('5'), L('6'), L('6'),
204 L('3'), L('6'), L('9'), M(1105, 4), L('2'), L('6'), L('5'), M(741, 4),
205 L('7'), L('8'), L('6'), L('2'), L('5'), L('5'), L('1'), M(987, 4),
206 L('1'), L('7'), L('5'), L('7'), L('4'), L('6'), L('7'), L('2'),
207 L('8'), L('9'), L('0'), L('9'), L('7'), L('7'), L('7'), L('7'),
208 M(1108, 5), L('0'), L('0'), L('0'), M(1534, 4), L('7'), L('0'), M(1248, 4),
209 L('6'), M(1002, 4), L('4'), L('9'), L('1'), M(1055, 4), M(664, 4), L('2'),
210 L('1'), L('4'), L('7'), L('7'), L('2'), L('3'), L('5'), L('0'),
211 L('1'), L('4'), L('1'), L('4'), M(1604, 4), L('3'), L('5'), L('6'),
212 M(1200, 4), L('1'), L('6'), L('1'), L('3'), L('6'), L('1'), L('1'),
213 L('5'), L('7'), L('3'), L('5'), L('2'), L('5'), M(1285, 4), L('3'),
214 L('4'), M(92, 4), L('1'), L('8'), M(1148, 4), L('8'), L('4'), M(1512, 4),
215 L('3'), L('3'), L('2'), L('3'), L('9'), L('0'), L('7'), L('3'),
216 L('9'), L('4'), L('1'), L('4'), L('3'), L('3'), L('3'), L('4'),
217 L('5'), L('4'), L('7'), L('7'), L('6'), L('2'), L('4'), M(579, 4),
218 L('2'), L('5'), L('1'), L('8'), L('9'), L('8'), L('3'), L('5'),
219 L('6'), L('9'), L('4'), L('8'), L('5'), L('5'), L('6'), L('2'),
220 L('0'), L('9'), L('9'), L('2'), L('1'), L('9'), L('2'), L('2'),
221 L('2'), L('1'), L('8'), L('4'), L('2'), L('7'), M(575, 4), L('2'),
222 M(187, 4), L('6'), L('8'), L('8'), L('7'), L('6'), L('7'), L('1'),
223 L('7'), L('9'), L('0'), M(86, 4), L('0'), M(263, 5), L('6'), L('6'),
224 M(1000, 4), L('8'), L('8'), L('6'), L('2'), L('7'), L('2'), M(1757, 4),
225 L('1'), L('7'), L('8'), L('6'), L('0'), L('8'), L('5'), L('7'),
226 M(116, 4), L('3'), M(765, 5), L('7'), L('9'), L('7'), L('6'), L('6'),
227 L('8'), L('1'), M(702, 4), L('0'), L('0'), L('9'), L('5'), L('3'),
228 L('8'), L('8'), M(1593, 4), L('3'), M(1702, 4), L('0'), L('6'), L('8'),
229 L('0'), L('0'), L('6'), L('4'), L('2'), L('2'), L('5'), L('1'),
230 L('2'), L('5'), L('2'), M(1404, 4), L('7'), L('3'), L('9'), L('2'),
231 M(664, 4), M(1141, 4), L('4'), M(1716, 5), L('8'), L('6'), L('2'), L('6'),
232 L('9'), L('4'), L('5'), M(486, 4), L('4'), L('1'), L('9'), L('6'),
233 L('5'), L('2'), L('8'), L('5'), L('0'), M(154, 4), M(925, 4), L('1'),
234 L('8'), L('6'), L('3'), M(447, 4), L('4'), M(341, 5), L('2'), L('0'),
235 L('3'), L('9'), M(1420, 4), L('4'), L('5'), M(701, 4), L('2'), L('3'),
236 L('7'), M(1069, 4), L('6'), M(1297, 4), L('5'), L('6'), M(1593, 4), L('7'),
237 L('1'), L('9'), L('1'), L('7'), L('2'), L('8'), M(370, 4), L('7'),
238 L('6'), L('4'), L('6'), L('5'), L('7'), L('5'), L('7'), L('3'),
239 L('9'), M(258, 4), L('3'), L('8'), L('9'), M(1865, 4), L('8'), L('3'),
240 L('2'), L('6'), L('4'), L('5'), L('9'), L('9'), L('5'), L('8'),
241 M(1704, 4), L('0'), L('4'), L('7'), L('8'), M(479, 4), M(809, 4), L('9'),
242 M(46, 4), L('6'), L('4'), L('0'), L('7'), L('8'), L('9'), L('5'),
243 L('1'), M(143, 4), L('6'), L('8'), L('3'), M(304, 4), L('2'), L('5'),
244 L('9'), L('5'), L('7'), L('0'), M(1129, 4), L('8'), L('2'), L('2'),
245 M(713, 4), L('2'), M(1564, 4), L('4'), L('0'), L('7'), L('7'), L('2'),
246 L('6'), L('7'), L('1'), L('9'), L('4'), L('7'), L('8'), M(794, 4),
247 L('8'), L('2'), L('6'), L('0'), L('1'), L('4'), L('7'), L('6'),
248 L('9'), L('9'), L('0'), L('9'), M(1257, 4), L('0'), L('1'), L('3'),
249 L('6'), L('3'), L('9'), L('4'), L('4'), L('3'), M(640, 4), L('3'),
250 L('0'), M(262, 4), L('2'), L('0'), L('3'), L('4'), L('9'), L('6'),
251 L('2'), L('5'), L('2'), L('4'), L('5'), L('1'), L('7'), M(950, 4),
252 L('9'), L('6'), L('5'), L('1'), L('4'), L('3'), L('1'), L('4'),
253 L('2'), L('9'), L('8'), L('0'), L('9'), L('1'), L('9'), L('0'),
254 L('6'), L('5'), L('9'), L('2'), M(643, 4), L('7'), L('2'), L('2'),
255 L('1'), L('6'), L('9'), L('6'), L('4'), L('6'), M(1050, 4), M(123, 4),
256 L('5'), M(1295, 4), L('4'), M(1382, 5), L('8'), M(1370, 4), L('9'), L('7'),
257 M(1404, 4), L('5'), L('4'), M(1182, 4), M(575, 4), L('7'), M(1627, 4), L('8'),
258 L('4'), L('6'), L('8'), L('1'), L('3'), M(141, 4), L('6'), L('8'),
259 L('3'), L('8'), L('6'), L('8'), L('9'), L('4'), L('2'), L('7'),
260 L('7'), L('4'), L('1'), L('5'), L('5'), L('9'), L('9'), L('1'),
261 L('8'), L('5'), M(91, 4), L('2'), L('4'), L('5'), L('9'), L('5'),
262 L('3'), L('9'), L('5'), L('9'), L('4'), L('3'), L('1'), M(1464, 4),
263 L('7'), M(19, 4), L('6'), L('8'), L('0'), L('8'), L('4'), L('5'),
264 M(744, 4), L('7'), L('3'), M(2079, 4), L('9'), L('5'), L('8'), L('4'),
265 L('8'), L('6'), L('5'), L('3'), L('8'), M(1769, 4), L('6'), L('2'),
266 M(243, 4), L('6'), L('0'), L('9'), M(1207, 4), L('6'), L('0'), L('8'),
267 L('0'), L('5'), L('1'), L('2'), L('4'), L('3'), L('8'), L('8'),
268 L('4'), M(315, 4), M(12, 4), L('4'), L('1'), L('3'), M(784, 4), L('7'),
269 L('6'), L('2'), L('7'), L('8'), M(834, 4), L('7'), L('1'), L('5'),
270 M(1436, 4), L('3'), L('5'), L('9'), L('9'), L('7'), L('7'), L('0'),
271 L('0'), L('1'), L('2'), L('9'), M(1139, 4), L('8'), L('9'), L('4'),
272 L('4'), L('1'), M(632, 4), L('6'), L('8'), L('5'), L('5'), M(96, 4),
273 L('4'), L('0'), L('6'), L('3'), M(2279, 4), L('2'), L('0'), L('7'),
274 L('2'), L('2'), M(345, 4), M(516, 5), L('4'), L('8'), L('1'), L('5'),
275 L('8'), M(518, 4), M(511, 4), M(635, 4), M(665, 4), L('3'), L('9'), L('4'),
276 L('5'), L('2'), L('2'), L('6'), L('7'), M(1175, 6), L('8'), M(1419, 4),
277 L('2'), L('1'), M(747, 4), L('2'), M(904, 4), L('5'), L('4'), L('6'),
278 L('6'), L('6'), M(1308, 4), L('2'), L('3'), L('9'), L('8'), L('6'),
279 L('4'), L('5'), L('6'), M(1221, 4), L('1'), L('6'), L('3'), L('5'),
280 M(596, 5), M(2066, 4), L('7'), M(2222, 4), L('9'), L('8'), M(1119, 4), L('9'),
281 L('3'), L('6'), L('3'), L('4'), M(1884, 4), L('7'), L('4'), L('3'),
282 L('2'), L('4'), M(1148, 4), L('1'), L('5'), L('0'), L('7'), L('6'),
283 M(1212, 4), L('7'), L('9'), L('4'), L('5'), L('1'), L('0'), L('9'),
284 M(63, 4), L('0'), L('9'), L('4'), L('0'), M(1703, 4), L('8'), L('8'),
285 L('7'), L('9'), L('7'), L('1'), L('0'), L('8'), L('9'), L('3'),
286 M(2289, 4), L('6'), L('9'), L('1'), L('3'), L('6'), L('8'), L('6'),
287 L('7'), L('2'), M(604, 4), M(511, 4), L('5'), M(1344, 4), M(1129, 4), M(2050, 4),
288 L('1'), L('7'), L('9'), L('2'), L('8'), L('6'), L('8'), M(2253, 4),
289 L('8'), L('7'), L('4'), L('7'), M(1951, 5), L('8'), L('2'), L('4'),
290 M(2427, 4), L('8'), M(604, 4), L('7'), L('1'), L('4'), L('9'), L('0'),
291 L('9'), L('6'), L('7'), L('5'), L('9'), L('8'), M(1776, 4), L('3'),
292 L('6'), L('5'), M(309, 4), L('8'), L('1'), M(93, 4), M(1862, 4), M(2359, 4),
293 L('6'), L('8'), L('2'), L('9'), M(1407, 4), L('8'), L('7'), L('2'),
294 L('2'), L('6'), L('5'), L('8'), L('8'), L('0'), M(1554, 4), L('5'),
295 M(586, 4), L('4'), L('2'), L('7'), L('0'), L('4'), L('7'), L('7'),
296 L('5'), L('5'), M(2079, 4), L('3'), L('7'), L('9'), L('6'), L('4'),
297 L('1'), L('4'), L('5'), L('1'), L('5'), L('2'), M(1534, 4), L('2'),
298 L('3'), L('4'), L('3'), L('6'), L('4'), L('5'), L('4'), M(1503, 4),
299 L('4'), L('4'), L('4'), L('7'), L('9'), L('5'), M(61, 4), M(1316, 4),
300 M(2279, 5), L('4'), L('1'), M(1323, 4), L('3'), M(773, 4), L('5'), L('2'),
301 L('3'), L('1'), M(2114, 5), L('1'), L('6'), L('6'), L('1'), M(2227, 4),
302 L('5'), L('9'), L('6'), L('9'), L('5'), L('3'), L('6'), L('2'),
303 L('3'), L('1'), L('4'), M(1536, 4), L('2'), L('4'), L('8'), L('4'),
304 L('9'), L('3'), L('7'), L('1'), L('8'), L('7'), L('1'), L('1'),
305 L('0'), L('1'), L('4'), L('5'), L('7'), L('6'), L('5'), L('4'),
306 M(1890, 4), L('0'), L('2'), L('7'), L('9'), L('9'), L('3'), L('4'),
307 L('4'), L('0'), L('3'), L('7'), L('4'), L('2'), L('0'), L('0'),
308 L('7'), M(2368, 4), L('7'), L('8'), L('5'), L('3'), L('9'), L('0'),
309 L('6'), L('2'), L('1'), L('9'), M(666, 5), M(838, 4), L('8'), L('4'),
310 L('7'), M(979, 5), L('8'), L('3'), L('3'), L('2'), L('1'), L('4'),
311 L('4'), L('5'), L('7'), L('1'), M(645, 4), M(1911, 4), L('4'), L('3'),
312 L('5'), L('0'), M(2345, 4), M(1129, 4), L('5'), L('3'), L('1'), L('9'),
313 L('1'), L('0'), L('4'), L('8'), L('4'), L('8'), L('1'), L('0'),
314 L('0'), L('5'), L('3'), L('7'), L('0'), L('6'), M(2237, 4), M(1438, 5),
315 M(1922, 5), L('1'), M(1370, 4), L('7'), M(796, 4), L('5'), M(2029, 4), M(1037, 4),
316 L('6'), L('3'), M(2013, 5), L('4'), M(2418, 4), M(847, 5), M(1014, 5), L('8'),
317 M(1326, 5), M(2184, 5), L('9'), M(392, 4), L('9'), L('1'), M(2255, 4), L('8'),
318 L('1'), L('4'), L('6'), L('7'), L('5'), L('1'), M(1580, 4), L('1'),
319 L('2'), L('3'), L('9'), M(426, 6), L('9'), L('0'), L('7'), L('1'),
320 L('8'), L('6'), L('4'), L('9'), L('4'), L('2'), L('3'), L('1'),
321 L('9'), L('6'), L('1'), L('5'), L('6'), M(493, 4), M(1725, 4), L('9'),
322 L('5'), M(2343, 4), M(1130, 4), M(284, 4), L('6'), L('0'), L('3'), L('8'),
323 M(2598, 4), M(368, 4), M(901, 4), L('6'), L('2'), M(1115, 4), L('5'), M(2125, 4),
324 L('6'), L('3'), L('8'), L('9'), L('3'), L('7'), L('7'), L('8'),
325 L('7'), M(2246, 4), M(249, 4), L('9'), L('7'), L('9'), L('2'), L('0'),
326 L('7'), L('7'), L('3'), M(1496, 4), L('2'), L('1'), L('8'), L('2'),
327 L('5'), L('6'), M(2016, 4), L('6'), L('6'), M(1751, 4), L('4'), L('2'),
328 M(1663, 5), L('6'), M(1767, 4), L('4'), L('4'), M(37, 4), L('5'), L('4'),
329 L('9'), L('2'), L('0'), L('2'), L('6'), L('0'), L('5'), M(2740, 4),
330 M(997, 5), L('2'), L('0'), L('1'), L('4'), L('9'), M(1235, 4), L('8'),
331 L('5'), L('0'), L('7'), L('3'), M(1434, 4), L('6'), L('6'), L('6'),
332 L('0'), M(405, 4), L('2'), L('4'), L('3'), L('4'), L('0'), M(136, 4),
333 L('0'), M(1900, 4), L('8'), L('6'), L('3'), M(2391, 4), M(2021, 4), M(1068, 4),
334 M(373, 4), L('5'), L('7'), L('9'), L('6'), L('2'), L('6'), L('8'),
335 L('5'), L('6'), M(321, 4), L('5'), L('0'), L('8'), M(1316, 4), L('5'),
336 L('8'), L('7'), L('9'), L('6'), L('9'), L('9'), M(1810, 4), L('5'),
337 L('7'), L('4'), M(2585, 4), L('8'), L('4'), L('0'), M(2228, 4), L('1'),
338 L('4'), L('5'), L('9'), L('1'), M(1933, 4), L('7'), L('0'), M(565, 4),
339 L('0'), L('1'), M(3048, 4), L('1'), L('2'), M(3189, 4), L('0'), M(964, 4),
340 L('3'), L('9'), M(2859, 4), M(275, 4), L('7'), L('1'), L('5'), M(945, 4),
341 L('4'), L('2'), L('0'), M(3059, 5), L('9'), M(3011, 4), L('0'), L('7'),
342 M(834, 4), M(1942, 4), M(2736, 4), M(3171, 4), L('2'), L('1'), M(2401, 4), L('2'),
343 L('5'), L('1'), M(1404, 4), M(2373, 4), L('9'), L('2'), M(435, 4), L('8'),
344 L('2'), L('6'), M(2919, 4), L('2'), M(633, 4), L('3'), L('2'), L('1'),
345 L('5'), L('7'), L('9'), L('1'), L('9'), L('8'), L('4'), L('1'),
346 L('4'), M(2172, 5), L('9'), L('1'), L('6'), L('4'), M(1769, 5), L('9'),
347 M(2905, 5), M(2268, 4), L('7'), L('2'), L('2'), M(802, 4), L('5'), M(2213, 4),
348 M(322, 4), L('9'), L('1'), L('0'), M(189, 4), M(3164, 4), L('5'), L('2'),
349 L('8'), L('0'), L('1'), L('7'), M(562, 4), L('7'), L('1'), L('2'),
350 M(2325, 4), L('8'), L('3'), L('2'), M(884, 4), L('1'), M(1418, 4), L('0'),
351 L('9'), L('3'), L('5'), L('3'), L('9'), L('6'), L('5'), L('7'),
352 M(1612, 4), L('1'), L('0'), L('8'), L('3'), M(106, 4), L('5'), L('1'),
353 M(1915, 4), M(3419, 4), L('1'), L('4'), L('4'), L('4'), L('2'), L('1'),
354 L('0'), L('0'), M(515, 4), L('0'), L('3'), M(413, 4), L('1'), L('1'),
355 L('0'), L('3'), M(3202, 4), M(10, 4), M(39, 4), M(1539, 6), L('5'), L('1'),
356 L('6'), M(1498, 4), M(2180, 5), M(2347, 4), L('5'), M(3139, 5), L('8'), L('5'),
357 L('1'), L('7'), L('1'), L('4'), L('3'), L('7'), M(1542, 4), M(110, 4),
358 L('1'), L('5'), L('5'), L('6'), L('5'), L('0'), L('8'), L('8'),
359 M(954, 4), L('9'), L('8'), L('9'), L('8'), L('5'), L('9'), L('9'),
360 L('8'), L('2'), L('3'), L('8'), M(464, 4), M(2491, 4), L('3'), M(365, 4),
361 M(1087, 4), M(2500, 4), L('8'), M(3590, 5), L('3'), L('2'), M(264, 4), L('5'),
362 M(774, 4), L('3'), M(459, 4), L('9'), M(1052, 4), L('9'), L('8'), M(2174, 4),
363 L('4'), M(3257, 4), L('7'), M(1612, 4), L('0'), L('7'), M(230, 4), L('4'),
364 L('8'), L('1'), L('4'), L('1'), M(1338, 4), L('8'), L('5'), L('9'),
365 L('4'), L('6'), L('1'), M(3018, 4), L('8'), L('0'),
366 },
367 },
368 TestCase{
369 .input = "huffman-rand-1k.input",
370 .want = "huffman-rand-1k.{s}.expect",
371 .want_no_input = "huffman-rand-1k.{s}.expect-noinput",
372 .tokens = &[_]Token{
373 L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xd), L(0x85), L(0x94), L(0x25), L(0x80), L(0xaf), L(0xc2), L(0xfe), L(0x8d),
374 L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde), L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b),
375 L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73), L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4),
376 L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15), L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde),
377 L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9), L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14),
378 L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xd), L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c),
379 L(0x99), L(0xdf), L(0xfd), L(0x70), L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5),
380 L(0xd4), L(0x80), L(0x59), L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20),
381 L(0x1b), L(0x46), L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4),
382 L(0x7b), L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
383 L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f), L(0xa0),
384 L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4), L(0x75), L(0xda),
385 L(0xea), L(0x9b), L(0xa), L(0x64), L(0xb), L(0xe0), L(0x23), L(0x29), L(0xbd), L(0xf7), L(0xe7), L(0x83), L(0x3c), L(0xfb),
386 L(0xdf), L(0xb3), L(0xae), L(0x4f), L(0xa4), L(0x47), L(0x55), L(0x99), L(0xde), L(0x2f), L(0x96), L(0x6e), L(0x1c), L(0x43),
387 L(0x4c), L(0x87), L(0xe2), L(0x7c), L(0xd9), L(0x5f), L(0x4c), L(0x7c), L(0xe8), L(0x90), L(0x3), L(0xdb), L(0x30), L(0x95),
388 L(0xd6), L(0x22), L(0xc), L(0x47), L(0xb8), L(0x4d), L(0x6b), L(0xbd), L(0x24), L(0x11), L(0xab), L(0x2c), L(0xd7), L(0xbe),
389 L(0x6e), L(0x7a), L(0xd6), L(0x8), L(0xa3), L(0x98), L(0xd8), L(0xdd), L(0x15), L(0x6a), L(0xfa), L(0x93), L(0x30), L(0x1),
390 L(0x25), L(0x1d), L(0xa2), L(0x74), L(0x86), L(0x4b), L(0x6a), L(0x95), L(0xe8), L(0xe1), L(0x4e), L(0xe), L(0x76), L(0xb9),
391 L(0x49), L(0xa9), L(0x5f), L(0xa0), L(0xa6), L(0x63), L(0x3c), L(0x7e), L(0x7e), L(0x20), L(0x13), L(0x4f), L(0xbb), L(0x66),
392 L(0x92), L(0xb8), L(0x2e), L(0xa4), L(0xfa), L(0x48), L(0xcb), L(0xae), L(0xb9), L(0x3c), L(0xaf), L(0xd3), L(0x1f), L(0xe1),
393 L(0xd5), L(0x8d), L(0x42), L(0x6d), L(0xf0), L(0xfc), L(0x8c), L(0xc), L(0x0), L(0xde), L(0x40), L(0xab), L(0x8b), L(0x47),
394 L(0x97), L(0x4e), L(0xa8), L(0xcf), L(0x8e), L(0xdb), L(0xa6), L(0x8b), L(0x20), L(0x9), L(0x84), L(0x7a), L(0x66), L(0xe5),
395 L(0x98), L(0x29), L(0x2), L(0x95), L(0xe6), L(0x38), L(0x32), L(0x60), L(0x3), L(0xe3), L(0x9a), L(0x1e), L(0x54), L(0xe8),
396 L(0x63), L(0x80), L(0x48), L(0x9c), L(0xe7), L(0x63), L(0x33), L(0x6e), L(0xa0), L(0x65), L(0x83), L(0xfa), L(0xc6), L(0xba),
397 L(0x7a), L(0x43), L(0x71), L(0x5), L(0xf5), L(0x68), L(0x69), L(0x85), L(0x9c), L(0xba), L(0x45), L(0xcd), L(0x6b), L(0xb),
398 L(0x19), L(0xd1), L(0xbb), L(0x7f), L(0x70), L(0x85), L(0x92), L(0xd1), L(0xb4), L(0x64), L(0x82), L(0xb1), L(0xe4), L(0x62),
399 L(0xc5), L(0x3c), L(0x46), L(0x1f), L(0x92), L(0x31), L(0x1c), L(0x4e), L(0x41), L(0x77), L(0xf7), L(0xe7), L(0x87), L(0xa2),
400 L(0xf), L(0x6e), L(0xe8), L(0x92), L(0x3), L(0x6b), L(0xa), L(0xe7), L(0xa9), L(0x3b), L(0x11), L(0xda), L(0x66), L(0x8a),
401 L(0x29), L(0xda), L(0x79), L(0xe1), L(0x64), L(0x8d), L(0xe3), L(0x54), L(0xd4), L(0xf5), L(0xef), L(0x64), L(0x87), L(0x3b),
402 L(0xf4), L(0xc2), L(0xf4), L(0x71), L(0x13), L(0xa9), L(0xe9), L(0xe0), L(0xa2), L(0x6), L(0x14), L(0xab), L(0x5d), L(0xa7),
403 L(0x96), L(0x0), L(0xd6), L(0xc3), L(0xcc), L(0x57), L(0xed), L(0x39), L(0x6a), L(0x25), L(0xcd), L(0x76), L(0xea), L(0xba),
404 L(0x3a), L(0xf2), L(0xa1), L(0x95), L(0x5d), L(0xe5), L(0x71), L(0xcf), L(0x9c), L(0x62), L(0x9e), L(0x6a), L(0xfa), L(0xd5),
405 L(0x31), L(0xd1), L(0xa8), L(0x66), L(0x30), L(0x33), L(0xaa), L(0x51), L(0x17), L(0x13), L(0x82), L(0x99), L(0xc8), L(0x14),
406 L(0x60), L(0x9f), L(0x4d), L(0x32), L(0x6d), L(0xda), L(0x19), L(0x26), L(0x21), L(0xdc), L(0x7e), L(0x2e), L(0x25), L(0x67),
407 L(0x72), L(0xca), L(0xf), L(0x92), L(0xcd), L(0xf6), L(0xd6), L(0xcb), L(0x97), L(0x8a), L(0x33), L(0x58), L(0x73), L(0x70),
408 L(0x91), L(0x1d), L(0xbf), L(0x28), L(0x23), L(0xa3), L(0xc), L(0xf1), L(0x83), L(0xc3), L(0xc8), L(0x56), L(0x77), L(0x68),
409 L(0xe3), L(0x82), L(0xba), L(0xb9), L(0x57), L(0x56), L(0x57), L(0x9c), L(0xc3), L(0xd6), L(0x14), L(0x5), L(0x3c), L(0xb1),
410 L(0xaf), L(0x93), L(0xc8), L(0x8a), L(0x57), L(0x7f), L(0x53), L(0xfa), L(0x2f), L(0xaa), L(0x6e), L(0x66), L(0x83), L(0xfa),
411 L(0x33), L(0xd1), L(0x21), L(0xab), L(0x1b), L(0x71), L(0xb4), L(0x7c), L(0xda), L(0xfd), L(0xfb), L(0x7f), L(0x20), L(0xab),
412 L(0x5e), L(0xd5), L(0xca), L(0xfd), L(0xdd), L(0xe0), L(0xee), L(0xda), L(0xba), L(0xa8), L(0x27), L(0x99), L(0x97), L(0x69),
413 L(0xc1), L(0x3c), L(0x82), L(0x8c), L(0xa), L(0x5c), L(0x2d), L(0x5b), L(0x88), L(0x3e), L(0x34), L(0x35), L(0x86), L(0x37),
414 L(0x46), L(0x79), L(0xe1), L(0xaa), L(0x19), L(0xfb), L(0xaa), L(0xde), L(0x15), L(0x9), L(0xd), L(0x1a), L(0x57), L(0xff),
415 L(0xb5), L(0xf), L(0xf3), L(0x2b), L(0x5a), L(0x6a), L(0x4d), L(0x19), L(0x77), L(0x71), L(0x45), L(0xdf), L(0x4f), L(0xb3),
416 L(0xec), L(0xf1), L(0xeb), L(0x18), L(0x53), L(0x3e), L(0x3b), L(0x47), L(0x8), L(0x9a), L(0x73), L(0xa0), L(0x5c), L(0x8c),
417 L(0x5f), L(0xeb), L(0xf), L(0x3a), L(0xc2), L(0x43), L(0x67), L(0xb4), L(0x66), L(0x67), L(0x80), L(0x58), L(0xe), L(0xc1),
418 L(0xec), L(0x40), L(0xd4), L(0x22), L(0x94), L(0xca), L(0xf9), L(0xe8), L(0x92), L(0xe4), L(0x69), L(0x38), L(0xbe), L(0x67),
419 L(0x64), L(0xca), L(0x50), L(0xc7), L(0x6), L(0x67), L(0x42), L(0x6e), L(0xa3), L(0xf0), L(0xb7), L(0x6c), L(0xf2), L(0xe8),
420 L(0x5f), L(0xb1), L(0xaf), L(0xe7), L(0xdb), L(0xbb), L(0x77), L(0xb5), L(0xf8), L(0xcb), L(0x8), L(0xc4), L(0x75), L(0x7e),
421 L(0xc0), L(0xf9), L(0x1c), L(0x7f), L(0x3c), L(0x89), L(0x2f), L(0xd2), L(0x58), L(0x3a), L(0xe2), L(0xf8), L(0x91), L(0xb6),
422 L(0x7b), L(0x24), L(0x27), L(0xe9), L(0xae), L(0x84), L(0x8b), L(0xde), L(0x74), L(0xac), L(0xfd), L(0xd9), L(0xb7), L(0x69),
423 L(0x2a), L(0xec), L(0x32), L(0x6f), L(0xf0), L(0x92), L(0x84), L(0xf1), L(0x40), L(0xc), L(0x8a), L(0xbc), L(0x39), L(0x6e),
424 L(0x2e), L(0x73), L(0xd4), L(0x6e), L(0x8a), L(0x74), L(0x2a), L(0xdc), L(0x60), L(0x1f), L(0xa3), L(0x7), L(0xde), L(0x75),
425 L(0x8b), L(0x74), L(0xc8), L(0xfe), L(0x63), L(0x75), L(0xf6), L(0x3d), L(0x63), L(0xac), L(0x33), L(0x89), L(0xc3), L(0xf0),
426 L(0xf8), L(0x2d), L(0x6b), L(0xb4), L(0x9e), L(0x74), L(0x8b), L(0x5c), L(0x33), L(0xb4), L(0xca), L(0xa8), L(0xe4), L(0x99),
427 L(0xb6), L(0x90), L(0xa1), L(0xef), L(0xf), L(0xd3), L(0x61), L(0xb2), L(0xc6), L(0x1a), L(0x94), L(0x7c), L(0x44), L(0x55),
428 L(0xf4), L(0x45), L(0xff), L(0x9e), L(0xa5), L(0x5a), L(0xc6), L(0xa0), L(0xe8), L(0x2a), L(0xc1), L(0x8d), L(0x6f), L(0x34),
429 L(0x11), L(0xb9), L(0xbe), L(0x4e), L(0xd9), L(0x87), L(0x97), L(0x73), L(0xcf), L(0x3d), L(0x23), L(0xae), L(0xd5), L(0x1a),
430 L(0x5e), L(0xae), L(0x5d), L(0x6a), L(0x3), L(0xf9), L(0x22), L(0xd), L(0x10), L(0xd9), L(0x47), L(0x69), L(0x15), L(0x3f),
431 L(0xee), L(0x52), L(0xa3), L(0x8), L(0xd2), L(0x3c), L(0x51), L(0xf4), L(0xf8), L(0x9d), L(0xe4), L(0x98), L(0x89), L(0xc8),
432 L(0x67), L(0x39), L(0xd5), L(0x5e), L(0x35), L(0x78), L(0x27), L(0xe8), L(0x3c), L(0x80), L(0xae), L(0x79), L(0x71), L(0xd2),
433 L(0x93), L(0xf4), L(0xaa), L(0x51), L(0x12), L(0x1c), L(0x4b), L(0x1b), L(0xe5), L(0x6e), L(0x15), L(0x6f), L(0xe4), L(0xbb),
434 L(0x51), L(0x9b), L(0x45), L(0x9f), L(0xf9), L(0xc4), L(0x8c), L(0x2a), L(0xfb), L(0x1a), L(0xdf), L(0x55), L(0xd3), L(0x48),
435 L(0x93), L(0x27), L(0x1), L(0x26), L(0xc2), L(0x6b), L(0x55), L(0x6d), L(0xa2), L(0xfb), L(0x84), L(0x8b), L(0xc9), L(0x9e),
436 L(0x28), L(0xc2), L(0xef), L(0x1a), L(0x24), L(0xec), L(0x9b), L(0xae), L(0xbd), L(0x60), L(0xe9), L(0x15), L(0x35), L(0xee),
437 L(0x42), L(0xa4), L(0x33), L(0x5b), L(0xfa), L(0xf), L(0xb6), L(0xf7), L(0x1), L(0xa6), L(0x2), L(0x4c), L(0xca), L(0x90),
438 L(0x58), L(0x3a), L(0x96), L(0x41), L(0xe7), L(0xcb), L(0x9), L(0x8c), L(0xdb), L(0x85), L(0x4d), L(0xa8), L(0x89), L(0xf3),
439 L(0xb5), L(0x8e), L(0xfd), L(0x75), L(0x5b), L(0x4f), L(0xed), L(0xde), L(0x3f), L(0xeb), L(0x38), L(0xa3), L(0xbe), L(0xb0),
440 L(0x73), L(0xfc), L(0xb8), L(0x54), L(0xf7), L(0x4c), L(0x30), L(0x67), L(0x2e), L(0x38), L(0xa2), L(0x54), L(0x18), L(0xba),
441 L(0x8), L(0xbf), L(0xf2), L(0x39), L(0xd5), L(0xfe), L(0xa5), L(0x41), L(0xc6), L(0x66), L(0x66), L(0xba), L(0x81), L(0xef),
442 L(0x67), L(0xe4), L(0xe6), L(0x3c), L(0xc), L(0xca), L(0xa4), L(0xa), L(0x79), L(0xb3), L(0x57), L(0x8b), L(0x8a), L(0x75),
443 L(0x98), L(0x18), L(0x42), L(0x2f), L(0x29), L(0xa3), L(0x82), L(0xef), L(0x9f), L(0x86), L(0x6), L(0x23), L(0xe1), L(0x75),
444 L(0xfa), L(0x8), L(0xb1), L(0xde), L(0x17), L(0x4a),
445 },
446 },
447 TestCase{
448 .input = "huffman-rand-limit.input",
449 .want = "huffman-rand-limit.{s}.expect",
450 .want_no_input = "huffman-rand-limit.{s}.expect-noinput",
451 .tokens = &[_]Token{
452 L(0x61), M(1, 74), L(0xa), L(0xf8), L(0x8b), L(0x96), L(0x76), L(0x48), L(0xa), L(0x85), L(0x94), L(0x25), L(0x80),
453 L(0xaf), L(0xc2), L(0xfe), L(0x8d), L(0xe8), L(0x20), L(0xeb), L(0x17), L(0x86), L(0xc9), L(0xb7), L(0xc5), L(0xde),
454 L(0x6), L(0xea), L(0x7d), L(0x18), L(0x8b), L(0xe7), L(0x3e), L(0x7), L(0xda), L(0xdf), L(0xff), L(0x6c), L(0x73),
455 L(0xde), L(0xcc), L(0xe7), L(0x6d), L(0x8d), L(0x4), L(0x19), L(0x49), L(0x7f), L(0x47), L(0x1f), L(0x48), L(0x15),
456 L(0xb0), L(0xe8), L(0x9e), L(0xf2), L(0x31), L(0x59), L(0xde), L(0x34), L(0xb4), L(0x5b), L(0xe5), L(0xe0), L(0x9),
457 L(0x11), L(0x30), L(0xc2), L(0x88), L(0x5b), L(0x7c), L(0x5d), L(0x14), L(0x13), L(0x6f), L(0x23), L(0xa9), L(0xa),
458 L(0xbc), L(0x2d), L(0x23), L(0xbe), L(0xd9), L(0xed), L(0x75), L(0x4), L(0x6c), L(0x99), L(0xdf), L(0xfd), L(0x70),
459 L(0x66), L(0xe6), L(0xee), L(0xd9), L(0xb1), L(0x9e), L(0x6e), L(0x83), L(0x59), L(0xd5), L(0xd4), L(0x80), L(0x59),
460 L(0x98), L(0x77), L(0x89), L(0x43), L(0x38), L(0xc9), L(0xaf), L(0x30), L(0x32), L(0x9a), L(0x20), L(0x1b), L(0x46),
461 L(0x3d), L(0x67), L(0x6e), L(0xd7), L(0x72), L(0x9e), L(0x4e), L(0x21), L(0x4f), L(0xc6), L(0xe0), L(0xd4), L(0x7b),
462 L(0x4), L(0x8d), L(0xa5), L(0x3), L(0xf6), L(0x5), L(0x9b), L(0x6b), L(0xdc), L(0x2a), L(0x93), L(0x77), L(0x28),
463 L(0xfd), L(0xb4), L(0x62), L(0xda), L(0x20), L(0xe7), L(0x1f), L(0xab), L(0x6b), L(0x51), L(0x43), L(0x39), L(0x2f),
464 L(0xa0), L(0x92), L(0x1), L(0x6c), L(0x75), L(0x3e), L(0xf4), L(0x35), L(0xfd), L(0x43), L(0x2e), L(0xf7), L(0xa4),
465 L(0x75), L(0xda), L(0xea), L(0x9b), L(0xa),
466 },
467 },
468 TestCase{
469 .input = "huffman-shifts.input",
470 .want = "huffman-shifts.{s}.expect",
471 .want_no_input = "huffman-shifts.{s}.expect-noinput",
472 .tokens = &[_]Token{
473 L('1'), L('0'), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
474 M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258),
475 M(2, 258), M(2, 76), L(0xd), L(0xa), L('2'), L('3'), M(2, 258), M(2, 258),
476 M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 258), M(2, 256),
477 },
478 },
479 TestCase{
480 .input = "huffman-text-shift.input",
481 .want = "huffman-text-shift.{s}.expect",
482 .want_no_input = "huffman-text-shift.{s}.expect-noinput",
483 .tokens = &[_]Token{
484 L('/'), L('/'), L('C'), L('o'), L('p'), L('y'), L('r'), L('i'),
485 L('g'), L('h'), L('t'), L('2'), L('0'), L('0'), L('9'), L('T'),
486 L('h'), L('G'), L('o'), L('A'), L('u'), L('t'), L('h'), L('o'),
487 L('r'), L('.'), L('A'), L('l'), L('l'), M(23, 5), L('r'), L('r'),
488 L('v'), L('d'), L('.'), L(0xd), L(0xa), L('/'), L('/'), L('U'),
489 L('o'), L('f'), L('t'), L('h'), L('i'), L('o'), L('u'), L('r'),
490 L('c'), L('c'), L('o'), L('d'), L('i'), L('g'), L('o'), L('v'),
491 L('r'), L('n'), L('d'), L('b'), L('y'), L('B'), L('S'), L('D'),
492 L('-'), L('t'), L('y'), L('l'), M(33, 4), L('l'), L('i'), L('c'),
493 L('n'), L('t'), L('h'), L('t'), L('c'), L('n'), L('b'), L('f'),
494 L('o'), L('u'), L('n'), L('d'), L('i'), L('n'), L('t'), L('h'),
495 L('L'), L('I'), L('C'), L('E'), L('N'), L('S'), L('E'), L('f'),
496 L('i'), L('l'), L('.'), L(0xd), L(0xa), L(0xd), L(0xa), L('p'),
497 L('c'), L('k'), L('g'), L('m'), L('i'), L('n'), M(11, 4), L('i'),
498 L('m'), L('p'), L('o'), L('r'), L('t'), L('"'), L('o'), L('"'),
499 M(13, 4), L('f'), L('u'), L('n'), L('c'), L('m'), L('i'), L('n'),
500 L('('), L(')'), L('{'), L(0xd), L(0xa), L(0x9), L('v'), L('r'),
501 L('b'), L('='), L('m'), L('k'), L('('), L('['), L(']'), L('b'),
502 L('y'), L('t'), L(','), L('6'), L('5'), L('5'), L('3'), L('5'),
503 L(')'), L(0xd), L(0xa), L(0x9), L('f'), L(','), L('_'), L(':'),
504 L('='), L('o'), L('.'), L('C'), L('r'), L('t'), L('('), L('"'),
505 L('h'), L('u'), L('f'), L('f'), L('m'), L('n'), L('-'), L('n'),
506 L('u'), L('l'), L('l'), L('-'), L('m'), L('x'), L('.'), L('i'),
507 L('n'), L('"'), M(34, 5), L('.'), L('W'), L('r'), L('i'), L('t'),
508 L('('), L('b'), L(')'), L(0xd), L(0xa), L('}'), L(0xd), L(0xa),
509 L('A'), L('B'), L('C'), L('D'), L('E'), L('F'), L('G'), L('H'),
510 L('I'), L('J'), L('K'), L('L'), L('M'), L('N'), L('O'), L('P'),
511 L('Q'), L('R'), L('S'), L('T'), L('U'), L('V'), L('X'), L('x'),
512 L('y'), L('z'), L('!'), L('"'), L('#'), L(0xc2), L(0xa4), L('%'),
513 L('&'), L('/'), L('?'), L('"'),
514 },
515 },
516 TestCase{
517 .input = "huffman-text.input",
518 .want = "huffman-text.{s}.expect",
519 .want_no_input = "huffman-text.{s}.expect-noinput",
520 .tokens = &[_]Token{
521 L('/'), L('/'), L(' '), L('z'), L('i'), L('g'), L(' '), L('v'),
522 L('0'), L('.'), L('1'), L('0'), L('.'), L('0'), L(0xa), L('/'),
523 L('/'), L(' '), L('c'), L('r'), L('e'), L('a'), L('t'), L('e'),
524 L(' '), L('a'), L(' '), L('f'), L('i'), L('l'), L('e'), M(5, 4),
525 L('l'), L('e'), L('d'), L(' '), L('w'), L('i'), L('t'), L('h'),
526 L(' '), L('0'), L('x'), L('0'), L('0'), L(0xa), L('c'), L('o'),
527 L('n'), L('s'), L('t'), L(' '), L('s'), L('t'), L('d'), L(' '),
528 L('='), L(' '), L('@'), L('i'), L('m'), L('p'), L('o'), L('r'),
529 L('t'), L('('), L('"'), L('s'), L('t'), L('d'), L('"'), L(')'),
530 L(';'), L(0xa), L(0xa), L('p'), L('u'), L('b'), L(' '), L('f'),
531 L('n'), L(' '), L('m'), L('a'), L('i'), L('n'), L('('), L(')'),
532 L(' '), L('!'), L('v'), L('o'), L('i'), L('d'), L(' '), L('{'),
533 L(0xa), L(' '), L(' '), L(' '), L(' '), L('v'), L('a'), L('r'),
534 L(' '), L('b'), L(' '), L('='), L(' '), L('['), L('1'), L(']'),
535 L('u'), L('8'), L('{'), L('0'), L('}'), L(' '), L('*'), L('*'),
536 L(' '), L('6'), L('5'), L('5'), L('3'), L('5'), L(';'), M(31, 5),
537 M(86, 6), L('f'), L(' '), L('='), L(' '), L('t'), L('r'), L('y'),
538 M(94, 4), L('.'), L('f'), L('s'), L('.'), L('c'), L('w'), L('d'),
539 L('('), L(')'), L('.'), M(144, 6), L('F'), L('i'), L('l'), L('e'),
540 L('('), M(43, 5), M(1, 4), L('"'), L('h'), L('u'), L('f'), L('f'),
541 L('m'), L('a'), L('n'), L('-'), L('n'), L('u'), L('l'), L('l'),
542 L('-'), L('m'), L('a'), L('x'), L('.'), L('i'), L('n'), L('"'),
543 L(','), M(31, 9), L('.'), L('{'), L(' '), L('.'), L('r'), L('e'),
544 L('a'), L('d'), M(79, 5), L('u'), L('e'), L(' '), L('}'), M(27, 6),
545 L(')'), M(108, 6), L('d'), L('e'), L('f'), L('e'), L('r'), L(' '),
546 L('f'), L('.'), L('c'), L('l'), L('o'), L('s'), L('e'), L('('),
547 M(183, 4), M(22, 4), L('_'), M(124, 7), L('f'), L('.'), L('w'), L('r'),
548 L('i'), L('t'), L('e'), L('A'), L('l'), L('l'), L('('), L('b'),
549 L('['), L('0'), L('.'), L('.'), L(']'), L(')'), L(';'), L(0xa),
550 L('}'), L(0xa),
551 },
552 },
553 TestCase{
554 .input = "huffman-zero.input",
555 .want = "huffman-zero.{s}.expect",
556 .want_no_input = "huffman-zero.{s}.expect-noinput",
557 .tokens = &[_]Token{ L(0x30), ml, M(1, 49) },
558 },
559 TestCase{
560 .input = "",
561 .want = "",
562 .want_no_input = "null-long-match.{s}.expect-noinput",
563 .tokens = &[_]Token{
564 L(0x0), ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
565 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
566 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
567 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
568 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
569 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
570 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
571 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
572 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
573 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
574 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
575 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
576 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
577 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
578 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
579 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
580 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
581 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
582 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
583 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
584 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
585 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
586 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
587 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
588 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
589 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
590 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
591 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
592 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
593 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
594 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
595 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
596 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
597 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
598 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
599 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
600 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
601 ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml, ml,
602 ml, ml, ml, M(1, 8),
603 },
604 },
605 };
606};
lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-null-max.input deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.input and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-null-max.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-pi.input deleted-1
......@@ -1 +0,0 @@
13.141592653589793238462643383279502884197169399375105820974944592307816406286208998628034825342117067982148086513282306647093844609550582231725359408128481117450284102701938521105559644622948954930381964428810975665933446128475648233786783165271201909145648566923460348610454326648213393607260249141273724587006606315588174881520920962829254091715364367892590360011330530548820466521384146951941511609433057270365759591953092186117381932611793105118548074462379962749567351885752724891227938183011949129833673362440656643086021394946395224737190702179860943702770539217176293176752384674818467669405132000568127145263560827785771342757789609173637178721468440901224953430146549585371050792279689258923542019956112129021960864034418159813629774771309960518707211349999998372978049951059731732816096318595024459455346908302642522308253344685035261931188171010003137838752886587533208381420617177669147303598253490428755468731159562863882353787593751957781857780532171226806613001927876611195909216420198938095257201065485863278865936153381827968230301952035301852968995773622599413891249721775283479131515574857242454150695950829533116861727855889075098381754637464939319255060400927701671139009848824012858361603563707660104710181942955596198946767837449448255379774726847104047534646208046684259069491293313677028989152104752162056966024058038150193511253382430035587640247496473263914199272604269922796782354781636009341721641219924586315030286182974555706749838505494588586926995690927210797509302955321165344987202755960236480665499119881834797753566369807426542527862551818417574672890977772793800081647060016145249192173217214772350141441973568548161361157352552133475741849468438523323907394143334547762416862518983569485562099219222184272550254256887671790494601653466804988627232791786085784383827967976681454100953883786360950680064225125205117392984896084128488626945604241965285022210661186306744278622039194945047123713786960956364371917287467764657573962413890865832645995813390478027590099465764078951269468398352595709825822620522489407726719478268482601476990902640136394437455305068203496252451749399651431429809190659250937221696461515709858387410597885959772975498930161753928468138268683868942774155991855925245953959431049972524680845987273644695848653836736222626099124608051243884390451244136549762780797715691435997700129616089441694868555848406353422072225828488648158456028506016842739452267467678895252138522549954666727823986456596116354886230577456498035593634568174324112515076069479451096596094025228879710893145669136867228748940560101503308617928680920874760917824938589009714909675985261365549781893129784821682998948722658804857564014270477555132379641451523746234364542858444795265867821051141354735739523113427166102135969536231442952484937187110145765403590279934403742007310578539062198387447808478489683321445713868751943506430218453191048481005370614680674919278191197939952061419663428754440643745123718192179998391015919561814675142691239748940907186494231961567945208095146550225231603881930142093762137855956638937787083039069792077346722182562599661501421503068038447734549202605414665925201497442850732518666002132434088190710486331734649651453905796268561005508106658796998163574736384052571459102897064140110971206280439039759515677157700420337869936007230558763176359421873125147120532928191826186125867321579198414848829164470609575270695722091756711672291098169091528017350671274858322287183520935396572512108357915136988209144421006751033467110314126711136990865851639831501970165151168517143765761835155650884909989859982387345528331635507647918535893226185489632132933089857064204675259070915481416549859461637180
\ No newline at end of file
lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-pi.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.input and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-1k.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.input deleted-4
......@@ -1,4 +0,0 @@
1aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
2���vH
3��%������ ��ɷ���}��>���ls���m�IGH����1Y�4�[�� 0ˆ[|]o#�
4�-#���ul���pf��ٱ�n�Y�ԀY�w�C8ɯ02� F=gn�r�N!O���{����k�*�w(��b� ��kQC9/��lu>�5�C.��u�
lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-limit.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-rand-max.input and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-shifts.input deleted-2
......@@ -1,2 +0,0 @@
1101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010101010
2232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323232323
\ No newline at end of file
lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-shifts.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text-shift.input deleted-14
......@@ -1,14 +0,0 @@
1//Copyright2009ThGoAuthor.Allrightrrvd.
2//UofthiourccodigovrndbyBSD-tyl
3//licnthtcnbfoundinthLICENSEfil.
4
5pckgmin
6
7import"o"
8
9funcmin(){
10 vrb=mk([]byt,65535)
11 f,_:=o.Crt("huffmn-null-mx.in")
12 f.Writ(b)
13}
14ABCDEFGHIJKLMNOPQRSTUVXxyz!"#¤%&/?"
\ No newline at end of file
lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text-shift.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text.input deleted-14
......@@ -1,14 +0,0 @@
1// zig v0.10.0
2// create a file filled with 0x00
3const std = @import("std");
4
5pub fn main() !void {
6 var b = [1]u8{0} ** 65535;
7 const f = try std.fs.cwd().createFile(
8 "huffman-null-max.in",
9 .{ .read = true },
10 );
11 defer f.close();
12
13 _ = try f.writeAll(b[0..]);
14}
lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-text.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.huff.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-zero.input deleted-1
......@@ -1 +0,0 @@
100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000
\ No newline at end of file
lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect and /dev/null differ
lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/huffman-zero.wb.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/null-long-match.dyn.expect-noinput and /dev/null differ
lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput deleted
Binary files a/lib/std/compress/flate/testdata/block_writer/null-long-match.wb.expect-noinput and /dev/null differ