diff --git a/lib/std/compress.zig b/lib/std/compress.zig index 05696c833f627517982b2b1eed5acb8939d83951..a6d0a40b26ef7d5ff28795b1c43e4ffcabc56280 100644 --- a/lib/std/compress.zig +++ b/lib/std/compress.zig @@ -1,14 +1,13 @@ const std = @import("std.zig"); +pub const flate = @import("compress/flate.zig"); +pub const gzip = @import("compress/gzip.zig"); +pub const zlib = @import("compress/zlib.zig"); pub const lzma = @import("compress/lzma.zig"); pub const lzma2 = @import("compress/lzma2.zig"); pub const xz = @import("compress/xz.zig"); pub const zstd = @import("compress/zstandard.zig"); -pub const flate = @import("compress/flate/root.zig").flate; -pub const gzip = @import("compress/flate/root.zig").gzip; -pub const zlib = @import("compress/flate/root.zig").zlib; - pub fn HashedReader( comptime ReaderType: anytype, comptime HasherType: anytype, diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig new file mode 100644 index 0000000000000000000000000000000000000000..1c96437bfdbdeafc416c9a1f4fe7e92a4858030e --- /dev/null +++ b/lib/std/compress/flate.zig @@ -0,0 +1,476 @@ +/// Deflate is a lossless data compression file format that uses a combination +/// of LZ77 and Huffman coding. +pub const deflate = @import("flate/deflate.zig"); + +/// Inflate is the decoding process that takes a Deflate bitstream for +/// decompression and correctly produces the original full-size data or file. +pub const inflate = @import("flate/inflate.zig"); + +/// Decompress compressed data from reader and write plain data to the writer. +pub fn decompress(reader: anytype, writer: anytype) !void { + try inflate.decompress(.raw, reader, writer); +} + +/// Decompressor type +pub fn Decompressor(comptime ReaderType: type) type { + return inflate.Inflate(.raw, ReaderType); +} + +/// Create Decompressor which will read compressed data from reader. +pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { + return inflate.decompressor(.raw, reader); +} + +/// Compression level, trades between speed and compression size. +pub const Options = deflate.Options; + +/// Compress plain data from reader and write compressed data to the writer. +pub fn compress(reader: anytype, writer: anytype, options: Options) !void { + try deflate.compress(.raw, reader, writer, options); +} + +/// Compressor type +pub fn Compressor(comptime WriterType: type) type { + return deflate.Compressor(.raw, WriterType); +} + +/// Create Compressor which outputs compressed data to the writer. +pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { + return try deflate.compressor(.raw, writer, options); +} + +/// Huffman only compression. Without Lempel-Ziv match searching. Faster +/// compression, less memory requirements but bigger compressed sizes. +pub const huffman = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.huffman.compress(.raw, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.huffman.Compressor(.raw, WriterType); + } + + pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { + return deflate.huffman.compressor(.raw, writer); + } +}; + +// No compression store only. Compressed size is slightly bigger than plain. +pub const store = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.store.compress(.raw, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.store.Compressor(.raw, WriterType); + } + + pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { + return deflate.store.compressor(.raw, writer); + } +}; + +/// Container defines header/footer arround deflate bit stream. Gzip and zlib +/// compression algorithms are containers arround deflate bit stream body. +const Container = @import("flate/container.zig").Container; +const std = @import("std"); +const testing = std.testing; +const fixedBufferStream = std.io.fixedBufferStream; +const print = std.debug.print; + +test "flate" { + _ = @import("flate/deflate.zig"); + _ = @import("flate/inflate.zig"); +} + +test "flate compress/decompress" { + var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer + var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer + + const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; + const cases = [_]struct { + data: []const u8, // uncompressed content + // compressed data sizes per level 4-9 + gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len, + huffman_only_size: usize = 0, + store_size: usize = 0, + }{ + .{ + .data = @embedFile("flate/testdata/rfc1951.txt"), + .gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 }, + .huffman_only_size = 20287, + .store_size = 36967, + }, + .{ + .data = @embedFile("flate/testdata/fuzz/roundtrip1.input"), + .gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 }, + .huffman_only_size = 393, + .store_size = 393, + }, + .{ + .data = @embedFile("flate/testdata/fuzz/roundtrip2.input"), + .gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 }, + .huffman_only_size = 394, + .store_size = 394, + }, + .{ + .data = @embedFile("flate/testdata/fuzz/deflate-stream.expect"), + .gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 }, + .huffman_only_size = 498, + .store_size = 747, + }, + }; + + for (cases, 0..) |case, case_no| { // for each case + const data = case.data; + + for (levels, 0..) |level, i| { // for each compression level + + inline for (Container.list) |container| { // for each wrapping + var compressed_size: usize = if (case.gzip_sizes[i] > 0) + case.gzip_sizes[i] - Container.gzip.size() + container.size() + else + 0; + + // compress original stream to compressed stream + { + var original = fixedBufferStream(data); + var compressed = fixedBufferStream(&cmp_buf); + try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level }); + if (compressed_size == 0) { + if (container == .gzip) + print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos }); + compressed_size = compressed.pos; + } + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompress compressed stream to decompressed stream + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var decompressed = fixedBufferStream(&dcm_buf); + try inflate.decompress(container, compressed.reader(), decompressed.writer()); + try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + } + + // compressor writer interface + { + var compressed = fixedBufferStream(&cmp_buf); + var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level }); + var cmp_wrt = cmp.writer(); + try cmp_wrt.writeAll(data); + try cmp.finish(); + + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompressor reader interface + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var dcm = inflate.decompressor(container, compressed.reader()); + var dcm_rdr = dcm.reader(); + const n = try dcm_rdr.readAll(&dcm_buf); + try testing.expectEqual(data.len, n); + try testing.expectEqualSlices(u8, data, dcm_buf[0..n]); + } + } + } + // huffman only compression + { + inline for (Container.list) |container| { // for each wrapping + var compressed_size: usize = if (case.huffman_only_size > 0) + case.huffman_only_size - Container.gzip.size() + container.size() + else + 0; + + // compress original stream to compressed stream + { + var original = fixedBufferStream(data); + var compressed = fixedBufferStream(&cmp_buf); + var cmp = try deflate.huffman.compressor(container, compressed.writer()); + try cmp.compress(original.reader()); + try cmp.finish(); + if (compressed_size == 0) { + if (container == .gzip) + print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos }); + compressed_size = compressed.pos; + } + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompress compressed stream to decompressed stream + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var decompressed = fixedBufferStream(&dcm_buf); + try inflate.decompress(container, compressed.reader(), decompressed.writer()); + try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + } + } + } + + // store only + { + inline for (Container.list) |container| { // for each wrapping + var compressed_size: usize = if (case.store_size > 0) + case.store_size - Container.gzip.size() + container.size() + else + 0; + + // compress original stream to compressed stream + { + var original = fixedBufferStream(data); + var compressed = fixedBufferStream(&cmp_buf); + var cmp = try deflate.store.compressor(container, compressed.writer()); + try cmp.compress(original.reader()); + try cmp.finish(); + if (compressed_size == 0) { + if (container == .gzip) + print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos }); + compressed_size = compressed.pos; + } + + try testing.expectEqual(compressed_size, compressed.pos); + } + // decompress compressed stream to decompressed stream + { + var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); + var decompressed = fixedBufferStream(&dcm_buf); + try inflate.decompress(container, compressed.reader(), decompressed.writer()); + try testing.expectEqualSlices(u8, data, decompressed.getWritten()); + } + } + } + } +} + +fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void { + var in = fixedBufferStream(compressed); + var out = std.ArrayList(u8).init(testing.allocator); + defer out.deinit(); + + try inflate.decompress(container, in.reader(), out.writer()); + try testing.expectEqualSlices(u8, expected_plain, out.items); +} + +test "flate don't read past deflate stream's end" { + try testDecompress(.zlib, &[_]u8{ + 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff, + 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01, + 0x83, 0x95, 0x0b, 0xf5, + }, &[_]u8{ + 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x00, 0xff, 0xff, 0xff, + }); +} + +test "flate zlib header" { + // Truncated header + try testing.expectError( + error.EndOfStream, + testDecompress(.zlib, &[_]u8{0x78}, ""), + ); + // Wrong CM + try testing.expectError( + error.BadZlibHeader, + testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""), + ); + // Wrong CINFO + try testing.expectError( + error.BadZlibHeader, + testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""), + ); + // Wrong checksum + try testing.expectError( + error.WrongZlibChecksum, + testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), + ); + // Truncated checksum + try testing.expectError( + error.EndOfStream, + testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), + ); +} + +test "flate gzip header" { + // Truncated header + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined), + ); + // Wrong CM + try testing.expectError( + error.BadGzipHeader, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, + }, undefined), + ); + + // Wrong checksum + try testing.expectError( + error.WrongGzipChecksum, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, + }, undefined), + ); + // Truncated checksum + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, + }, undefined), + ); + // Wrong initial size + try testing.expectError( + error.WrongGzipSize, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, + }, undefined), + ); + // Truncated initial size field + try testing.expectError( + error.EndOfStream, + testDecompress(.gzip, &[_]u8{ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, + }, undefined), + ); + + try testDecompress(.gzip, &[_]u8{ + // GZIP header + 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, + // header.FHCRC (should cover entire header) + 0x99, 0xd6, + // GZIP data + 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + }, ""); +} + +test "flate public interface" { + const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a }; + + // deflate final stored block, header + plain (stored) data + const deflate_block = [_]u8{ + 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen + } ++ plain_data; + + // gzip header/footer + deflate block + const gzip_data = + [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes) + deflate_block ++ + [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes) + + // zlib header/footer + deflate block + const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)} + deflate_block ++ + [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum + + const gzip = @import("gzip.zig"); + const zlib = @import("zlib.zig"); + const flate = @This(); + + try testInterface(gzip, &gzip_data, &plain_data); + try testInterface(zlib, &zlib_data, &plain_data); + try testInterface(flate, &deflate_block, &plain_data); +} + +fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void { + var buffer1: [64]u8 = undefined; + var buffer2: [64]u8 = undefined; + + var compressed = fixedBufferStream(&buffer1); + var plain = fixedBufferStream(&buffer2); + + // decompress + { + var in = fixedBufferStream(gzip_data); + try pkg.decompress(in.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // compress/decompress + { + var in = fixedBufferStream(plain_data); + try pkg.compress(in.reader(), compressed.writer(), .{}); + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // compressor/decompressor + { + var in = fixedBufferStream(plain_data); + var cmp = try pkg.compressor(compressed.writer(), .{}); + try cmp.compress(in.reader()); + try cmp.finish(); + + compressed.reset(); + var dcp = pkg.decompressor(compressed.reader()); + try dcp.decompress(plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // huffman + { + // huffman compress/decompress + { + var in = fixedBufferStream(plain_data); + try pkg.huffman.compress(in.reader(), compressed.writer()); + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // huffman compressor/decompressor + { + var in = fixedBufferStream(plain_data); + var cmp = try pkg.huffman.compressor(compressed.writer()); + try cmp.compress(in.reader()); + try cmp.finish(); + + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + } + plain.reset(); + compressed.reset(); + + // store + { + // store compress/decompress + { + var in = fixedBufferStream(plain_data); + try pkg.store.compress(in.reader(), compressed.writer()); + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + plain.reset(); + compressed.reset(); + + // store compressor/decompressor + { + var in = fixedBufferStream(plain_data); + var cmp = try pkg.store.compressor(compressed.writer()); + try cmp.compress(in.reader()); + try cmp.finish(); + + compressed.reset(); + try pkg.decompress(compressed.reader(), plain.writer()); + try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); + } + } +} diff --git a/lib/std/compress/flate/flate.zig b/lib/std/compress/flate/flate.zig deleted file mode 100644 index 82f46d2f79c3bfed9abc36903ef16eeb0d4f27c7..0000000000000000000000000000000000000000 --- a/lib/std/compress/flate/flate.zig +++ /dev/null @@ -1,347 +0,0 @@ -/// Deflate is a lossless data compression file format that uses a combination -/// of LZ77 and Huffman coding. -pub const deflate = @import("deflate.zig"); - -/// Inflate is the decoding process that takes a Deflate bitstream for -/// decompression and correctly produces the original full-size data or file. -pub const inflate = @import("inflate.zig"); - -/// Decompress compressed data from reader and write plain data to the writer. -pub fn decompress(reader: anytype, writer: anytype) !void { - try inflate.decompress(.raw, reader, writer); -} - -/// Decompressor type -pub fn Decompressor(comptime ReaderType: type) type { - return inflate.Inflate(.raw, ReaderType); -} - -/// Create Decompressor which will read compressed data from reader. -pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { - return inflate.decompressor(.raw, reader); -} - -/// Compression level, trades between speed and compression size. -pub const Options = deflate.Options; - -/// Compress plain data from reader and write compressed data to the writer. -pub fn compress(reader: anytype, writer: anytype, options: Options) !void { - try deflate.compress(.raw, reader, writer, options); -} - -/// Compressor type -pub fn Compressor(comptime WriterType: type) type { - return deflate.Compressor(.raw, WriterType); -} - -/// Create Compressor which outputs compressed data to the writer. -pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { - return try deflate.compressor(.raw, writer, options); -} - -/// Huffman only compression. Without Lempel-Ziv match searching. Faster -/// compression, less memory requirements but bigger compressed sizes. -pub const huffman = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.huffman.compress(.raw, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.huffman.Compressor(.raw, WriterType); - } - - pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { - return deflate.huffman.compressor(.raw, writer); - } -}; - -// No compression store only. Compressed size is slightly bigger than plain. -pub const store = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.store.compress(.raw, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.store.Compressor(.raw, WriterType); - } - - pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { - return deflate.store.compressor(.raw, writer); - } -}; - -/// Container defines header/footer arround deflate bit stream. Gzip and zlib -/// compression algorithms are containers arround deflate bit stream body. -const Container = @import("container.zig").Container; -const std = @import("std"); -const testing = std.testing; -const fixedBufferStream = std.io.fixedBufferStream; -const print = std.debug.print; - -test "flate compress/decompress" { - var cmp_buf: [64 * 1024]u8 = undefined; // compressed data buffer - var dcm_buf: [64 * 1024]u8 = undefined; // decompressed data buffer - - const levels = [_]deflate.Level{ .level_4, .level_5, .level_6, .level_7, .level_8, .level_9 }; - const cases = [_]struct { - data: []const u8, // uncompressed content - // compressed data sizes per level 4-9 - gzip_sizes: [levels.len]usize = [_]usize{0} ** levels.len, - huffman_only_size: usize = 0, - store_size: usize = 0, - }{ - .{ - .data = @embedFile("testdata/rfc1951.txt"), - .gzip_sizes = [_]usize{ 11513, 11217, 11139, 11126, 11122, 11119 }, - .huffman_only_size = 20287, - .store_size = 36967, - }, - .{ - .data = @embedFile("testdata/fuzz/roundtrip1.input"), - .gzip_sizes = [_]usize{ 373, 370, 370, 370, 370, 370 }, - .huffman_only_size = 393, - .store_size = 393, - }, - .{ - .data = @embedFile("testdata/fuzz/roundtrip2.input"), - .gzip_sizes = [_]usize{ 373, 373, 373, 373, 373, 373 }, - .huffman_only_size = 394, - .store_size = 394, - }, - .{ - .data = @embedFile("testdata/fuzz/deflate-stream.expect"), - .gzip_sizes = [_]usize{ 351, 347, 347, 347, 347, 347 }, - .huffman_only_size = 498, - .store_size = 747, - }, - }; - - for (cases, 0..) |case, case_no| { // for each case - const data = case.data; - - for (levels, 0..) |level, i| { // for each compression level - - inline for (Container.list) |container| { // for each wrapping - var compressed_size: usize = if (case.gzip_sizes[i] > 0) - case.gzip_sizes[i] - Container.gzip.size() + container.size() - else - 0; - - // compress original stream to compressed stream - { - var original = fixedBufferStream(data); - var compressed = fixedBufferStream(&cmp_buf); - try deflate.compress(container, original.reader(), compressed.writer(), .{ .level = level }); - if (compressed_size == 0) { - if (container == .gzip) - print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos }); - compressed_size = compressed.pos; - } - try testing.expectEqual(compressed_size, compressed.pos); - } - // decompress compressed stream to decompressed stream - { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var decompressed = fixedBufferStream(&dcm_buf); - try inflate.decompress(container, compressed.reader(), decompressed.writer()); - try testing.expectEqualSlices(u8, data, decompressed.getWritten()); - } - - // compressor writer interface - { - var compressed = fixedBufferStream(&cmp_buf); - var cmp = try deflate.compressor(container, compressed.writer(), .{ .level = level }); - var cmp_wrt = cmp.writer(); - try cmp_wrt.writeAll(data); - try cmp.finish(); - - try testing.expectEqual(compressed_size, compressed.pos); - } - // decompressor reader interface - { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var dcm = inflate.decompressor(container, compressed.reader()); - var dcm_rdr = dcm.reader(); - const n = try dcm_rdr.readAll(&dcm_buf); - try testing.expectEqual(data.len, n); - try testing.expectEqualSlices(u8, data, dcm_buf[0..n]); - } - } - } - // huffman only compression - { - inline for (Container.list) |container| { // for each wrapping - var compressed_size: usize = if (case.huffman_only_size > 0) - case.huffman_only_size - Container.gzip.size() + container.size() - else - 0; - - // compress original stream to compressed stream - { - var original = fixedBufferStream(data); - var compressed = fixedBufferStream(&cmp_buf); - var cmp = try deflate.huffman.compressor(container, compressed.writer()); - try cmp.compress(original.reader()); - try cmp.finish(); - if (compressed_size == 0) { - if (container == .gzip) - print("case {d} huffman only compressed size: {d}\n", .{ case_no, compressed.pos }); - compressed_size = compressed.pos; - } - try testing.expectEqual(compressed_size, compressed.pos); - } - // decompress compressed stream to decompressed stream - { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var decompressed = fixedBufferStream(&dcm_buf); - try inflate.decompress(container, compressed.reader(), decompressed.writer()); - try testing.expectEqualSlices(u8, data, decompressed.getWritten()); - } - } - } - - // store only - { - inline for (Container.list) |container| { // for each wrapping - var compressed_size: usize = if (case.store_size > 0) - case.store_size - Container.gzip.size() + container.size() - else - 0; - - // compress original stream to compressed stream - { - var original = fixedBufferStream(data); - var compressed = fixedBufferStream(&cmp_buf); - var cmp = try deflate.store.compressor(container, compressed.writer()); - try cmp.compress(original.reader()); - try cmp.finish(); - if (compressed_size == 0) { - if (container == .gzip) - print("case {d} store only compressed size: {d}\n", .{ case_no, compressed.pos }); - compressed_size = compressed.pos; - } - - try testing.expectEqual(compressed_size, compressed.pos); - } - // decompress compressed stream to decompressed stream - { - var compressed = fixedBufferStream(cmp_buf[0..compressed_size]); - var decompressed = fixedBufferStream(&dcm_buf); - try inflate.decompress(container, compressed.reader(), decompressed.writer()); - try testing.expectEqualSlices(u8, data, decompressed.getWritten()); - } - } - } - } -} - -fn testDecompress(comptime container: Container, compressed: []const u8, expected_plain: []const u8) !void { - var in_stream = std.io.fixedBufferStream(compressed); - var al = std.ArrayList(u8).init(testing.allocator); - defer al.deinit(); - - try inflate.decompress(container, in_stream.reader(), al.writer()); - - try testing.expectEqualSlices(u8, expected_plain, al.items); -} - -test "flate don't read past deflate stream's end" { - try testDecompress(.zlib, &[_]u8{ - 0x08, 0xd7, 0x63, 0xf8, 0xcf, 0xc0, 0xc0, 0x00, 0xc1, 0xff, - 0xff, 0x43, 0x30, 0x03, 0x03, 0xc3, 0xff, 0xff, 0xff, 0x01, - 0x83, 0x95, 0x0b, 0xf5, - }, &[_]u8{ - 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, 0x00, 0x00, 0x00, 0xff, - 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x00, 0xff, 0xff, 0xff, - }); -} - -test "flate zlib header" { - // Truncated header - try testing.expectError( - error.EndOfStream, - testDecompress(.zlib, &[_]u8{0x78}, ""), - ); - // Wrong CM - try testing.expectError( - error.BadZlibHeader, - testDecompress(.zlib, &[_]u8{ 0x79, 0x94 }, ""), - ); - // Wrong CINFO - try testing.expectError( - error.BadZlibHeader, - testDecompress(.zlib, &[_]u8{ 0x88, 0x98 }, ""), - ); - // Wrong checksum - try testing.expectError( - error.WrongZlibChecksum, - testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00 }, ""), - ); - // Truncated checksum - try testing.expectError( - error.EndOfStream, - testDecompress(.zlib, &[_]u8{ 0x78, 0xda, 0x03, 0x00, 0x00 }, ""), - ); -} - -test "flate gzip header" { - // Truncated header - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ 0x1f, 0x8B }, undefined), - ); - // Wrong CM - try testing.expectError( - error.BadGzipHeader, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, - }, undefined), - ); - - // Wrong checksum - try testing.expectError( - error.WrongGzipChecksum, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, - }, undefined), - ); - // Truncated checksum - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, - }, undefined), - ); - // Wrong initial size - try testing.expectError( - error.WrongGzipSize, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, - }, undefined), - ); - // Truncated initial size field - try testing.expectError( - error.EndOfStream, - testDecompress(.gzip, &[_]u8{ - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, - }, undefined), - ); - - try testDecompress(.gzip, &[_]u8{ - // GZIP header - 0x1f, 0x8b, 0x08, 0x12, 0x00, 0x09, 0x6e, 0x88, 0x00, 0xff, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x00, - // header.FHCRC (should cover entire header) - 0x99, 0xd6, - // GZIP data - 0x01, 0x00, 0x00, 0xff, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - }, ""); -} diff --git a/lib/std/compress/flate/gzip.zig b/lib/std/compress/flate/gzip.zig deleted file mode 100644 index feb9ae07b5caf109369375315b84ceb93979f3b9..0000000000000000000000000000000000000000 --- a/lib/std/compress/flate/gzip.zig +++ /dev/null @@ -1,66 +0,0 @@ -const deflate = @import("deflate.zig"); -const inflate = @import("inflate.zig"); - -/// Decompress compressed data from reader and write plain data to the writer. -pub fn decompress(reader: anytype, writer: anytype) !void { - try inflate.decompress(.gzip, reader, writer); -} - -/// Decompressor type -pub fn Decompressor(comptime ReaderType: type) type { - return inflate.Inflate(.gzip, ReaderType); -} - -/// Create Decompressor which will read compressed data from reader. -pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { - return inflate.decompressor(.gzip, reader); -} - -/// Compression level, trades between speed and compression size. -pub const Options = deflate.Options; - -/// Compress plain data from reader and write compressed data to the writer. -pub fn compress(reader: anytype, writer: anytype, options: Options) !void { - try deflate.compress(.gzip, reader, writer, options); -} - -/// Compressor type -pub fn Compressor(comptime WriterType: type) type { - return deflate.Compressor(.gzip, WriterType); -} - -/// Create Compressor which outputs compressed data to the writer. -pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { - return try deflate.compressor(.gzip, writer, options); -} - -/// Huffman only compression. Without Lempel-Ziv match searching. Faster -/// compression, less memory requirements but bigger compressed sizes. -pub const huffman = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.huffman.compress(.gzip, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.huffman.Compressor(.gzip, WriterType); - } - - pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { - return deflate.huffman.compressor(.gzip, writer); - } -}; - -// No compression store only. Compressed size is slightly bigger than plain. -pub const store = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.store.compress(.gzip, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.store.Compressor(.gzip, WriterType); - } - - pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { - return deflate.store.compressor(.gzip, writer); - } -}; diff --git a/lib/std/compress/flate/root.zig b/lib/std/compress/flate/root.zig deleted file mode 100644 index da385b21b01863088aa0242792c9de30dfd5d934..0000000000000000000000000000000000000000 --- a/lib/std/compress/flate/root.zig +++ /dev/null @@ -1,133 +0,0 @@ -pub const flate = @import("flate.zig"); -pub const gzip = @import("gzip.zig"); -pub const zlib = @import("zlib.zig"); - -test "flate" { - _ = @import("deflate.zig"); - _ = @import("inflate.zig"); -} - -test "flate public interface" { - const plain_data = [_]u8{ 'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a }; - - // deflate final stored block, header + plain (stored) data - const deflate_block = [_]u8{ - 0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen - } ++ plain_data; - - // gzip header/footer + deflate block - const gzip_data = - [_]u8{ 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03 } ++ // gzip header (10 bytes) - deflate_block ++ - [_]u8{ 0xd5, 0xe0, 0x39, 0xb7, 0x0c, 0x00, 0x00, 0x00 }; // gzip footer checksum (4 byte), size (4 bytes) - - // zlib header/footer + deflate block - const zlib_data = [_]u8{ 0x78, 0b10_0_11100 } ++ // zlib header (2 bytes)} - deflate_block ++ - [_]u8{ 0x1c, 0xf2, 0x04, 0x47 }; // zlib footer: checksum - - try testInterface(gzip, &gzip_data, &plain_data); - try testInterface(zlib, &zlib_data, &plain_data); - try testInterface(flate, &deflate_block, &plain_data); -} - -fn testInterface(comptime pkg: type, gzip_data: []const u8, plain_data: []const u8) !void { - const std = @import("std"); - const testing = std.testing; - const fixedBufferStream = std.io.fixedBufferStream; - - var buffer1: [64]u8 = undefined; - var buffer2: [64]u8 = undefined; - - var compressed = fixedBufferStream(&buffer1); - var plain = fixedBufferStream(&buffer2); - - // decompress - { - var in = fixedBufferStream(gzip_data); - try pkg.decompress(in.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); - } - plain.reset(); - compressed.reset(); - - // compress/decompress - { - var in = fixedBufferStream(plain_data); - try pkg.compress(in.reader(), compressed.writer(), .{}); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); - } - plain.reset(); - compressed.reset(); - - // compressor/decompressor - { - var in = fixedBufferStream(plain_data); - var cmp = try pkg.compressor(compressed.writer(), .{}); - try cmp.compress(in.reader()); - try cmp.finish(); - - compressed.reset(); - var dcp = pkg.decompressor(compressed.reader()); - try dcp.decompress(plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); - } - plain.reset(); - compressed.reset(); - - // huffman - { - // huffman compress/decompress - { - var in = fixedBufferStream(plain_data); - try pkg.huffman.compress(in.reader(), compressed.writer()); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); - } - plain.reset(); - compressed.reset(); - - // huffman compressor/decompressor - { - var in = fixedBufferStream(plain_data); - var cmp = try pkg.huffman.compressor(compressed.writer()); - try cmp.compress(in.reader()); - try cmp.finish(); - - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); - } - } - plain.reset(); - compressed.reset(); - - // store - { - // store compress/decompress - { - var in = fixedBufferStream(plain_data); - try pkg.store.compress(in.reader(), compressed.writer()); - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); - } - plain.reset(); - compressed.reset(); - - // store compressor/decompressor - { - var in = fixedBufferStream(plain_data); - var cmp = try pkg.store.compressor(compressed.writer()); - try cmp.compress(in.reader()); - try cmp.finish(); - - compressed.reset(); - try pkg.decompress(compressed.reader(), plain.writer()); - try testing.expectEqualSlices(u8, plain_data, plain.getWritten()); - } - } -} diff --git a/lib/std/compress/flate/zlib.zig b/lib/std/compress/flate/zlib.zig deleted file mode 100644 index 328e625119dffabe130575aa0503b363606ac3b4..0000000000000000000000000000000000000000 --- a/lib/std/compress/flate/zlib.zig +++ /dev/null @@ -1,66 +0,0 @@ -const deflate = @import("deflate.zig"); -const inflate = @import("inflate.zig"); - -/// Decompress compressed data from reader and write plain data to the writer. -pub fn decompress(reader: anytype, writer: anytype) !void { - try inflate.decompress(.zlib, reader, writer); -} - -/// Decompressor type -pub fn Decompressor(comptime ReaderType: type) type { - return inflate.Inflate(.zlib, ReaderType); -} - -/// Create Decompressor which will read compressed data from reader. -pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { - return inflate.decompressor(.zlib, reader); -} - -/// Compression level, trades between speed and compression size. -pub const Options = deflate.Options; - -/// Compress plain data from reader and write compressed data to the writer. -pub fn compress(reader: anytype, writer: anytype, options: Options) !void { - try deflate.compress(.zlib, reader, writer, options); -} - -/// Compressor type -pub fn Compressor(comptime WriterType: type) type { - return deflate.Compressor(.zlib, WriterType); -} - -/// Create Compressor which outputs compressed data to the writer. -pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { - return try deflate.compressor(.zlib, writer, options); -} - -/// Huffman only compression. Without Lempel-Ziv match searching. Faster -/// compression, less memory requirements but bigger compressed sizes. -pub const huffman = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.huffman.compress(.zlib, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.huffman.Compressor(.zlib, WriterType); - } - - pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { - return deflate.huffman.compressor(.zlib, writer); - } -}; - -// No compression store only. Compressed size is slightly bigger than plain. -pub const store = struct { - pub fn compress(reader: anytype, writer: anytype) !void { - try deflate.store.compress(.zlib, reader, writer); - } - - pub fn Compressor(comptime WriterType: type) type { - return deflate.store.Compressor(.zlib, WriterType); - } - - pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { - return deflate.store.compressor(.zlib, writer); - } -}; diff --git a/lib/std/compress/gzip.zig b/lib/std/compress/gzip.zig new file mode 100644 index 0000000000000000000000000000000000000000..8bb09c612ab081c8217313f13ab72eb3dd2d1d25 --- /dev/null +++ b/lib/std/compress/gzip.zig @@ -0,0 +1,66 @@ +const deflate = @import("flate/deflate.zig"); +const inflate = @import("flate/inflate.zig"); + +/// Decompress compressed data from reader and write plain data to the writer. +pub fn decompress(reader: anytype, writer: anytype) !void { + try inflate.decompress(.gzip, reader, writer); +} + +/// Decompressor type +pub fn Decompressor(comptime ReaderType: type) type { + return inflate.Inflate(.gzip, ReaderType); +} + +/// Create Decompressor which will read compressed data from reader. +pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { + return inflate.decompressor(.gzip, reader); +} + +/// Compression level, trades between speed and compression size. +pub const Options = deflate.Options; + +/// Compress plain data from reader and write compressed data to the writer. +pub fn compress(reader: anytype, writer: anytype, options: Options) !void { + try deflate.compress(.gzip, reader, writer, options); +} + +/// Compressor type +pub fn Compressor(comptime WriterType: type) type { + return deflate.Compressor(.gzip, WriterType); +} + +/// Create Compressor which outputs compressed data to the writer. +pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { + return try deflate.compressor(.gzip, writer, options); +} + +/// Huffman only compression. Without Lempel-Ziv match searching. Faster +/// compression, less memory requirements but bigger compressed sizes. +pub const huffman = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.huffman.compress(.gzip, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.huffman.Compressor(.gzip, WriterType); + } + + pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { + return deflate.huffman.compressor(.gzip, writer); + } +}; + +// No compression store only. Compressed size is slightly bigger than plain. +pub const store = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.store.compress(.gzip, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.store.Compressor(.gzip, WriterType); + } + + pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { + return deflate.store.compressor(.gzip, writer); + } +}; diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig new file mode 100644 index 0000000000000000000000000000000000000000..33401ce8457cfcf79c8ed0ed688407b40d35bc19 --- /dev/null +++ b/lib/std/compress/zlib.zig @@ -0,0 +1,66 @@ +const deflate = @import("flate/deflate.zig"); +const inflate = @import("flate/inflate.zig"); + +/// Decompress compressed data from reader and write plain data to the writer. +pub fn decompress(reader: anytype, writer: anytype) !void { + try inflate.decompress(.zlib, reader, writer); +} + +/// Decompressor type +pub fn Decompressor(comptime ReaderType: type) type { + return inflate.Inflate(.zlib, ReaderType); +} + +/// Create Decompressor which will read compressed data from reader. +pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) { + return inflate.decompressor(.zlib, reader); +} + +/// Compression level, trades between speed and compression size. +pub const Options = deflate.Options; + +/// Compress plain data from reader and write compressed data to the writer. +pub fn compress(reader: anytype, writer: anytype, options: Options) !void { + try deflate.compress(.zlib, reader, writer, options); +} + +/// Compressor type +pub fn Compressor(comptime WriterType: type) type { + return deflate.Compressor(.zlib, WriterType); +} + +/// Create Compressor which outputs compressed data to the writer. +pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) { + return try deflate.compressor(.zlib, writer, options); +} + +/// Huffman only compression. Without Lempel-Ziv match searching. Faster +/// compression, less memory requirements but bigger compressed sizes. +pub const huffman = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.huffman.compress(.zlib, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.huffman.Compressor(.zlib, WriterType); + } + + pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) { + return deflate.huffman.compressor(.zlib, writer); + } +}; + +// No compression store only. Compressed size is slightly bigger than plain. +pub const store = struct { + pub fn compress(reader: anytype, writer: anytype) !void { + try deflate.store.compress(.zlib, reader, writer); + } + + pub fn Compressor(comptime WriterType: type) type { + return deflate.store.Compressor(.zlib, WriterType); + } + + pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) { + return deflate.store.compressor(.zlib, writer); + } +};