authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-24 15:04:56-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-24 15:24:19-07:00
logea9ded87582a8b9d0ed3afd3360a1d75f0359a5c
tree9767d5562df1fd55bcc68472d20c6e5b9733fa46
parent06ce15e8f719756cc12d928cfdae12be99a9e4c2

std.compress.xz public API cleanup

* add xz to std.compress * prefer importing std.zig by file name, to reduce reliance on the standard library being a special case. * extract some types from inside generic functions. These types are the same regardless of the generic parameters. * expose some more types in the std.compress.xz namespace. * rename xz.stream to xz.decompress * rename check.Kind to Check * use std.leb for LEB instead of a redundant implementation

9 files changed, 234 insertions(+), 263 deletions(-)

lib/std/compress.zig+2
...@@ -3,6 +3,7 @@ const std = @import("std.zig");...@@ -3,6 +3,7 @@ const std = @import("std.zig");
3pub const deflate = @import("compress/deflate.zig");3pub const deflate = @import("compress/deflate.zig");
4pub const gzip = @import("compress/gzip.zig");4pub const gzip = @import("compress/gzip.zig");
5pub const zlib = @import("compress/zlib.zig");5pub const zlib = @import("compress/zlib.zig");
6pub const xz = @import("compress/xz.zig");
67
7pub fn HashedReader(8pub fn HashedReader(
8 comptime ReaderType: anytype,9 comptime ReaderType: anytype,
...@@ -38,4 +39,5 @@ test {...@@ -38,4 +39,5 @@ test {
38 _ = deflate;39 _ = deflate;
39 _ = gzip;40 _ = gzip;
40 _ = zlib;41 _ = zlib;
42 _ = xz;
41}43}
lib/std/compress/xz.zig+139-2
...@@ -1,5 +1,142 @@...@@ -1,5 +1,142 @@
1pub usingnamespace @import("xz/stream.zig");1const std = @import("std");
2const block = @import("xz/block.zig");
3const Allocator = std.mem.Allocator;
4const Crc32 = std.hash.Crc32;
5
6pub const Flags = packed struct(u16) {
7 reserved1: u8,
8 check_kind: Check,
9 reserved2: u4,
10};
11
12pub const Header = extern struct {
13 magic: [6]u8,
14 flags: Flags,
15 crc32: u32,
16};
17
18pub const Footer = extern struct {
19 crc32: u32,
20 backward_size: u32,
21 flags: Flags,
22 magic: [2]u8,
23};
24
25pub const Check = enum(u4) {
26 none = 0x00,
27 crc32 = 0x01,
28 crc64 = 0x04,
29 sha256 = 0x0A,
30 _,
31};
32
33pub fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
34 return Decompress(@TypeOf(reader)).init(allocator, reader);
35}
36
37pub fn Decompress(comptime ReaderType: type) type {
38 return struct {
39 const Self = @This();
40
41 pub const Error = ReaderType.Error || block.Decoder(ReaderType).Error;
42 pub const Reader = std.io.Reader(*Self, Error, read);
43
44 allocator: Allocator,
45 block_decoder: block.Decoder(ReaderType),
46 in_reader: ReaderType,
47
48 fn init(allocator: Allocator, source: ReaderType) !Self {
49 const header = try source.readStruct(Header);
50
51 if (!std.mem.eql(u8, &header.magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
52 return error.BadHeader;
53
54 if (header.flags.reserved1 != 0 or header.flags.reserved2 != 0)
55 return error.BadHeader;
56
57 const hash = Crc32.hash(std.mem.asBytes(&header.flags));
58 if (hash != header.crc32)
59 return error.WrongChecksum;
60
61 return Self{
62 .allocator = allocator,
63 .block_decoder = try block.decoder(allocator, source, header.flags.check_kind),
64 .in_reader = source,
65 };
66 }
67
68 pub fn deinit(self: *Self) void {
69 self.block_decoder.deinit();
70 }
71
72 pub fn reader(self: *Self) Reader {
73 return .{ .context = self };
74 }
75
76 pub fn read(self: *Self, buffer: []u8) Error!usize {
77 if (buffer.len == 0)
78 return 0;
79
80 const r = try self.block_decoder.read(buffer);
81 if (r != 0)
82 return r;
83
84 const index_size = blk: {
85 var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
86 hasher.hasher.update(&[1]u8{0x00});
87
88 var counter = std.io.countingReader(hasher.reader());
89 counter.bytes_read += 1;
90
91 const counting_reader = counter.reader();
92
93 const record_count = try std.leb.readULEB128(u64, counting_reader);
94 if (record_count != self.block_decoder.block_count)
95 return error.CorruptInput;
96
97 var i: usize = 0;
98 while (i < record_count) : (i += 1) {
99 // TODO: validate records
100 _ = try std.leb.readULEB128(u64, counting_reader);
101 _ = try std.leb.readULEB128(u64, counting_reader);
102 }
103
104 while (counter.bytes_read % 4 != 0) {
105 if (try counting_reader.readByte() != 0)
106 return error.CorruptInput;
107 }
108
109 const hash_a = hasher.hasher.final();
110 const hash_b = try counting_reader.readIntLittle(u32);
111 if (hash_a != hash_b)
112 return error.WrongChecksum;
113
114 break :blk counter.bytes_read;
115 };
116
117 const footer = try self.in_reader.readStruct(Footer);
118 const backward_size = (footer.backward_size + 1) * 4;
119 if (backward_size != index_size)
120 return error.CorruptInput;
121
122 if (footer.flags.reserved1 != 0 or footer.flags.reserved2 != 0)
123 return error.CorruptInput;
124
125 var hasher = Crc32.init();
126 hasher.update(std.mem.asBytes(&footer.backward_size));
127 hasher.update(std.mem.asBytes(&footer.flags));
128 const hash = hasher.final();
129 if (hash != footer.crc32)
130 return error.WrongChecksum;
131
132 if (!std.mem.eql(u8, &footer.magic, &.{ 'Y', 'Z' }))
133 return error.CorruptInput;
134
135 return 0;
136 }
137 };
138}
2139
3test {140test {
4 _ = @import("xz/stream.zig");141 _ = @import("xz/test.zig");
5}142}
lib/std/compress/xz/block.zig+12-14
...@@ -1,11 +1,10 @@...@@ -1,11 +1,10 @@
1const std = @import("std");1const std = @import("../../std.zig");
2const check = @import("check.zig");
3const lzma = @import("lzma.zig");2const lzma = @import("lzma.zig");
4const multibyte = @import("multibyte.zig");
5const Allocator = std.mem.Allocator;3const Allocator = std.mem.Allocator;
6const Crc32 = std.hash.Crc32;4const Crc32 = std.hash.Crc32;
7const Crc64 = std.hash.crc.Crc64Xz;5const Crc64 = std.hash.crc.Crc64Xz;
8const Sha256 = std.crypto.hash.sha2.Sha256;6const Sha256 = std.crypto.hash.sha2.Sha256;
7const xz = std.compress.xz;
98
10const DecodeError = error{9const DecodeError = error{
11 CorruptInput,10 CorruptInput,
...@@ -16,8 +15,8 @@ const DecodeError = error{...@@ -16,8 +15,8 @@ const DecodeError = error{
16 Overflow,15 Overflow,
17};16};
1817
19pub fn decoder(allocator: Allocator, reader: anytype, check_kind: check.Kind) !Decoder(@TypeOf(reader)) {18pub fn decoder(allocator: Allocator, reader: anytype, check: xz.Check) !Decoder(@TypeOf(reader)) {
20 return Decoder(@TypeOf(reader)).init(allocator, reader, check_kind);19 return Decoder(@TypeOf(reader)).init(allocator, reader, check);
21}20}
2221
23pub fn Decoder(comptime ReaderType: type) type {22pub fn Decoder(comptime ReaderType: type) type {
...@@ -31,17 +30,17 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -31,17 +30,17 @@ pub fn Decoder(comptime ReaderType: type) type {
3130
32 allocator: Allocator,31 allocator: Allocator,
33 inner_reader: ReaderType,32 inner_reader: ReaderType,
34 check_kind: check.Kind,33 check: xz.Check,
35 err: ?Error,34 err: ?Error,
36 accum: lzma.LzAccumBuffer,35 accum: lzma.LzAccumBuffer,
37 lzma_state: lzma.DecoderState,36 lzma_state: lzma.DecoderState,
38 block_count: usize,37 block_count: usize,
3938
40 fn init(allocator: Allocator, in_reader: ReaderType, check_kind: check.Kind) !Self {39 fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {
41 return Self{40 return Self{
42 .allocator = allocator,41 .allocator = allocator,
43 .inner_reader = in_reader,42 .inner_reader = in_reader,
44 .check_kind = check_kind,43 .check = check,
45 .err = null,44 .err = null,
46 .accum = .{},45 .accum = .{},
47 .lzma_state = try lzma.DecoderState.init(allocator),46 .lzma_state = try lzma.DecoderState.init(allocator),
...@@ -116,10 +115,10 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -116,10 +115,10 @@ pub fn Decoder(comptime ReaderType: type) type {
116 return error.Unsupported;115 return error.Unsupported;
117116
118 if (flags.has_packed_size)117 if (flags.has_packed_size)
119 packed_size = try multibyte.readInt(header_reader);118 packed_size = try std.leb.readULEB128(u64, header_reader);
120119
121 if (flags.has_unpacked_size)120 if (flags.has_unpacked_size)
122 unpacked_size = try multibyte.readInt(header_reader);121 unpacked_size = try std.leb.readULEB128(u64, header_reader);
123122
124 const FilterId = enum(u64) {123 const FilterId = enum(u64) {
125 lzma2 = 0x21,124 lzma2 = 0x21,
...@@ -128,7 +127,7 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -128,7 +127,7 @@ pub fn Decoder(comptime ReaderType: type) type {
128127
129 const filter_id = @intToEnum(128 const filter_id = @intToEnum(
130 FilterId,129 FilterId,
131 try multibyte.readInt(header_reader),130 try std.leb.readULEB128(u64, header_reader),
132 );131 );
133132
134 if (@enumToInt(filter_id) >= 0x4000_0000_0000_0000)133 if (@enumToInt(filter_id) >= 0x4000_0000_0000_0000)
...@@ -137,7 +136,7 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -137,7 +136,7 @@ pub fn Decoder(comptime ReaderType: type) type {
137 if (filter_id != .lzma2)136 if (filter_id != .lzma2)
138 return error.Unsupported;137 return error.Unsupported;
139138
140 const properties_size = try multibyte.readInt(header_reader);139 const properties_size = try std.leb.readULEB128(u64, header_reader);
141 if (properties_size != 1)140 if (properties_size != 1)
142 return error.CorruptInput;141 return error.CorruptInput;
143142
...@@ -177,8 +176,7 @@ pub fn Decoder(comptime ReaderType: type) type {...@@ -177,8 +176,7 @@ pub fn Decoder(comptime ReaderType: type) type {
177 return error.CorruptInput;176 return error.CorruptInput;
178 }177 }
179178
180 // Check179 switch (self.check) {
181 switch (self.check_kind) {
182 .none => {},180 .none => {},
183 .crc32 => {181 .crc32 => {
184 const hash_a = Crc32.hash(unpacked_bytes);182 const hash_a = Crc32.hash(unpacked_bytes);
lib/std/compress/xz/check.zig deleted-7
...@@ -1,7 +0,0 @@
1pub const Kind = enum(u4) {
2 none = 0x00,
3 crc32 = 0x01,
4 crc64 = 0x04,
5 sha256 = 0x0A,
6 _,
7};
lib/std/compress/xz/lzma.zig+1-1
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1// Ported from https://github.com/gendx/lzma-rs1// Ported from https://github.com/gendx/lzma-rs
22
3const std = @import("std");3const std = @import("../../std.zig");
4const assert = std.debug.assert;4const assert = std.debug.assert;
5const Allocator = std.mem.Allocator;5const Allocator = std.mem.Allocator;
6const ArrayListUnmanaged = std.ArrayListUnmanaged;6const ArrayListUnmanaged = std.ArrayListUnmanaged;
lib/std/compress/xz/multibyte.zig deleted-23
...@@ -1,23 +0,0 @@
1const Multibyte = packed struct(u8) {
2 value: u7,
3 more: bool,
4};
5
6pub fn readInt(reader: anytype) !u64 {
7 const max_size = 9;
8
9 var chunk = try reader.readStruct(Multibyte);
10 var num: u64 = chunk.value;
11 var i: u6 = 0;
12
13 while (chunk.more) {
14 chunk = try reader.readStruct(Multibyte);
15 i += 1;
16 if (i >= max_size or @bitCast(u8, chunk) == 0x00)
17 return error.CorruptInput;
18
19 num |= @as(u64, chunk.value) << (i * 7);
20 }
21
22 return num;
23}
lib/std/compress/xz/stream.zig deleted-136
...@@ -1,136 +0,0 @@
1const std = @import("std");
2const block = @import("block.zig");
3const check = @import("check.zig");
4const multibyte = @import("multibyte.zig");
5const Allocator = std.mem.Allocator;
6const Crc32 = std.hash.Crc32;
7
8test {
9 _ = @import("stream_test.zig");
10}
11
12const Flags = packed struct(u16) {
13 reserved1: u8,
14 check_kind: check.Kind,
15 reserved2: u4,
16};
17
18pub fn stream(allocator: Allocator, reader: anytype) !Stream(@TypeOf(reader)) {
19 return Stream(@TypeOf(reader)).init(allocator, reader);
20}
21
22pub fn Stream(comptime ReaderType: type) type {
23 return struct {
24 const Self = @This();
25
26 pub const Error = ReaderType.Error || block.Decoder(ReaderType).Error;
27 pub const Reader = std.io.Reader(*Self, Error, read);
28
29 allocator: Allocator,
30 block_decoder: block.Decoder(ReaderType),
31 in_reader: ReaderType,
32
33 fn init(allocator: Allocator, source: ReaderType) !Self {
34 const Header = extern struct {
35 magic: [6]u8,
36 flags: Flags,
37 crc32: u32,
38 };
39
40 const header = try source.readStruct(Header);
41
42 if (!std.mem.eql(u8, &header.magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
43 return error.BadHeader;
44
45 if (header.flags.reserved1 != 0 or header.flags.reserved2 != 0)
46 return error.BadHeader;
47
48 const hash = Crc32.hash(std.mem.asBytes(&header.flags));
49 if (hash != header.crc32)
50 return error.WrongChecksum;
51
52 return Self{
53 .allocator = allocator,
54 .block_decoder = try block.decoder(allocator, source, header.flags.check_kind),
55 .in_reader = source,
56 };
57 }
58
59 pub fn deinit(self: *Self) void {
60 self.block_decoder.deinit();
61 }
62
63 pub fn reader(self: *Self) Reader {
64 return .{ .context = self };
65 }
66
67 pub fn read(self: *Self, buffer: []u8) Error!usize {
68 if (buffer.len == 0)
69 return 0;
70
71 const r = try self.block_decoder.read(buffer);
72 if (r != 0)
73 return r;
74
75 const index_size = blk: {
76 var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
77 hasher.hasher.update(&[1]u8{0x00});
78
79 var counter = std.io.countingReader(hasher.reader());
80 counter.bytes_read += 1;
81
82 const counting_reader = counter.reader();
83
84 const record_count = try multibyte.readInt(counting_reader);
85 if (record_count != self.block_decoder.block_count)
86 return error.CorruptInput;
87
88 var i: usize = 0;
89 while (i < record_count) : (i += 1) {
90 // TODO: validate records
91 _ = try multibyte.readInt(counting_reader);
92 _ = try multibyte.readInt(counting_reader);
93 }
94
95 while (counter.bytes_read % 4 != 0) {
96 if (try counting_reader.readByte() != 0)
97 return error.CorruptInput;
98 }
99
100 const hash_a = hasher.hasher.final();
101 const hash_b = try counting_reader.readIntLittle(u32);
102 if (hash_a != hash_b)
103 return error.WrongChecksum;
104
105 break :blk counter.bytes_read;
106 };
107
108 const Footer = extern struct {
109 crc32: u32,
110 backward_size: u32,
111 flags: Flags,
112 magic: [2]u8,
113 };
114
115 const footer = try self.in_reader.readStruct(Footer);
116 const backward_size = (footer.backward_size + 1) * 4;
117 if (backward_size != index_size)
118 return error.CorruptInput;
119
120 if (footer.flags.reserved1 != 0 or footer.flags.reserved2 != 0)
121 return error.CorruptInput;
122
123 var hasher = Crc32.init();
124 hasher.update(std.mem.asBytes(&footer.backward_size));
125 hasher.update(std.mem.asBytes(&footer.flags));
126 const hash = hasher.final();
127 if (hash != footer.crc32)
128 return error.WrongChecksum;
129
130 if (!std.mem.eql(u8, &footer.magic, &.{ 'Y', 'Z' }))
131 return error.CorruptInput;
132
133 return 0;
134 }
135 };
136}
lib/std/compress/xz/stream_test.zig deleted-80
...@@ -1,80 +0,0 @@
1const std = @import("std");
2const testing = std.testing;
3const stream = @import("stream.zig").stream;
4
5fn decompress(data: []const u8) ![]u8 {
6 var in_stream = std.io.fixedBufferStream(data);
7
8 var xz_stream = try stream(testing.allocator, in_stream.reader());
9 defer xz_stream.deinit();
10
11 return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
12}
13
14fn testReader(data: []const u8, comptime expected: []const u8) !void {
15 const buf = try decompress(data);
16 defer testing.allocator.free(buf);
17
18 try testing.expectEqualSlices(u8, expected, buf);
19}
20
21test "compressed data" {
22 try testReader(@embedFile("testdata/good-0-empty.xz"), "");
23
24 inline for ([_][]const u8{
25 "good-1-check-none.xz",
26 "good-1-check-crc32.xz",
27 "good-1-check-crc64.xz",
28 "good-1-check-sha256.xz",
29 "good-2-lzma2.xz",
30 "good-1-block_header-1.xz",
31 "good-1-block_header-2.xz",
32 "good-1-block_header-3.xz",
33 }) |filename| {
34 try testReader(@embedFile("testdata/" ++ filename),
35 \\Hello
36 \\World!
37 \\
38 );
39 }
40
41 inline for ([_][]const u8{
42 "good-1-lzma2-1.xz",
43 "good-1-lzma2-2.xz",
44 "good-1-lzma2-3.xz",
45 "good-1-lzma2-4.xz",
46 }) |filename| {
47 try testReader(@embedFile("testdata/" ++ filename),
48 \\Lorem ipsum dolor sit amet, consectetur adipisicing
49 \\elit, sed do eiusmod tempor incididunt ut
50 \\labore et dolore magna aliqua. Ut enim
51 \\ad minim veniam, quis nostrud exercitation ullamco
52 \\laboris nisi ut aliquip ex ea commodo
53 \\consequat. Duis aute irure dolor in reprehenderit
54 \\in voluptate velit esse cillum dolore eu
55 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
56 \\non proident, sunt in culpa qui officia
57 \\deserunt mollit anim id est laborum.
58 \\
59 );
60 }
61
62 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
63}
64
65test "unsupported" {
66 inline for ([_][]const u8{
67 "good-1-delta-lzma2.tiff.xz",
68 "good-1-x86-lzma2.xz",
69 "good-1-sparc-lzma2.xz",
70 "good-1-arm64-lzma2-1.xz",
71 "good-1-arm64-lzma2-2.xz",
72 "good-1-3delta-lzma2.xz",
73 "good-1-empty-bcj-lzma2.xz",
74 }) |filename| {
75 try testing.expectError(
76 error.Unsupported,
77 decompress(@embedFile("testdata/" ++ filename)),
78 );
79 }
80}
lib/std/compress/xz/test.zig created+80
...@@ -0,0 +1,80 @@
1const std = @import("../../std.zig");
2const testing = std.testing;
3const xz = std.compress.xz;
4
5fn decompress(data: []const u8) ![]u8 {
6 var in_stream = std.io.fixedBufferStream(data);
7
8 var xz_stream = try xz.decompress(testing.allocator, in_stream.reader());
9 defer xz_stream.deinit();
10
11 return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
12}
13
14fn testReader(data: []const u8, comptime expected: []const u8) !void {
15 const buf = try decompress(data);
16 defer testing.allocator.free(buf);
17
18 try testing.expectEqualSlices(u8, expected, buf);
19}
20
21test "compressed data" {
22 try testReader(@embedFile("testdata/good-0-empty.xz"), "");
23
24 inline for ([_][]const u8{
25 "good-1-check-none.xz",
26 "good-1-check-crc32.xz",
27 "good-1-check-crc64.xz",
28 "good-1-check-sha256.xz",
29 "good-2-lzma2.xz",
30 "good-1-block_header-1.xz",
31 "good-1-block_header-2.xz",
32 "good-1-block_header-3.xz",
33 }) |filename| {
34 try testReader(@embedFile("testdata/" ++ filename),
35 \\Hello
36 \\World!
37 \\
38 );
39 }
40
41 inline for ([_][]const u8{
42 "good-1-lzma2-1.xz",
43 "good-1-lzma2-2.xz",
44 "good-1-lzma2-3.xz",
45 "good-1-lzma2-4.xz",
46 }) |filename| {
47 try testReader(@embedFile("testdata/" ++ filename),
48 \\Lorem ipsum dolor sit amet, consectetur adipisicing
49 \\elit, sed do eiusmod tempor incididunt ut
50 \\labore et dolore magna aliqua. Ut enim
51 \\ad minim veniam, quis nostrud exercitation ullamco
52 \\laboris nisi ut aliquip ex ea commodo
53 \\consequat. Duis aute irure dolor in reprehenderit
54 \\in voluptate velit esse cillum dolore eu
55 \\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
56 \\non proident, sunt in culpa qui officia
57 \\deserunt mollit anim id est laborum.
58 \\
59 );
60 }
61
62 try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
63}
64
65test "unsupported" {
66 inline for ([_][]const u8{
67 "good-1-delta-lzma2.tiff.xz",
68 "good-1-x86-lzma2.xz",
69 "good-1-sparc-lzma2.xz",
70 "good-1-arm64-lzma2-1.xz",
71 "good-1-arm64-lzma2-2.xz",
72 "good-1-3delta-lzma2.xz",
73 "good-1-empty-bcj-lzma2.xz",
74 }) |filename| {
75 try testing.expectError(
76 error.Unsupported,
77 decompress(@embedFile("testdata/" ++ filename)),
78 );
79 }
80}