Commit ea9ded87582a8b9d0ed3afd3360a1d75f0359a5c | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-01-24 15:04:56-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-01-24 15:24:19-07:00
log	ea9ded87582a8b9d0ed3afd3360a1d75f0359a5c
tree	9767d5562df1fd55bcc68472d20c6e5b9733fa46
parent	06ce15e8f719756cc12d928cfdae12be99a9e4c2

std.compress.xz public API cleanup

 * add xz to std.compress
 * prefer importing std.zig by file name, to reduce reliance on the
   standard library being a special case.
 * extract some types from inside generic functions. These types are the
   same regardless of the generic parameters.
 * expose some more types in the std.compress.xz namespace.
 * rename xz.stream to xz.decompress
 * rename check.Kind to Check
 * use std.leb for LEB instead of a redundant implementation

9 files changed, 234 insertions(+), 263 deletions(-)

lib
- std
  - compress.zig
  - compress
    xz.zig
    xz
    block.zig
    check.zig
    lzma.zig
    multibyte.zig
    stream.zig
    stream_test.zig
    test.zig

lib/std/compress.zig+2

...	...	@@ -3,6 +3,7 @@ const std = @import("std.zig");
3	3	pub const deflate = @import("compress/deflate.zig");
4	4	pub const gzip = @import("compress/gzip.zig");
5	5	pub const zlib = @import("compress/zlib.zig");
	6	pub const xz = @import("compress/xz.zig");
6	7
7	8	pub fn HashedReader(
8	9	comptime ReaderType: anytype,
...	...	@@ -38,4 +39,5 @@ test {
38	39	_ = deflate;
39	40	_ = gzip;
40	41	_ = zlib;
	42	_ = xz;
41	43	}

lib/std/compress/xz.zig+139-2

...	...	@@ -1,5 +1,142 @@
1		pub usingnamespace @import("xz/stream.zig");
	1	const std = @import("std");
	2	const block = @import("xz/block.zig");
	3	const Allocator = std.mem.Allocator;
	4	const Crc32 = std.hash.Crc32;
	5
	6	pub const Flags = packed struct(u16) {
	7	reserved1: u8,
	8	check_kind: Check,
	9	reserved2: u4,
	10	};
	11
	12	pub const Header = extern struct {
	13	magic: [6]u8,
	14	flags: Flags,
	15	crc32: u32,
	16	};
	17
	18	pub const Footer = extern struct {
	19	crc32: u32,
	20	backward_size: u32,
	21	flags: Flags,
	22	magic: [2]u8,
	23	};
	24
	25	pub const Check = enum(u4) {
	26	none = 0x00,
	27	crc32 = 0x01,
	28	crc64 = 0x04,
	29	sha256 = 0x0A,
	30	_,
	31	};
	32
	33	pub fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
	34	return Decompress(@TypeOf(reader)).init(allocator, reader);
	35	}
	36
	37	pub fn Decompress(comptime ReaderType: type) type {
	38	return struct {
	39	const Self = @This();
	40
	41	pub const Error = ReaderType.Error \|\| block.Decoder(ReaderType).Error;
	42	pub const Reader = std.io.Reader(*Self, Error, read);
	43
	44	allocator: Allocator,
	45	block_decoder: block.Decoder(ReaderType),
	46	in_reader: ReaderType,
	47
	48	fn init(allocator: Allocator, source: ReaderType) !Self {
	49	const header = try source.readStruct(Header);
	50
	51	if (!std.mem.eql(u8, &header.magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
	52	return error.BadHeader;
	53
	54	if (header.flags.reserved1 != 0 or header.flags.reserved2 != 0)
	55	return error.BadHeader;
	56
	57	const hash = Crc32.hash(std.mem.asBytes(&header.flags));
	58	if (hash != header.crc32)
	59	return error.WrongChecksum;
	60
	61	return Self{
	62	.allocator = allocator,
	63	.block_decoder = try block.decoder(allocator, source, header.flags.check_kind),
	64	.in_reader = source,
	65	};
	66	}
	67
	68	pub fn deinit(self: *Self) void {
	69	self.block_decoder.deinit();
	70	}
	71
	72	pub fn reader(self: *Self) Reader {
	73	return .{ .context = self };
	74	}
	75
	76	pub fn read(self: *Self, buffer: []u8) Error!usize {
	77	if (buffer.len == 0)
	78	return 0;
	79
	80	const r = try self.block_decoder.read(buffer);
	81	if (r != 0)
	82	return r;
	83
	84	const index_size = blk: {
	85	var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
	86	hasher.hasher.update(&[1]u8{0x00});
	87
	88	var counter = std.io.countingReader(hasher.reader());
	89	counter.bytes_read += 1;
	90
	91	const counting_reader = counter.reader();
	92
	93	const record_count = try std.leb.readULEB128(u64, counting_reader);
	94	if (record_count != self.block_decoder.block_count)
	95	return error.CorruptInput;
	96
	97	var i: usize = 0;
	98	while (i < record_count) : (i += 1) {
	99	// TODO: validate records
	100	_ = try std.leb.readULEB128(u64, counting_reader);
	101	_ = try std.leb.readULEB128(u64, counting_reader);
	102	}
	103
	104	while (counter.bytes_read % 4 != 0) {
	105	if (try counting_reader.readByte() != 0)
	106	return error.CorruptInput;
	107	}
	108
	109	const hash_a = hasher.hasher.final();
	110	const hash_b = try counting_reader.readIntLittle(u32);
	111	if (hash_a != hash_b)
	112	return error.WrongChecksum;
	113
	114	break :blk counter.bytes_read;
	115	};
	116
	117	const footer = try self.in_reader.readStruct(Footer);
	118	const backward_size = (footer.backward_size + 1) * 4;
	119	if (backward_size != index_size)
	120	return error.CorruptInput;
	121
	122	if (footer.flags.reserved1 != 0 or footer.flags.reserved2 != 0)
	123	return error.CorruptInput;
	124
	125	var hasher = Crc32.init();
	126	hasher.update(std.mem.asBytes(&footer.backward_size));
	127	hasher.update(std.mem.asBytes(&footer.flags));
	128	const hash = hasher.final();
	129	if (hash != footer.crc32)
	130	return error.WrongChecksum;
	131
	132	if (!std.mem.eql(u8, &footer.magic, &.{ 'Y', 'Z' }))
	133	return error.CorruptInput;
	134
	135	return 0;
	136	}
	137	};
	138	}
2	139
3	140	test {
4		_ = @import("xz/stream.zig");
	141	_ = @import("xz/test.zig");
5	142	}

lib/std/compress/xz/block.zig+12-14

...	...	@@ -1,11 +1,10 @@
1		const std = @import("std");
2		const check = @import("check.zig");
	1	const std = @import("../../std.zig");
3	2	const lzma = @import("lzma.zig");
4		const multibyte = @import("multibyte.zig");
5	3	const Allocator = std.mem.Allocator;
6	4	const Crc32 = std.hash.Crc32;
7	5	const Crc64 = std.hash.crc.Crc64Xz;
8	6	const Sha256 = std.crypto.hash.sha2.Sha256;
	7	const xz = std.compress.xz;
9	8
10	9	const DecodeError = error{
11	10	CorruptInput,
...	...	@@ -16,8 +15,8 @@ const DecodeError = error{
16	15	Overflow,
17	16	};
18	17
19		pub fn decoder(allocator: Allocator, reader: anytype, check_kind: check.Kind) !Decoder(@TypeOf(reader)) {
20		return Decoder(@TypeOf(reader)).init(allocator, reader, check_kind);
	18	pub fn decoder(allocator: Allocator, reader: anytype, check: xz.Check) !Decoder(@TypeOf(reader)) {
	19	return Decoder(@TypeOf(reader)).init(allocator, reader, check);
21	20	}
22	21
23	22	pub fn Decoder(comptime ReaderType: type) type {
...	...	@@ -31,17 +30,17 @@ pub fn Decoder(comptime ReaderType: type) type {
31	30
32	31	allocator: Allocator,
33	32	inner_reader: ReaderType,
34		check_kind: check.Kind,
	33	check: xz.Check,
35	34	err: ?Error,
36	35	accum: lzma.LzAccumBuffer,
37	36	lzma_state: lzma.DecoderState,
38	37	block_count: usize,
39	38
40		fn init(allocator: Allocator, in_reader: ReaderType, check_kind: check.Kind) !Self {
	39	fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {
41	40	return Self{
42	41	.allocator = allocator,
43	42	.inner_reader = in_reader,
44		.check_kind = check_kind,
	43	.check = check,
45	44	.err = null,
46	45	.accum = .{},
47	46	.lzma_state = try lzma.DecoderState.init(allocator),
...	...	@@ -116,10 +115,10 @@ pub fn Decoder(comptime ReaderType: type) type {
116	115	return error.Unsupported;
117	116
118	117	if (flags.has_packed_size)
119		packed_size = try multibyte.readInt(header_reader);
	118	packed_size = try std.leb.readULEB128(u64, header_reader);
120	119
121	120	if (flags.has_unpacked_size)
122		unpacked_size = try multibyte.readInt(header_reader);
	121	unpacked_size = try std.leb.readULEB128(u64, header_reader);
123	122
124	123	const FilterId = enum(u64) {
125	124	lzma2 = 0x21,
...	...	@@ -128,7 +127,7 @@ pub fn Decoder(comptime ReaderType: type) type {
128	127
129	128	const filter_id = @intToEnum(
130	129	FilterId,
131		try multibyte.readInt(header_reader),
	130	try std.leb.readULEB128(u64, header_reader),
132	131	);
133	132
134	133	if (@enumToInt(filter_id) >= 0x4000_0000_0000_0000)
...	...	@@ -137,7 +136,7 @@ pub fn Decoder(comptime ReaderType: type) type {
137	136	if (filter_id != .lzma2)
138	137	return error.Unsupported;
139	138
140		const properties_size = try multibyte.readInt(header_reader);
	139	const properties_size = try std.leb.readULEB128(u64, header_reader);
141	140	if (properties_size != 1)
142	141	return error.CorruptInput;
143	142
...	...	@@ -177,8 +176,7 @@ pub fn Decoder(comptime ReaderType: type) type {
177	176	return error.CorruptInput;
178	177	}
179	178
180		// Check
181		switch (self.check_kind) {
	179	switch (self.check) {
182	180	.none => {},
183	181	.crc32 => {
184	182	const hash_a = Crc32.hash(unpacked_bytes);

lib/std/compress/xz/check.zig deleted-7

...	...	@@ -1,7 +0,0 @@
1		pub const Kind = enum(u4) {
2		none = 0x00,
3		crc32 = 0x01,
4		crc64 = 0x04,
5		sha256 = 0x0A,
6		_,
7		};

lib/std/compress/xz/lzma.zig+1-1

...	...	@@ -1,6 +1,6 @@
1	1	// Ported from https://github.com/gendx/lzma-rs
2	2
3		const std = @import("std");
	3	const std = @import("../../std.zig");
4	4	const assert = std.debug.assert;
5	5	const Allocator = std.mem.Allocator;
6	6	const ArrayListUnmanaged = std.ArrayListUnmanaged;

lib/std/compress/xz/multibyte.zig deleted-23

...	...	@@ -1,23 +0,0 @@
1		const Multibyte = packed struct(u8) {
2		value: u7,
3		more: bool,
4		};
5
6		pub fn readInt(reader: anytype) !u64 {
7		const max_size = 9;
8
9		var chunk = try reader.readStruct(Multibyte);
10		var num: u64 = chunk.value;
11		var i: u6 = 0;
12
13		while (chunk.more) {
14		chunk = try reader.readStruct(Multibyte);
15		i += 1;
16		if (i >= max_size or @bitCast(u8, chunk) == 0x00)
17		return error.CorruptInput;
18
19		num \|= @as(u64, chunk.value) << (i * 7);
20		}
21
22		return num;
23		}

lib/std/compress/xz/stream.zig deleted-136

...	...	@@ -1,136 +0,0 @@
1		const std = @import("std");
2		const block = @import("block.zig");
3		const check = @import("check.zig");
4		const multibyte = @import("multibyte.zig");
5		const Allocator = std.mem.Allocator;
6		const Crc32 = std.hash.Crc32;
7
8		test {
9		_ = @import("stream_test.zig");
10		}
11
12		const Flags = packed struct(u16) {
13		reserved1: u8,
14		check_kind: check.Kind,
15		reserved2: u4,
16		};
17
18		pub fn stream(allocator: Allocator, reader: anytype) !Stream(@TypeOf(reader)) {
19		return Stream(@TypeOf(reader)).init(allocator, reader);
20		}
21
22		pub fn Stream(comptime ReaderType: type) type {
23		return struct {
24		const Self = @This();
25
26		pub const Error = ReaderType.Error \|\| block.Decoder(ReaderType).Error;
27		pub const Reader = std.io.Reader(*Self, Error, read);
28
29		allocator: Allocator,
30		block_decoder: block.Decoder(ReaderType),
31		in_reader: ReaderType,
32
33		fn init(allocator: Allocator, source: ReaderType) !Self {
34		const Header = extern struct {
35		magic: [6]u8,
36		flags: Flags,
37		crc32: u32,
38		};
39
40		const header = try source.readStruct(Header);
41
42		if (!std.mem.eql(u8, &header.magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
43		return error.BadHeader;
44
45		if (header.flags.reserved1 != 0 or header.flags.reserved2 != 0)
46		return error.BadHeader;
47
48		const hash = Crc32.hash(std.mem.asBytes(&header.flags));
49		if (hash != header.crc32)
50		return error.WrongChecksum;
51
52		return Self{
53		.allocator = allocator,
54		.block_decoder = try block.decoder(allocator, source, header.flags.check_kind),
55		.in_reader = source,
56		};
57		}
58
59		pub fn deinit(self: *Self) void {
60		self.block_decoder.deinit();
61		}
62
63		pub fn reader(self: *Self) Reader {
64		return .{ .context = self };
65		}
66
67		pub fn read(self: *Self, buffer: []u8) Error!usize {
68		if (buffer.len == 0)
69		return 0;
70
71		const r = try self.block_decoder.read(buffer);
72		if (r != 0)
73		return r;
74
75		const index_size = blk: {
76		var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
77		hasher.hasher.update(&[1]u8{0x00});
78
79		var counter = std.io.countingReader(hasher.reader());
80		counter.bytes_read += 1;
81
82		const counting_reader = counter.reader();
83
84		const record_count = try multibyte.readInt(counting_reader);
85		if (record_count != self.block_decoder.block_count)
86		return error.CorruptInput;
87
88		var i: usize = 0;
89		while (i < record_count) : (i += 1) {
90		// TODO: validate records
91		_ = try multibyte.readInt(counting_reader);
92		_ = try multibyte.readInt(counting_reader);
93		}
94
95		while (counter.bytes_read % 4 != 0) {
96		if (try counting_reader.readByte() != 0)
97		return error.CorruptInput;
98		}
99
100		const hash_a = hasher.hasher.final();
101		const hash_b = try counting_reader.readIntLittle(u32);
102		if (hash_a != hash_b)
103		return error.WrongChecksum;
104
105		break :blk counter.bytes_read;
106		};
107
108		const Footer = extern struct {
109		crc32: u32,
110		backward_size: u32,
111		flags: Flags,
112		magic: [2]u8,
113		};
114
115		const footer = try self.in_reader.readStruct(Footer);
116		const backward_size = (footer.backward_size + 1) * 4;
117		if (backward_size != index_size)
118		return error.CorruptInput;
119
120		if (footer.flags.reserved1 != 0 or footer.flags.reserved2 != 0)
121		return error.CorruptInput;
122
123		var hasher = Crc32.init();
124		hasher.update(std.mem.asBytes(&footer.backward_size));
125		hasher.update(std.mem.asBytes(&footer.flags));
126		const hash = hasher.final();
127		if (hash != footer.crc32)
128		return error.WrongChecksum;
129
130		if (!std.mem.eql(u8, &footer.magic, &.{ 'Y', 'Z' }))
131		return error.CorruptInput;
132
133		return 0;
134		}
135		};
136		}

lib/std/compress/xz/stream_test.zig deleted-80

...	...	@@ -1,80 +0,0 @@
1		const std = @import("std");
2		const testing = std.testing;
3		const stream = @import("stream.zig").stream;
4
5		fn decompress(data: []const u8) ![]u8 {
6		var in_stream = std.io.fixedBufferStream(data);
7
8		var xz_stream = try stream(testing.allocator, in_stream.reader());
9		defer xz_stream.deinit();
10
11		return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
12		}
13
14		fn testReader(data: []const u8, comptime expected: []const u8) !void {
15		const buf = try decompress(data);
16		defer testing.allocator.free(buf);
17
18		try testing.expectEqualSlices(u8, expected, buf);
19		}
20
21		test "compressed data" {
22		try testReader(@embedFile("testdata/good-0-empty.xz"), "");
23
24		inline for ([_][]const u8{
25		"good-1-check-none.xz",
26		"good-1-check-crc32.xz",
27		"good-1-check-crc64.xz",
28		"good-1-check-sha256.xz",
29		"good-2-lzma2.xz",
30		"good-1-block_header-1.xz",
31		"good-1-block_header-2.xz",
32		"good-1-block_header-3.xz",
33		}) \|filename\| {
34		try testReader(@embedFile("testdata/" ++ filename),
35		\\Hello
36		\\World!
37		\\
38		);
39		}
40
41		inline for ([_][]const u8{
42		"good-1-lzma2-1.xz",
43		"good-1-lzma2-2.xz",
44		"good-1-lzma2-3.xz",
45		"good-1-lzma2-4.xz",
46		}) \|filename\| {
47		try testReader(@embedFile("testdata/" ++ filename),
48		\\Lorem ipsum dolor sit amet, consectetur adipisicing
49		\\elit, sed do eiusmod tempor incididunt ut
50		\\labore et dolore magna aliqua. Ut enim
51		\\ad minim veniam, quis nostrud exercitation ullamco
52		\\laboris nisi ut aliquip ex ea commodo
53		\\consequat. Duis aute irure dolor in reprehenderit
54		\\in voluptate velit esse cillum dolore eu
55		\\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
56		\\non proident, sunt in culpa qui officia
57		\\deserunt mollit anim id est laborum.
58		\\
59		);
60		}
61
62		try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
63		}
64
65		test "unsupported" {
66		inline for ([_][]const u8{
67		"good-1-delta-lzma2.tiff.xz",
68		"good-1-x86-lzma2.xz",
69		"good-1-sparc-lzma2.xz",
70		"good-1-arm64-lzma2-1.xz",
71		"good-1-arm64-lzma2-2.xz",
72		"good-1-3delta-lzma2.xz",
73		"good-1-empty-bcj-lzma2.xz",
74		}) \|filename\| {
75		try testing.expectError(
76		error.Unsupported,
77		decompress(@embedFile("testdata/" ++ filename)),
78		);
79		}
80		}

lib/std/compress/xz/test.zig created+80

...	...	@@ -0,0 +1,80 @@
	1	const std = @import("../../std.zig");
	2	const testing = std.testing;
	3	const xz = std.compress.xz;
	4
	5	fn decompress(data: []const u8) ![]u8 {
	6	var in_stream = std.io.fixedBufferStream(data);
	7
	8	var xz_stream = try xz.decompress(testing.allocator, in_stream.reader());
	9	defer xz_stream.deinit();
	10
	11	return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
	12	}
	13
	14	fn testReader(data: []const u8, comptime expected: []const u8) !void {
	15	const buf = try decompress(data);
	16	defer testing.allocator.free(buf);
	17
	18	try testing.expectEqualSlices(u8, expected, buf);
	19	}
	20
	21	test "compressed data" {
	22	try testReader(@embedFile("testdata/good-0-empty.xz"), "");
	23
	24	inline for ([_][]const u8{
	25	"good-1-check-none.xz",
	26	"good-1-check-crc32.xz",
	27	"good-1-check-crc64.xz",
	28	"good-1-check-sha256.xz",
	29	"good-2-lzma2.xz",
	30	"good-1-block_header-1.xz",
	31	"good-1-block_header-2.xz",
	32	"good-1-block_header-3.xz",
	33	}) \|filename\| {
	34	try testReader(@embedFile("testdata/" ++ filename),
	35	\\Hello
	36	\\World!
	37	\\
	38	);
	39	}
	40
	41	inline for ([_][]const u8{
	42	"good-1-lzma2-1.xz",
	43	"good-1-lzma2-2.xz",
	44	"good-1-lzma2-3.xz",
	45	"good-1-lzma2-4.xz",
	46	}) \|filename\| {
	47	try testReader(@embedFile("testdata/" ++ filename),
	48	\\Lorem ipsum dolor sit amet, consectetur adipisicing
	49	\\elit, sed do eiusmod tempor incididunt ut
	50	\\labore et dolore magna aliqua. Ut enim
	51	\\ad minim veniam, quis nostrud exercitation ullamco
	52	\\laboris nisi ut aliquip ex ea commodo
	53	\\consequat. Duis aute irure dolor in reprehenderit
	54	\\in voluptate velit esse cillum dolore eu
	55	\\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
	56	\\non proident, sunt in culpa qui officia
	57	\\deserunt mollit anim id est laborum.
	58	\\
	59	);
	60	}
	61
	62	try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
	63	}
	64
	65	test "unsupported" {
	66	inline for ([_][]const u8{
	67	"good-1-delta-lzma2.tiff.xz",
	68	"good-1-x86-lzma2.xz",
	69	"good-1-sparc-lzma2.xz",
	70	"good-1-arm64-lzma2-1.xz",
	71	"good-1-arm64-lzma2-2.xz",
	72	"good-1-3delta-lzma2.xz",
	73	"good-1-empty-bcj-lzma2.xz",
	74	}) \|filename\| {
	75	try testing.expectError(
	76	error.Unsupported,
	77	decompress(@embedFile("testdata/" ++ filename)),
	78	);
	79	}
	80	}