Commit 96a55f6ce86dc2e25c275ee3211b2cde0e3d92ab | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-01-25 20:31:55-05:00
committer	GitHub <noreply@github.com> 2023-01-25 20:31:55-05:00
log	96a55f6ce86dc2e25c275ee3211b2cde0e3d92ab
tree	503fbb57b3d3855f67751429e27a1ef199fa4afe
parent	fcef728b9bff885c50ad06cbb1e87fb27cb43f62
parent	d0dedefde97aef61db31de6f0cd66846082720d6
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14434 from FnControlOption/xz

Add xz decoder

closes #14300
closes #2851

32 files changed, 1230 insertions(+), 16 deletions(-)

build.zig+2

...	...	@@ -122,6 +122,8 @@ pub fn build(b: *Builder) !void {
122	122	"compress-gettysburg.txt",
123	123	"compress-pi.txt",
124	124	"rfc1951.txt",
	125	// exclude files from lib/std/compress/xz/testdata
	126	".xz",
125	127	// exclude files from lib/std/tz/
126	128	".tzif",
127	129	// others

lib/std/compress.zig+2

...	...	@@ -3,6 +3,7 @@ const std = @import("std.zig");
3	3	pub const deflate = @import("compress/deflate.zig");
4	4	pub const gzip = @import("compress/gzip.zig");
5	5	pub const zlib = @import("compress/zlib.zig");
	6	pub const xz = @import("compress/xz.zig");
6	7
7	8	pub fn HashedReader(
8	9	comptime ReaderType: anytype,
...	...	@@ -38,4 +39,5 @@ test {
38	39	_ = deflate;
39	40	_ = gzip;
40	41	_ = zlib;
	42	_ = xz;
41	43	}

lib/std/compress/gzip.zig+5-8

...	...	@@ -1,7 +1,7 @@
1	1	//
2	2	// Decompressor for GZIP data streams (RFC1952)
3	3
4		const std = @import("std");
	4	const std = @import("../std.zig");
5	5	const io = std.io;
6	6	const fs = std.fs;
7	7	const testing = std.testing;
...	...	@@ -17,10 +17,7 @@ const FCOMMENT = 1 << 4;
17	17
18	18	const max_string_len = 1024;
19	19
20		/// TODO: the fully qualified namespace to this declaration is
21		/// std.compress.gzip.GzipStream which has a redundant "gzip" in the name.
22		/// Instead, it should be `std.compress.gzip.Stream`.
23		pub fn GzipStream(comptime ReaderType: type) type {
	20	pub fn Decompress(comptime ReaderType: type) type {
24	21	return struct {
25	22	const Self = @This();
26	23
...	...	@@ -154,14 +151,14 @@ pub fn GzipStream(comptime ReaderType: type) type {
154	151	};
155	152	}
156	153
157		pub fn gzipStream(allocator: mem.Allocator, reader: anytype) !GzipStream(@TypeOf(reader)) {
158		return GzipStream(@TypeOf(reader)).init(allocator, reader);
	154	pub fn decompress(allocator: mem.Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
	155	return Decompress(@TypeOf(reader)).init(allocator, reader);
159	156	}
160	157
161	158	fn testReader(data: []const u8, comptime expected: []const u8) !void {
162	159	var in_stream = io.fixedBufferStream(data);
163	160
164		var gzip_stream = try gzipStream(testing.allocator, in_stream.reader());
	161	var gzip_stream = try decompress(testing.allocator, in_stream.reader());
165	162	defer gzip_stream.deinit();
166	163
167	164	// Read and decompress the whole file

lib/std/compress/xz.zig created+145

...	...	@@ -0,0 +1,145 @@
	1	const std = @import("std");
	2	const block = @import("xz/block.zig");
	3	const Allocator = std.mem.Allocator;
	4	const Crc32 = std.hash.Crc32;
	5
	6	pub const Check = enum(u4) {
	7	none = 0x00,
	8	crc32 = 0x01,
	9	crc64 = 0x04,
	10	sha256 = 0x0A,
	11	_,
	12	};
	13
	14	fn readStreamFlags(reader: anytype, check: *Check) !void {
	15	var bit_reader = std.io.bitReader(.Little, reader);
	16
	17	const reserved1 = try bit_reader.readBitsNoEof(u8, 8);
	18	if (reserved1 != 0)
	19	return error.CorruptInput;
	20
	21	check.* = @intToEnum(Check, try bit_reader.readBitsNoEof(u4, 4));
	22
	23	const reserved2 = try bit_reader.readBitsNoEof(u4, 4);
	24	if (reserved2 != 0)
	25	return error.CorruptInput;
	26	}
	27
	28	pub fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
	29	return Decompress(@TypeOf(reader)).init(allocator, reader);
	30	}
	31
	32	pub fn Decompress(comptime ReaderType: type) type {
	33	return struct {
	34	const Self = @This();
	35
	36	pub const Error = ReaderType.Error \|\| block.Decoder(ReaderType).Error;
	37	pub const Reader = std.io.Reader(*Self, Error, read);
	38
	39	allocator: Allocator,
	40	block_decoder: block.Decoder(ReaderType),
	41	in_reader: ReaderType,
	42
	43	fn init(allocator: Allocator, source: ReaderType) !Self {
	44	const magic = try source.readBytesNoEof(6);
	45	if (!std.mem.eql(u8, &magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
	46	return error.BadHeader;
	47
	48	var check: Check = undefined;
	49	const hash_a = blk: {
	50	var hasher = std.compress.hashedReader(source, Crc32.init());
	51	try readStreamFlags(hasher.reader(), &check);
	52	break :blk hasher.hasher.final();
	53	};
	54
	55	const hash_b = try source.readIntLittle(u32);
	56	if (hash_a != hash_b)
	57	return error.WrongChecksum;
	58
	59	return Self{
	60	.allocator = allocator,
	61	.block_decoder = try block.decoder(allocator, source, check),
	62	.in_reader = source,
	63	};
	64	}
	65
	66	pub fn deinit(self: *Self) void {
	67	self.block_decoder.deinit();
	68	}
	69
	70	pub fn reader(self: *Self) Reader {
	71	return .{ .context = self };
	72	}
	73
	74	pub fn read(self: *Self, buffer: []u8) Error!usize {
	75	if (buffer.len == 0)
	76	return 0;
	77
	78	const r = try self.block_decoder.read(buffer);
	79	if (r != 0)
	80	return r;
	81
	82	const index_size = blk: {
	83	var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
	84	hasher.hasher.update(&[1]u8{0x00});
	85
	86	var counter = std.io.countingReader(hasher.reader());
	87	counter.bytes_read += 1;
	88
	89	const counting_reader = counter.reader();
	90
	91	const record_count = try std.leb.readULEB128(u64, counting_reader);
	92	if (record_count != self.block_decoder.block_count)
	93	return error.CorruptInput;
	94
	95	var i: usize = 0;
	96	while (i < record_count) : (i += 1) {
	97	// TODO: validate records
	98	_ = try std.leb.readULEB128(u64, counting_reader);
	99	_ = try std.leb.readULEB128(u64, counting_reader);
	100	}
	101
	102	while (counter.bytes_read % 4 != 0) {
	103	if (try counting_reader.readByte() != 0)
	104	return error.CorruptInput;
	105	}
	106
	107	const hash_a = hasher.hasher.final();
	108	const hash_b = try counting_reader.readIntLittle(u32);
	109	if (hash_a != hash_b)
	110	return error.WrongChecksum;
	111
	112	break :blk counter.bytes_read;
	113	};
	114
	115	const hash_a = try self.in_reader.readIntLittle(u32);
	116
	117	const hash_b = blk: {
	118	var hasher = std.compress.hashedReader(self.in_reader, Crc32.init());
	119	const hashed_reader = hasher.reader();
	120
	121	const backward_size = (try hashed_reader.readIntLittle(u32) + 1) * 4;
	122	if (backward_size != index_size)
	123	return error.CorruptInput;
	124
	125	var check: Check = undefined;
	126	try readStreamFlags(hashed_reader, &check);
	127
	128	break :blk hasher.hasher.final();
	129	};
	130
	131	if (hash_a != hash_b)
	132	return error.WrongChecksum;
	133
	134	const magic = try self.in_reader.readBytesNoEof(2);
	135	if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
	136	return error.CorruptInput;
	137
	138	return 0;
	139	}
	140	};
	141	}
	142
	143	test {
	144	_ = @import("xz/test.zig");
	145	}

lib/std/compress/xz/block.zig created+317

...	...	@@ -0,0 +1,317 @@
	1	const std = @import("../../std.zig");
	2	const lzma = @import("lzma.zig");
	3	const Allocator = std.mem.Allocator;
	4	const Crc32 = std.hash.Crc32;
	5	const Crc64 = std.hash.crc.Crc64Xz;
	6	const Sha256 = std.crypto.hash.sha2.Sha256;
	7	const xz = std.compress.xz;
	8
	9	const DecodeError = error{
	10	CorruptInput,
	11	EndOfStream,
	12	EndOfStreamWithNoError,
	13	WrongChecksum,
	14	Unsupported,
	15	Overflow,
	16	};
	17
	18	pub fn decoder(allocator: Allocator, reader: anytype, check: xz.Check) !Decoder(@TypeOf(reader)) {
	19	return Decoder(@TypeOf(reader)).init(allocator, reader, check);
	20	}
	21
	22	pub fn Decoder(comptime ReaderType: type) type {
	23	return struct {
	24	const Self = @This();
	25	pub const Error =
	26	ReaderType.Error \|\|
	27	DecodeError \|\|
	28	Allocator.Error;
	29	pub const Reader = std.io.Reader(*Self, Error, read);
	30
	31	allocator: Allocator,
	32	inner_reader: ReaderType,
	33	check: xz.Check,
	34	err: ?Error,
	35	accum: lzma.LzAccumBuffer,
	36	lzma_state: lzma.DecoderState,
	37	block_count: usize,
	38
	39	fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {
	40	return Self{
	41	.allocator = allocator,
	42	.inner_reader = in_reader,
	43	.check = check,
	44	.err = null,
	45	.accum = .{},
	46	.lzma_state = try lzma.DecoderState.init(allocator),
	47	.block_count = 0,
	48	};
	49	}
	50
	51	pub fn deinit(self: *Self) void {
	52	self.accum.deinit(self.allocator);
	53	self.lzma_state.deinit(self.allocator);
	54	}
	55
	56	pub fn reader(self: *Self) Reader {
	57	return .{ .context = self };
	58	}
	59
	60	pub fn read(self: *Self, output: []u8) Error!usize {
	61	while (true) {
	62	if (self.accum.to_read.items.len > 0) {
	63	const n = self.accum.read(output);
	64	if (self.accum.to_read.items.len == 0 and self.err != null) {
	65	if (self.err.? == DecodeError.EndOfStreamWithNoError) {
	66	return n;
	67	}
	68	return self.err.?;
	69	}
	70	return n;
	71	}
	72	if (self.err != null) {
	73	if (self.err.? == DecodeError.EndOfStreamWithNoError) {
	74	return 0;
	75	}
	76	return self.err.?;
	77	}
	78	self.readBlock() catch \|e\| {
	79	self.err = e;
	80	if (self.accum.to_read.items.len == 0) {
	81	try self.accum.reset(self.allocator);
	82	}
	83	};
	84	}
	85	}
	86
	87	fn readBlock(self: *Self) Error!void {
	88	const unpacked_pos = self.accum.to_read.items.len;
	89
	90	var block_counter = std.io.countingReader(self.inner_reader);
	91	const block_reader = block_counter.reader();
	92
	93	var packed_size: ?u64 = null;
	94	var unpacked_size: ?u64 = null;
	95
	96	// Block Header
	97	{
	98	var header_hasher = std.compress.hashedReader(block_reader, Crc32.init());
	99	const header_reader = header_hasher.reader();
	100
	101	const header_size = try header_reader.readByte() * 4;
	102	if (header_size == 0)
	103	return error.EndOfStreamWithNoError;
	104
	105	const Flags = packed struct(u8) {
	106	last_filter_index: u2,
	107	reserved: u4,
	108	has_packed_size: bool,
	109	has_unpacked_size: bool,
	110	};
	111
	112	const flags = @bitCast(Flags, try header_reader.readByte());
	113	const filter_count = @as(u3, flags.last_filter_index) + 1;
	114	if (filter_count > 1)
	115	return error.Unsupported;
	116
	117	if (flags.has_packed_size)
	118	packed_size = try std.leb.readULEB128(u64, header_reader);
	119
	120	if (flags.has_unpacked_size)
	121	unpacked_size = try std.leb.readULEB128(u64, header_reader);
	122
	123	const FilterId = enum(u64) {
	124	lzma2 = 0x21,
	125	_,
	126	};
	127
	128	const filter_id = @intToEnum(
	129	FilterId,
	130	try std.leb.readULEB128(u64, header_reader),
	131	);
	132
	133	if (@enumToInt(filter_id) >= 0x4000_0000_0000_0000)
	134	return error.CorruptInput;
	135
	136	if (filter_id != .lzma2)
	137	return error.Unsupported;
	138
	139	const properties_size = try std.leb.readULEB128(u64, header_reader);
	140	if (properties_size != 1)
	141	return error.CorruptInput;
	142
	143	// TODO: use filter properties
	144	_ = try header_reader.readByte();
	145
	146	while (block_counter.bytes_read != header_size) {
	147	if (try header_reader.readByte() != 0)
	148	return error.CorruptInput;
	149	}
	150
	151	const hash_a = header_hasher.hasher.final();
	152	const hash_b = try header_reader.readIntLittle(u32);
	153	if (hash_a != hash_b)
	154	return error.WrongChecksum;
	155	}
	156
	157	// Compressed Data
	158	var packed_counter = std.io.countingReader(block_reader);
	159	const packed_reader = packed_counter.reader();
	160	while (try self.readLzma2Chunk(packed_reader)) {}
	161
	162	if (packed_size) \|s\| {
	163	if (s != packed_counter.bytes_read)
	164	return error.CorruptInput;
	165	}
	166
	167	const unpacked_bytes = self.accum.to_read.items[unpacked_pos..];
	168	if (unpacked_size) \|s\| {
	169	if (s != unpacked_bytes.len)
	170	return error.CorruptInput;
	171	}
	172
	173	// Block Padding
	174	while (block_counter.bytes_read % 4 != 0) {
	175	if (try block_reader.readByte() != 0)
	176	return error.CorruptInput;
	177	}
	178
	179	switch (self.check) {
	180	.none => {},
	181	.crc32 => {
	182	const hash_a = Crc32.hash(unpacked_bytes);
	183	const hash_b = try self.inner_reader.readIntLittle(u32);
	184	if (hash_a != hash_b)
	185	return error.WrongChecksum;
	186	},
	187	.crc64 => {
	188	const hash_a = Crc64.hash(unpacked_bytes);
	189	const hash_b = try self.inner_reader.readIntLittle(u64);
	190	if (hash_a != hash_b)
	191	return error.WrongChecksum;
	192	},
	193	.sha256 => {
	194	var hash_a: [Sha256.digest_length]u8 = undefined;
	195	Sha256.hash(unpacked_bytes, &hash_a, .{});
	196
	197	var hash_b: [Sha256.digest_length]u8 = undefined;
	198	try self.inner_reader.readNoEof(&hash_b);
	199
	200	if (!std.mem.eql(u8, &hash_a, &hash_b))
	201	return error.WrongChecksum;
	202	},
	203	else => return error.Unsupported,
	204	}
	205
	206	self.block_count += 1;
	207	}
	208
	209	fn readLzma2Chunk(self: *Self, packed_reader: anytype) Error!bool {
	210	const status = try packed_reader.readByte();
	211	switch (status) {
	212	0 => {
	213	try self.accum.reset(self.allocator);
	214	return false;
	215	},
	216	1, 2 => {
	217	if (status == 1)
	218	try self.accum.reset(self.allocator);
	219
	220	const size = try packed_reader.readIntBig(u16) + 1;
	221	try self.accum.ensureUnusedCapacity(self.allocator, size);
	222
	223	var i: usize = 0;
	224	while (i < size) : (i += 1)
	225	self.accum.appendAssumeCapacity(try packed_reader.readByte());
	226
	227	return true;
	228	},
	229	else => {
	230	if (status & 0x80 == 0)
	231	return error.CorruptInput;
	232
	233	const Reset = struct {
	234	dict: bool,
	235	state: bool,
	236	props: bool,
	237	};
	238
	239	const reset = switch ((status >> 5) & 0x3) {
	240	0 => Reset{
	241	.dict = false,
	242	.state = false,
	243	.props = false,
	244	},
	245	1 => Reset{
	246	.dict = false,
	247	.state = true,
	248	.props = false,
	249	},
	250	2 => Reset{
	251	.dict = false,
	252	.state = true,
	253	.props = true,
	254	},
	255	3 => Reset{
	256	.dict = true,
	257	.state = true,
	258	.props = true,
	259	},
	260	else => unreachable,
	261	};
	262
	263	const unpacked_size = blk: {
	264	var tmp: u64 = status & 0x1F;
	265	tmp <<= 16;
	266	tmp \|= try packed_reader.readIntBig(u16);
	267	break :blk tmp + 1;
	268	};
	269
	270	const packed_size = blk: {
	271	const tmp: u17 = try packed_reader.readIntBig(u16);
	272	break :blk tmp + 1;
	273	};
	274
	275	if (reset.dict)
	276	try self.accum.reset(self.allocator);
	277
	278	if (reset.state) {
	279	var new_props = self.lzma_state.lzma_props;
	280
	281	if (reset.props) {
	282	var props = try packed_reader.readByte();
	283	if (props >= 225)
	284	return error.CorruptInput;
	285
	286	const lc = @intCast(u4, props % 9);
	287	props /= 9;
	288	const lp = @intCast(u3, props % 5);
	289	props /= 5;
	290	const pb = @intCast(u3, props);
	291
	292	if (lc + lp > 4)
	293	return error.CorruptInput;
	294
	295	new_props = .{ .lc = lc, .lp = lp, .pb = pb };
	296	}
	297
	298	try self.lzma_state.reset_state(self.allocator, new_props);
	299	}
	300
	301	self.lzma_state.unpacked_size = unpacked_size + self.accum.len();
	302
	303	const buffer = try self.allocator.alloc(u8, packed_size);
	304	defer self.allocator.free(buffer);
	305
	306	for (buffer) \|*b\|
	307	b.* = try packed_reader.readByte();
	308
	309	var rangecoder = try lzma.RangeDecoder.init(buffer);
	310	try self.lzma_state.process(self.allocator, &self.accum, &rangecoder);
	311
	312	return true;
	313	},
	314	}
	315	}
	316	};
	317	}

lib/std/compress/xz/lzma.zig created+658

...	...	@@ -0,0 +1,658 @@
	1	// Ported from https://github.com/gendx/lzma-rs
	2
	3	const std = @import("../../std.zig");
	4	const assert = std.debug.assert;
	5	const Allocator = std.mem.Allocator;
	6	const ArrayListUnmanaged = std.ArrayListUnmanaged;
	7
	8	const LzmaProperties = struct {
	9	lc: u4,
	10	lp: u3,
	11	pb: u3,
	12
	13	fn validate(self: LzmaProperties) void {
	14	assert(self.lc <= 8);
	15	assert(self.lp <= 4);
	16	assert(self.pb <= 4);
	17	}
	18	};
	19
	20	pub const DecoderState = struct {
	21	lzma_props: LzmaProperties,
	22	unpacked_size: ?u64,
	23	literal_probs: Vec2D(u16),
	24	pos_slot_decoder: [4]BitTree,
	25	align_decoder: BitTree,
	26	pos_decoders: [115]u16,
	27	is_match: [192]u16,
	28	is_rep: [12]u16,
	29	is_rep_g0: [12]u16,
	30	is_rep_g1: [12]u16,
	31	is_rep_g2: [12]u16,
	32	is_rep_0long: [192]u16,
	33	state: usize,
	34	rep: [4]usize,
	35	len_decoder: LenDecoder,
	36	rep_len_decoder: LenDecoder,
	37
	38	pub fn init(allocator: Allocator) !DecoderState {
	39	return .{
	40	.lzma_props = LzmaProperties{ .lc = 0, .lp = 0, .pb = 0 },
	41	.unpacked_size = null,
	42	.literal_probs = try Vec2D(u16).init(allocator, 0x400, 1, 0x300),
	43	.pos_slot_decoder = .{
	44	try BitTree.init(allocator, 6),
	45	try BitTree.init(allocator, 6),
	46	try BitTree.init(allocator, 6),
	47	try BitTree.init(allocator, 6),
	48	},
	49	.align_decoder = try BitTree.init(allocator, 4),
	50	.pos_decoders = .{0x400} ** 115,
	51	.is_match = .{0x400} ** 192,
	52	.is_rep = .{0x400} ** 12,
	53	.is_rep_g0 = .{0x400} ** 12,
	54	.is_rep_g1 = .{0x400} ** 12,
	55	.is_rep_g2 = .{0x400} ** 12,
	56	.is_rep_0long = .{0x400} ** 192,
	57	.state = 0,
	58	.rep = .{0} ** 4,
	59	.len_decoder = try LenDecoder.init(allocator),
	60	.rep_len_decoder = try LenDecoder.init(allocator),
	61	};
	62	}
	63
	64	pub fn deinit(self: *DecoderState, allocator: Allocator) void {
	65	self.literal_probs.deinit(allocator);
	66	for (self.pos_slot_decoder) \|*t\| t.deinit(allocator);
	67	self.align_decoder.deinit(allocator);
	68	self.len_decoder.deinit(allocator);
	69	self.rep_len_decoder.deinit(allocator);
	70	}
	71
	72	pub fn reset_state(self: *DecoderState, allocator: Allocator, new_props: LzmaProperties) !void {
	73	new_props.validate();
	74	if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
	75	self.literal_probs.fill(0x400);
	76	} else {
	77	self.literal_probs.deinit(allocator);
	78	self.literal_probs = try Vec2D(u16).init(allocator, 0x400, @as(usize, 1) << (new_props.lc + new_props.lp), 0x300);
	79	}
	80
	81	self.lzma_props = new_props;
	82	for (self.pos_slot_decoder) \|*t\| t.reset();
	83	self.align_decoder.reset();
	84	self.pos_decoders = .{0x400} ** 115;
	85	self.is_match = .{0x400} ** 192;
	86	self.is_rep = .{0x400} ** 12;
	87	self.is_rep_g0 = .{0x400} ** 12;
	88	self.is_rep_g1 = .{0x400} ** 12;
	89	self.is_rep_g2 = .{0x400} ** 12;
	90	self.is_rep_0long = .{0x400} ** 192;
	91	self.state = 0;
	92	self.rep = .{0} ** 4;
	93	self.len_decoder.reset();
	94	self.rep_len_decoder.reset();
	95	}
	96
	97	fn processNextInner(
	98	self: *DecoderState,
	99	allocator: Allocator,
	100	output: *LzAccumBuffer,
	101	rangecoder: *RangeDecoder,
	102	update: bool,
	103	) !ProcessingStatus {
	104	const pos_state = output.len() & ((@as(usize, 1) << self.lzma_props.pb) - 1);
	105
	106	if (!try rangecoder.decodeBit(
	107	&self.is_match[(self.state << 4) + pos_state],
	108	update,
	109	)) {
	110	const byte: u8 = try self.decodeLiteral(output, rangecoder, update);
	111
	112	if (update) {
	113	try output.appendLiteral(allocator, byte);
	114
	115	self.state = if (self.state < 4)
	116	0
	117	else if (self.state < 10)
	118	self.state - 3
	119	else
	120	self.state - 6;
	121	}
	122	return .continue_;
	123	}
	124
	125	var len: usize = undefined;
	126	if (try rangecoder.decodeBit(&self.is_rep[self.state], update)) {
	127	if (!try rangecoder.decodeBit(&self.is_rep_g0[self.state], update)) {
	128	if (!try rangecoder.decodeBit(
	129	&self.is_rep_0long[(self.state << 4) + pos_state],
	130	update,
	131	)) {
	132	if (update) {
	133	self.state = if (self.state < 7) 9 else 11;
	134	const dist = self.rep[0] + 1;
	135	try output.appendLz(allocator, 1, dist);
	136	}
	137	return .continue_;
	138	}
	139	} else {
	140	const idx: usize = if (!try rangecoder.decodeBit(&self.is_rep_g1[self.state], update))
	141	1
	142	else if (!try rangecoder.decodeBit(&self.is_rep_g2[self.state], update))
	143	2
	144	else
	145	3;
	146	if (update) {
	147	const dist = self.rep[idx];
	148	var i = idx;
	149	while (i > 0) : (i -= 1) {
	150	self.rep[i] = self.rep[i - 1];
	151	}
	152	self.rep[0] = dist;
	153	}
	154	}
	155
	156	len = try self.rep_len_decoder.decode(rangecoder, pos_state, update);
	157
	158	if (update) {
	159	self.state = if (self.state < 7) 8 else 11;
	160	}
	161	} else {
	162	if (update) {
	163	self.rep[3] = self.rep[2];
	164	self.rep[2] = self.rep[1];
	165	self.rep[1] = self.rep[0];
	166	}
	167
	168	len = try self.len_decoder.decode(rangecoder, pos_state, update);
	169
	170	if (update) {
	171	self.state = if (self.state < 7) 7 else 10;
	172	}
	173
	174	const rep_0 = try self.decodeDistance(rangecoder, len, update);
	175
	176	if (update) {
	177	self.rep[0] = rep_0;
	178	if (self.rep[0] == 0xFFFF_FFFF) {
	179	if (rangecoder.isFinished()) {
	180	return .finished;
	181	}
	182	return error.CorruptInput;
	183	}
	184	}
	185	}
	186
	187	if (update) {
	188	len += 2;
	189
	190	const dist = self.rep[0] + 1;
	191	try output.appendLz(allocator, len, dist);
	192	}
	193
	194	return .continue_;
	195	}
	196
	197	fn processNext(
	198	self: *DecoderState,
	199	allocator: Allocator,
	200	output: *LzAccumBuffer,
	201	rangecoder: *RangeDecoder,
	202	) !ProcessingStatus {
	203	return self.processNextInner(allocator, output, rangecoder, true);
	204	}
	205
	206	pub fn process(
	207	self: *DecoderState,
	208	allocator: Allocator,
	209	output: *LzAccumBuffer,
	210	rangecoder: *RangeDecoder,
	211	) !void {
	212	while (true) {
	213	if (self.unpacked_size) \|unpacked_size\| {
	214	if (output.len() >= unpacked_size) {
	215	break;
	216	}
	217	} else if (rangecoder.isFinished()) {
	218	break;
	219	}
	220
	221	if (try self.processNext(allocator, output, rangecoder) == .finished) {
	222	break;
	223	}
	224	}
	225
	226	if (self.unpacked_size) \|len\| {
	227	if (len != output.len()) {
	228	return error.CorruptInput;
	229	}
	230	}
	231	}
	232
	233	fn decodeLiteral(
	234	self: *DecoderState,
	235	output: *LzAccumBuffer,
	236	rangecoder: *RangeDecoder,
	237	update: bool,
	238	) !u8 {
	239	const def_prev_byte = 0;
	240	const prev_byte = @as(usize, output.lastOr(def_prev_byte));
	241
	242	var result: usize = 1;
	243	const lit_state = ((output.len() & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
	244	(prev_byte >> (8 - self.lzma_props.lc));
	245	const probs = try self.literal_probs.get(lit_state);
	246
	247	if (self.state >= 7) {
	248	var match_byte = @as(usize, try output.lastN(self.rep[0] + 1));
	249
	250	while (result < 0x100) {
	251	const match_bit = (match_byte >> 7) & 1;
	252	match_byte <<= 1;
	253	const bit = @boolToInt(try rangecoder.decodeBit(
	254	&probs[((@as(usize, 1) + match_bit) << 8) + result],
	255	update,
	256	));
	257	result = (result << 1) ^ bit;
	258	if (match_bit != bit) {
	259	break;
	260	}
	261	}
	262	}
	263
	264	while (result < 0x100) {
	265	result = (result << 1) ^ @boolToInt(try rangecoder.decodeBit(&probs[result], update));
	266	}
	267
	268	return @truncate(u8, result - 0x100);
	269	}
	270
	271	fn decodeDistance(
	272	self: *DecoderState,
	273	rangecoder: *RangeDecoder,
	274	length: usize,
	275	update: bool,
	276	) !usize {
	277	const len_state = if (length > 3) 3 else length;
	278
	279	const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(rangecoder, update));
	280	if (pos_slot < 4)
	281	return pos_slot;
	282
	283	const num_direct_bits = @intCast(u5, (pos_slot >> 1) - 1);
	284	var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
	285
	286	if (pos_slot < 14) {
	287	result += try rangecoder.parseReverseBitTree(
	288	num_direct_bits,
	289	&self.pos_decoders,
	290	result - pos_slot,
	291	update,
	292	);
	293	} else {
	294	result += @as(usize, try rangecoder.get(num_direct_bits - 4)) << 4;
	295	result += try self.align_decoder.parseReverse(rangecoder, update);
	296	}
	297
	298	return result;
	299	}
	300	};
	301
	302	const ProcessingStatus = enum {
	303	continue_,
	304	finished,
	305	};
	306
	307	pub const LzAccumBuffer = struct {
	308	to_read: ArrayListUnmanaged(u8) = .{},
	309	buf: ArrayListUnmanaged(u8) = .{},
	310
	311	pub fn deinit(self: *LzAccumBuffer, allocator: Allocator) void {
	312	self.to_read.deinit(allocator);
	313	self.buf.deinit(allocator);
	314	}
	315
	316	pub fn read(self: *LzAccumBuffer, output: []u8) usize {
	317	const input = self.to_read.items;
	318	const n = std.math.min(input.len, output.len);
	319	std.mem.copy(u8, output[0..n], input[0..n]);
	320	std.mem.copy(u8, input, input[n..]);
	321	self.to_read.shrinkRetainingCapacity(input.len - n);
	322	return n;
	323	}
	324
	325	pub fn ensureUnusedCapacity(
	326	self: *LzAccumBuffer,
	327	allocator: Allocator,
	328	additional_count: usize,
	329	) !void {
	330	try self.buf.ensureUnusedCapacity(allocator, additional_count);
	331	}
	332
	333	pub fn appendAssumeCapacity(self: *LzAccumBuffer, byte: u8) void {
	334	self.buf.appendAssumeCapacity(byte);
	335	}
	336
	337	pub fn reset(self: *LzAccumBuffer, allocator: Allocator) !void {
	338	try self.to_read.appendSlice(allocator, self.buf.items);
	339	self.buf.clearRetainingCapacity();
	340	}
	341
	342	pub fn len(self: *const LzAccumBuffer) usize {
	343	return self.buf.items.len;
	344	}
	345
	346	pub fn lastOr(self: *const LzAccumBuffer, lit: u8) u8 {
	347	const buf_len = self.buf.items.len;
	348	return if (buf_len == 0)
	349	lit
	350	else
	351	self.buf.items[buf_len - 1];
	352	}
	353
	354	pub fn lastN(self: *const LzAccumBuffer, dist: usize) !u8 {
	355	const buf_len = self.buf.items.len;
	356	if (dist > buf_len) {
	357	return error.CorruptInput;
	358	}
	359
	360	return self.buf.items[buf_len - dist];
	361	}
	362
	363	pub fn appendLiteral(self: *LzAccumBuffer, allocator: Allocator, lit: u8) !void {
	364	try self.buf.append(allocator, lit);
	365	}
	366
	367	pub fn appendLz(self: *LzAccumBuffer, allocator: Allocator, length: usize, dist: usize) !void {
	368	const buf_len = self.buf.items.len;
	369	if (dist > buf_len) {
	370	return error.CorruptInput;
	371	}
	372
	373	var offset = buf_len - dist;
	374	var i: usize = 0;
	375	while (i < length) : (i += 1) {
	376	const x = self.buf.items[offset];
	377	try self.buf.append(allocator, x);
	378	offset += 1;
	379	}
	380	}
	381	};
	382
	383	pub const RangeDecoder = struct {
	384	stream: std.io.FixedBufferStream([]const u8),
	385	range: u32,
	386	code: u32,
	387
	388	pub fn init(buffer: []const u8) !RangeDecoder {
	389	var dec = RangeDecoder{
	390	.stream = std.io.fixedBufferStream(buffer),
	391	.range = 0xFFFF_FFFF,
	392	.code = 0,
	393	};
	394	const reader = dec.stream.reader();
	395	_ = try reader.readByte();
	396	dec.code = try reader.readIntBig(u32);
	397	return dec;
	398	}
	399
	400	pub fn fromParts(
	401	buffer: []const u8,
	402	range: u32,
	403	code: u32,
	404	) RangeDecoder {
	405	return .{
	406	.stream = std.io.fixedBufferStream(buffer),
	407	.range = range,
	408	.code = code,
	409	};
	410	}
	411
	412	pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
	413	self.range = range;
	414	self.code = code;
	415	}
	416
	417	pub fn readInto(self: *RangeDecoder, dest: []u8) !usize {
	418	return self.stream.read(dest);
	419	}
	420
	421	pub inline fn isFinished(self: *const RangeDecoder) bool {
	422	return self.code == 0 and self.isEof();
	423	}
	424
	425	pub inline fn isEof(self: *const RangeDecoder) bool {
	426	return self.stream.pos == self.stream.buffer.len;
	427	}
	428
	429	inline fn normalize(self: *RangeDecoder) !void {
	430	if (self.range < 0x0100_0000) {
	431	self.range <<= 8;
	432	self.code = (self.code << 8) ^ @as(u32, try self.stream.reader().readByte());
	433	}
	434	}
	435
	436	inline fn getBit(self: *RangeDecoder) !bool {
	437	self.range >>= 1;
	438
	439	const bit = self.code >= self.range;
	440	if (bit)
	441	self.code -= self.range;
	442
	443	try self.normalize();
	444	return bit;
	445	}
	446
	447	fn get(self: *RangeDecoder, count: usize) !u32 {
	448	var result: u32 = 0;
	449	var i: usize = 0;
	450	while (i < count) : (i += 1)
	451	result = (result << 1) ^ @boolToInt(try self.getBit());
	452	return result;
	453	}
	454
	455	pub inline fn decodeBit(self: RangeDecoder, prob: u16, update: bool) !bool {
	456	const bound = (self.range >> 11) * prob.*;
	457
	458	if (self.code < bound) {
	459	if (update)
	460	prob.* += (0x800 - prob.*) >> 5;
	461	self.range = bound;
	462
	463	try self.normalize();
	464	return false;
	465	} else {
	466	if (update)
	467	prob.* -= prob.* >> 5;
	468	self.code -= bound;
	469	self.range -= bound;
	470
	471	try self.normalize();
	472	return true;
	473	}
	474	}
	475
	476	fn parseBitTree(
	477	self: *RangeDecoder,
	478	num_bits: u5,
	479	probs: []u16,
	480	update: bool,
	481	) !u32 {
	482	var tmp: u32 = 1;
	483	var i: u5 = 0;
	484	while (i < num_bits) : (i += 1) {
	485	const bit = try self.decodeBit(&probs[tmp], update);
	486	tmp = (tmp << 1) ^ @boolToInt(bit);
	487	}
	488	return tmp - (@as(u32, 1) << num_bits);
	489	}
	490
	491	pub fn parseReverseBitTree(
	492	self: *RangeDecoder,
	493	num_bits: u5,
	494	probs: []u16,
	495	offset: usize,
	496	update: bool,
	497	) !u32 {
	498	var result: u32 = 0;
	499	var tmp: usize = 1;
	500	var i: u5 = 0;
	501	while (i < num_bits) : (i += 1) {
	502	const bit = @boolToInt(try self.decodeBit(&probs[offset + tmp], update));
	503	tmp = (tmp << 1) ^ bit;
	504	result ^= @as(u32, bit) << i;
	505	}
	506	return result;
	507	}
	508	};
	509
	510	fn Vec2D(comptime T: type) type {
	511	return struct {
	512	data: []T,
	513	cols: usize,
	514
	515	const Self = @This();
	516
	517	pub fn init(allocator: Allocator, data: T, rows: usize, cols: usize) !Self {
	518	const len = try std.math.mul(usize, rows, cols);
	519	var vec2d = Self{
	520	.data = try allocator.alloc(T, len),
	521	.cols = cols,
	522	};
	523	vec2d.fill(data);
	524	return vec2d;
	525	}
	526
	527	pub fn deinit(self: *Self, allocator: Allocator) void {
	528	allocator.free(self.data);
	529	}
	530
	531	pub fn fill(self: *Self, value: T) void {
	532	std.mem.set(T, self.data, value);
	533	}
	534
	535	pub fn get(self: *Self, row: usize) ![]T {
	536	const start_row = try std.math.mul(usize, row, self.cols);
	537	return self.data[start_row .. start_row + self.cols];
	538	}
	539	};
	540	}
	541
	542	const BitTree = struct {
	543	num_bits: u5,
	544	probs: ArrayListUnmanaged(u16),
	545
	546	pub fn init(allocator: Allocator, num_bits: u5) !BitTree {
	547	var probs_len = @as(usize, 1) << num_bits;
	548	var probs = try ArrayListUnmanaged(u16).initCapacity(allocator, probs_len);
	549	while (probs_len > 0) : (probs_len -= 1)
	550	probs.appendAssumeCapacity(0x400);
	551	return .{ .num_bits = num_bits, .probs = probs };
	552	}
	553
	554	pub fn deinit(self: *BitTree, allocator: Allocator) void {
	555	self.probs.deinit(allocator);
	556	}
	557
	558	pub fn parse(
	559	self: *BitTree,
	560	rangecoder: *RangeDecoder,
	561	update: bool,
	562	) !u32 {
	563	return rangecoder.parseBitTree(self.num_bits, self.probs.items, update);
	564	}
	565
	566	pub fn parseReverse(
	567	self: *BitTree,
	568	rangecoder: *RangeDecoder,
	569	update: bool,
	570	) !u32 {
	571	return rangecoder.parseReverseBitTree(self.num_bits, self.probs.items, 0, update);
	572	}
	573
	574	pub fn reset(self: *BitTree) void {
	575	std.mem.set(u16, self.probs.items, 0x400);
	576	}
	577	};
	578
	579	const LenDecoder = struct {
	580	choice: u16,
	581	choice2: u16,
	582	low_coder: [16]BitTree,
	583	mid_coder: [16]BitTree,
	584	high_coder: BitTree,
	585
	586	pub fn init(allocator: Allocator) !LenDecoder {
	587	return .{
	588	.choice = 0x400,
	589	.choice2 = 0x400,
	590	.low_coder = .{
	591	try BitTree.init(allocator, 3),
	592	try BitTree.init(allocator, 3),
	593	try BitTree.init(allocator, 3),
	594	try BitTree.init(allocator, 3),
	595	try BitTree.init(allocator, 3),
	596	try BitTree.init(allocator, 3),
	597	try BitTree.init(allocator, 3),
	598	try BitTree.init(allocator, 3),
	599	try BitTree.init(allocator, 3),
	600	try BitTree.init(allocator, 3),
	601	try BitTree.init(allocator, 3),
	602	try BitTree.init(allocator, 3),
	603	try BitTree.init(allocator, 3),
	604	try BitTree.init(allocator, 3),
	605	try BitTree.init(allocator, 3),
	606	try BitTree.init(allocator, 3),
	607	},
	608	.mid_coder = .{
	609	try BitTree.init(allocator, 3),
	610	try BitTree.init(allocator, 3),
	611	try BitTree.init(allocator, 3),
	612	try BitTree.init(allocator, 3),
	613	try BitTree.init(allocator, 3),
	614	try BitTree.init(allocator, 3),
	615	try BitTree.init(allocator, 3),
	616	try BitTree.init(allocator, 3),
	617	try BitTree.init(allocator, 3),
	618	try BitTree.init(allocator, 3),
	619	try BitTree.init(allocator, 3),
	620	try BitTree.init(allocator, 3),
	621	try BitTree.init(allocator, 3),
	622	try BitTree.init(allocator, 3),
	623	try BitTree.init(allocator, 3),
	624	try BitTree.init(allocator, 3),
	625	},
	626	.high_coder = try BitTree.init(allocator, 8),
	627	};
	628	}
	629
	630	pub fn deinit(self: *LenDecoder, allocator: Allocator) void {
	631	for (self.low_coder) \|*t\| t.deinit(allocator);
	632	for (self.mid_coder) \|*t\| t.deinit(allocator);
	633	self.high_coder.deinit(allocator);
	634	}
	635
	636	pub fn decode(
	637	self: *LenDecoder,
	638	rangecoder: *RangeDecoder,
	639	pos_state: usize,
	640	update: bool,
	641	) !usize {
	642	if (!try rangecoder.decodeBit(&self.choice, update)) {
	643	return @as(usize, try self.low_coder[pos_state].parse(rangecoder, update));
	644	} else if (!try rangecoder.decodeBit(&self.choice2, update)) {
	645	return @as(usize, try self.mid_coder[pos_state].parse(rangecoder, update)) + 8;
	646	} else {
	647	return @as(usize, try self.high_coder.parse(rangecoder, update)) + 16;
	648	}
	649	}
	650
	651	pub fn reset(self: *LenDecoder) void {
	652	self.choice = 0x400;
	653	self.choice2 = 0x400;
	654	for (self.low_coder) \|*t\| t.reset();
	655	for (self.mid_coder) \|*t\| t.reset();
	656	self.high_coder.reset();
	657	}
	658	};

lib/std/compress/xz/test.zig created+80

...	...	@@ -0,0 +1,80 @@
	1	const std = @import("../../std.zig");
	2	const testing = std.testing;
	3	const xz = std.compress.xz;
	4
	5	fn decompress(data: []const u8) ![]u8 {
	6	var in_stream = std.io.fixedBufferStream(data);
	7
	8	var xz_stream = try xz.decompress(testing.allocator, in_stream.reader());
	9	defer xz_stream.deinit();
	10
	11	return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
	12	}
	13
	14	fn testReader(data: []const u8, comptime expected: []const u8) !void {
	15	const buf = try decompress(data);
	16	defer testing.allocator.free(buf);
	17
	18	try testing.expectEqualSlices(u8, expected, buf);
	19	}
	20
	21	test "compressed data" {
	22	try testReader(@embedFile("testdata/good-0-empty.xz"), "");
	23
	24	inline for ([_][]const u8{
	25	"good-1-check-none.xz",
	26	"good-1-check-crc32.xz",
	27	"good-1-check-crc64.xz",
	28	"good-1-check-sha256.xz",
	29	"good-2-lzma2.xz",
	30	"good-1-block_header-1.xz",
	31	"good-1-block_header-2.xz",
	32	"good-1-block_header-3.xz",
	33	}) \|filename\| {
	34	try testReader(@embedFile("testdata/" ++ filename),
	35	\\Hello
	36	\\World!
	37	\\
	38	);
	39	}
	40
	41	inline for ([_][]const u8{
	42	"good-1-lzma2-1.xz",
	43	"good-1-lzma2-2.xz",
	44	"good-1-lzma2-3.xz",
	45	"good-1-lzma2-4.xz",
	46	}) \|filename\| {
	47	try testReader(@embedFile("testdata/" ++ filename),
	48	\\Lorem ipsum dolor sit amet, consectetur adipisicing
	49	\\elit, sed do eiusmod tempor incididunt ut
	50	\\labore et dolore magna aliqua. Ut enim
	51	\\ad minim veniam, quis nostrud exercitation ullamco
	52	\\laboris nisi ut aliquip ex ea commodo
	53	\\consequat. Duis aute irure dolor in reprehenderit
	54	\\in voluptate velit esse cillum dolore eu
	55	\\fugiat nulla pariatur. Excepteur sint occaecat cupidatat
	56	\\non proident, sunt in culpa qui officia
	57	\\deserunt mollit anim id est laborum.
	58	\\
	59	);
	60	}
	61
	62	try testReader(@embedFile("testdata/good-1-lzma2-5.xz"), "");
	63	}
	64
	65	test "unsupported" {
	66	inline for ([_][]const u8{
	67	"good-1-delta-lzma2.tiff.xz",
	68	"good-1-x86-lzma2.xz",
	69	"good-1-sparc-lzma2.xz",
	70	"good-1-arm64-lzma2-1.xz",
	71	"good-1-arm64-lzma2-2.xz",
	72	"good-1-3delta-lzma2.xz",
	73	"good-1-empty-bcj-lzma2.xz",
	74	}) \|filename\| {
	75	try testing.expectError(
	76	error.Unsupported,
	77	decompress(@embedFile("testdata/" ++ filename)),
	78	);
	79	}
	80	}

lib/std/compress/xz/testdata/good-0-empty.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0-empty.xz differ

lib/std/compress/xz/testdata/good-0cat-empty.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0cat-empty.xz differ

lib/std/compress/xz/testdata/good-0catpad-empty.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0catpad-empty.xz differ

lib/std/compress/xz/testdata/good-0pad-empty.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-0pad-empty.xz differ

lib/std/compress/xz/testdata/good-1-3delta-lzma2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-3delta-lzma2.xz differ

lib/std/compress/xz/testdata/good-1-arm64-lzma2-1.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-arm64-lzma2-1.xz differ

lib/std/compress/xz/testdata/good-1-arm64-lzma2-2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-arm64-lzma2-2.xz differ

lib/std/compress/xz/testdata/good-1-block_header-1.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-block_header-1.xz differ

lib/std/compress/xz/testdata/good-1-block_header-2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-block_header-2.xz differ

lib/std/compress/xz/testdata/good-1-block_header-3.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-block_header-3.xz differ

lib/std/compress/xz/testdata/good-1-check-crc32.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-crc32.xz differ

lib/std/compress/xz/testdata/good-1-check-crc64.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-crc64.xz differ

lib/std/compress/xz/testdata/good-1-check-none.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-none.xz differ

lib/std/compress/xz/testdata/good-1-check-sha256.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-check-sha256.xz differ

lib/std/compress/xz/testdata/good-1-delta-lzma2.tiff.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-delta-lzma2.tiff.xz differ

lib/std/compress/xz/testdata/good-1-empty-bcj-lzma2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-empty-bcj-lzma2.xz differ

lib/std/compress/xz/testdata/good-1-lzma2-1.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-1.xz differ

lib/std/compress/xz/testdata/good-1-lzma2-2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-2.xz differ

lib/std/compress/xz/testdata/good-1-lzma2-3.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-3.xz differ

lib/std/compress/xz/testdata/good-1-lzma2-4.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-4.xz differ

lib/std/compress/xz/testdata/good-1-lzma2-5.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-lzma2-5.xz differ

lib/std/compress/xz/testdata/good-1-sparc-lzma2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-sparc-lzma2.xz differ

lib/std/compress/xz/testdata/good-1-x86-lzma2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-1-x86-lzma2.xz differ

lib/std/compress/xz/testdata/good-2-lzma2.xz created

Binary files /dev/null and b/lib/std/compress/xz/testdata/good-2-lzma2.xz differ

src/Package.zig+21-8

...	...	@@ -370,14 +370,11 @@ fn fetchAndUnpack(
370	370	if (mem.endsWith(u8, uri.path, ".tar.gz")) {
371	371	// I observed the gzip stream to read 1 byte at a time, so I am using a
372	372	// buffered reader on the front of it.
373		var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader());
374
375		var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader());
376		defer gzip_stream.deinit();
377
378		try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{
379		.strip_components = 1,
380		});
	373	try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip);
	374	} else if (mem.endsWith(u8, uri.path, ".tar.xz")) {
	375	// I have not checked what buffer sizes the xz decompression implementation uses
	376	// by default, so the same logic applies for buffering the reader as for gzip.
	377	try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz);
381	378	} else {
382	379	return reportError(
383	380	ini,
...	...	@@ -430,6 +427,22 @@ fn fetchAndUnpack(
430	427	return createWithDir(gpa, fqn, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
431	428	}
432	429
	430	fn unpackTarball(
	431	gpa: Allocator,
	432	req: *std.http.Client.Request,
	433	out_dir: fs.Dir,
	434	comptime compression: type,
	435	) !void {
	436	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader());
	437
	438	var decompress = try compression.decompress(gpa, br.reader());
	439	defer decompress.deinit();
	440
	441	try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{
	442	.strip_components = 1,
	443	});
	444	}
	445
433	446	fn reportError(
434	447	ini: std.Ini,
435	448	comp_directory: Compilation.Directory,