Commit 58e60697e2930f4311ae9e744ae1c2877e0b69ed | zig

author	Andrew Kelley <andrew@ziglang.org> 2025-08-25 18:03:48-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-08-26 21:00:58-07:00
log	58e60697e2930f4311ae9e744ae1c2877e0b69ed
tree	fd84142e826d2252f23eaae9b002ae0d3f43e341
parent	6464e0d4fc9937e154c34567891bae84c63732b9

std.compress.lzma: update for new I/O API

5 files changed, 556 insertions(+), 653 deletions(-)

lib
- std
  - compress
    lzma.zig
    lzma
    test.zig
    lzma2.zig
    xz.zig
    xz
    Decompress.zig

lib/std/compress/lzma.zig+214-232

...	...	@@ -4,49 +4,34 @@ const mem = std.mem;
4	4	const Allocator = std.mem.Allocator;
5	5	const assert = std.debug.assert;
6	6	const ArrayList = std.ArrayList;
	7	const Writer = std.Io.Writer;
	8	const Reader = std.Io.Reader;
7	9
8	10	pub const RangeDecoder = struct {
9	11	range: u32,
10	12	code: u32,
11	13
12		pub fn init(reader: anytype) !RangeDecoder {
13		const reserved = try reader.readByte();
14		if (reserved != 0) {
15		return error.CorruptInput;
16		}
17		return RangeDecoder{
18		.range = 0xFFFF_FFFF,
19		.code = try reader.readInt(u32, .big),
20		};
21		}
22
23		pub fn fromParts(
24		range: u32,
25		code: u32,
26		) RangeDecoder {
	14	pub fn init(reader: *Reader) !RangeDecoder {
	15	const reserved = try reader.takeByte();
	16	if (reserved != 0) return error.InvalidRangeCode;
27	17	return .{
28		.range = range,
29		.code = code,
	18	.range = 0xFFFF_FFFF,
	19	.code = try reader.takeInt(u32, .big),
30	20	};
31	21	}
32	22
33		pub fn set(self: *RangeDecoder, range: u32, code: u32) void {
34		self.range = range;
35		self.code = code;
36		}
37
38		pub inline fn isFinished(self: RangeDecoder) bool {
	23	pub fn isFinished(self: RangeDecoder) bool {
39	24	return self.code == 0;
40	25	}
41	26
42		inline fn normalize(self: *RangeDecoder, reader: anytype) !void {
	27	fn normalize(self: RangeDecoder, reader: Reader) !void {
43	28	if (self.range < 0x0100_0000) {
44	29	self.range <<= 8;
45		self.code = (self.code << 8) ^ @as(u32, try reader.readByte());
	30	self.code = (self.code << 8) ^ @as(u32, try reader.takeByte());
46	31	}
47	32	}
48	33
49		inline fn getBit(self: *RangeDecoder, reader: anytype) !bool {
	34	fn getBit(self: RangeDecoder, reader: Reader) !bool {
50	35	self.range >>= 1;
51	36
52	37	const bit = self.code >= self.range;
...	...	@@ -57,7 +42,7 @@ pub const RangeDecoder = struct {
57	42	return bit;
58	43	}
59	44
60		pub fn get(self: *RangeDecoder, reader: anytype, count: usize) !u32 {
	45	pub fn get(self: RangeDecoder, reader: Reader, count: usize) !u32 {
61	46	var result: u32 = 0;
62	47	var i: usize = 0;
63	48	while (i < count) : (i += 1)
...	...	@@ -65,7 +50,7 @@ pub const RangeDecoder = struct {
65	50	return result;
66	51	}
67	52
68		pub inline fn decodeBit(self: RangeDecoder, reader: anytype, prob: u16, update: bool) !bool {
	53	pub fn decodeBit(self: RangeDecoder, reader: Reader, prob: *u16, update: bool) !bool {
69	54	const bound = (self.range >> 11) * prob.*;
70	55
71	56	if (self.code < bound) {
...	...	@@ -88,7 +73,7 @@ pub const RangeDecoder = struct {
88	73
89	74	fn parseBitTree(
90	75	self: *RangeDecoder,
91		reader: anytype,
	76	reader: *Reader,
92	77	num_bits: u5,
93	78	probs: []u16,
94	79	update: bool,
...	...	@@ -104,7 +89,7 @@ pub const RangeDecoder = struct {
104	89
105	90	pub fn parseReverseBitTree(
106	91	self: *RangeDecoder,
107		reader: anytype,
	92	reader: *Reader,
108	93	num_bits: u5,
109	94	probs: []u16,
110	95	offset: usize,
...	...	@@ -123,7 +108,7 @@ pub const RangeDecoder = struct {
123	108	};
124	109
125	110	pub const Decode = struct {
126		lzma_props: Properties,
	111	properties: Properties,
127	112	unpacked_size: ?u64,
128	113	literal_probs: Vec2d,
129	114	pos_slot_decoder: [4]BitTree(6),
...	...	@@ -141,14 +126,14 @@ pub const Decode = struct {
141	126	rep_len_decoder: LenDecoder,
142	127
143	128	pub fn init(
144		allocator: Allocator,
145		lzma_props: Properties,
	129	gpa: Allocator,
	130	properties: Properties,
146	131	unpacked_size: ?u64,
147	132	) !Decode {
148	133	return .{
149		.lzma_props = lzma_props,
	134	.properties = properties,
150	135	.unpacked_size = unpacked_size,
151		.literal_probs = try Vec2d.init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
	136	.literal_probs = try Vec2d.init(gpa, 0x400, .{ @as(usize, 1) << (properties.lc + properties.lp), 0x300 }),
152	137	.pos_slot_decoder = @splat(.{}),
153	138	.align_decoder = .{},
154	139	.pos_decoders = @splat(0x400),
...	...	@@ -165,21 +150,21 @@ pub const Decode = struct {
165	150	};
166	151	}
167	152
168		pub fn deinit(self: *Decode, allocator: Allocator) void {
169		self.literal_probs.deinit(allocator);
	153	pub fn deinit(self: *Decode, gpa: Allocator) void {
	154	self.literal_probs.deinit(gpa);
170	155	self.* = undefined;
171	156	}
172	157
173		pub fn resetState(self: *Decode, allocator: Allocator, new_props: Properties) !void {
	158	pub fn resetState(self: *Decode, gpa: Allocator, new_props: Properties) !void {
174	159	new_props.validate();
175		if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
	160	if (self.properties.lc + self.properties.lp == new_props.lc + new_props.lp) {
176	161	self.literal_probs.fill(0x400);
177	162	} else {
178		self.literal_probs.deinit(allocator);
179		self.literal_probs = try Vec2d.init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
	163	self.literal_probs.deinit(gpa);
	164	self.literal_probs = try Vec2d.init(gpa, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
180	165	}
181	166
182		self.lzma_props = new_props;
	167	self.properties = new_props;
183	168	for (&self.pos_slot_decoder) \|*t\| t.reset();
184	169	self.align_decoder.reset();
185	170	self.pos_decoders = @splat(0x400);
...	...	@@ -195,26 +180,23 @@ pub const Decode = struct {
195	180	self.rep_len_decoder.reset();
196	181	}
197	182
198		fn processNextInner(
	183	fn processNext(
199	184	self: *Decode,
200		allocator: Allocator,
201		reader: anytype,
202		writer: anytype,
203		buffer: anytype,
	185	reader: *Reader,
	186	allocating: *Writer.Allocating,
	187	buffer: *CircularBuffer,
204	188	decoder: *RangeDecoder,
205	189	update: bool,
206	190	) !ProcessingStatus {
207		const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
	191	const gpa = allocating.allocator;
	192	const writer = &allocating.writer;
	193	const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
208	194
209		if (!try decoder.decodeBit(
210		reader,
211		&self.is_match[(self.state << 4) + pos_state],
212		update,
213		)) {
	195	if (!try decoder.decodeBit(reader, &self.is_match[(self.state << 4) + pos_state], update)) {
214	196	const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
215	197
216	198	if (update) {
217		try buffer.appendLiteral(allocator, byte, writer);
	199	try buffer.appendLiteral(gpa, byte, writer);
218	200
219	201	self.state = if (self.state < 4)
220	202	0
...	...	@@ -223,7 +205,7 @@ pub const Decode = struct {
223	205	else
224	206	self.state - 6;
225	207	}
226		return .continue_;
	208	return .more;
227	209	}
228	210
229	211	var len: usize = undefined;
...	...	@@ -237,9 +219,9 @@ pub const Decode = struct {
237	219	if (update) {
238	220	self.state = if (self.state < 7) 9 else 11;
239	221	const dist = self.rep[0] + 1;
240		try buffer.appendLz(allocator, 1, dist, writer);
	222	try buffer.appendLz(gpa, 1, dist, writer);
241	223	}
242		return .continue_;
	224	return .more;
243	225	}
244	226	} else {
245	227	const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
...	...	@@ -293,31 +275,19 @@ pub const Decode = struct {
293	275	len += 2;
294	276
295	277	const dist = self.rep[0] + 1;
296		try buffer.appendLz(allocator, len, dist, writer);
	278	try buffer.appendLz(gpa, len, dist, writer);
297	279	}
298	280
299		return .continue_;
300		}
301
302		fn processNext(
303		self: *Decode,
304		allocator: Allocator,
305		reader: anytype,
306		writer: anytype,
307		buffer: anytype,
308		decoder: *RangeDecoder,
309		) !ProcessingStatus {
310		return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
	281	return .more;
311	282	}
312	283
313	284	pub fn process(
314	285	self: *Decode,
315		allocator: Allocator,
316		reader: anytype,
317		writer: anytype,
318		buffer: anytype,
	286	reader: *Reader,
	287	allocating: *Writer.Allocating,
	288	buffer: *CircularBuffer,
319	289	decoder: *RangeDecoder,
320		) !ProcessingStatus {
	290	) !void {
321	291	process_next: {
322	292	if (self.unpacked_size) \|unpacked_size\| {
323	293	if (buffer.len >= unpacked_size) {
...	...	@@ -326,26 +296,24 @@ pub const Decode = struct {
326	296	} else if (decoder.isFinished()) {
327	297	break :process_next;
328	298	}
329
330		switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
331		.continue_ => return .continue_,
332		.finished => break :process_next,
	299	switch (try self.processNext(reader, allocating, buffer, decoder, true)) {
	300	.more => return,
	301	.finished => {},
333	302	}
334	303	}
335	304
336	305	if (self.unpacked_size) \|unpacked_size\| {
337		if (buffer.len != unpacked_size) {
338		return error.CorruptInput;
339		}
	306	if (buffer.len != unpacked_size) return error.DecompressedSizeMismatch;
340	307	}
341	308
342		return .finished;
	309	try buffer.finish(&allocating.writer);
	310	self.state = math.maxInt(usize);
343	311	}
344	312
345	313	fn decodeLiteral(
346	314	self: *Decode,
347		reader: anytype,
348		buffer: anytype,
	315	reader: *Reader,
	316	buffer: *CircularBuffer,
349	317	decoder: *RangeDecoder,
350	318	update: bool,
351	319	) !u8 {
...	...	@@ -353,9 +321,9 @@ pub const Decode = struct {
353	321	const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
354	322
355	323	var result: usize = 1;
356		const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
357		(prev_byte >> (8 - self.lzma_props.lc));
358		const probs = try self.literal_probs.getMut(lit_state);
	324	const lit_state = ((buffer.len & ((@as(usize, 1) << self.properties.lp) - 1)) << self.properties.lc) +
	325	(prev_byte >> (8 - self.properties.lc));
	326	const probs = try self.literal_probs.get(lit_state);
359	327
360	328	if (self.state >= 7) {
361	329	var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
...	...	@@ -384,7 +352,7 @@ pub const Decode = struct {
384	352
385	353	fn decodeDistance(
386	354	self: *Decode,
387		reader: anytype,
	355	reader: *Reader,
388	356	decoder: *RangeDecoder,
389	357	length: usize,
390	358	update: bool,
...	...	@@ -415,46 +383,40 @@ pub const Decode = struct {
415	383	}
416	384
417	385	/// A circular buffer for LZ sequences
418		pub const LzCircularBuffer = struct {
	386	pub const CircularBuffer = struct {
419	387	/// Circular buffer
420	388	buf: ArrayList(u8),
421
422	389	/// Length of the buffer
423	390	dict_size: usize,
424
425	391	/// Buffer memory limit
426		memlimit: usize,
427
	392	mem_limit: usize,
428	393	/// Current position
429	394	cursor: usize,
430
431	395	/// Total number of bytes sent through the buffer
432	396	len: usize,
433	397
434		const Self = @This();
435
436		pub fn init(dict_size: usize, memlimit: usize) Self {
437		return Self{
	398	pub fn init(dict_size: usize, mem_limit: usize) CircularBuffer {
	399	return .{
438	400	.buf = .{},
439	401	.dict_size = dict_size,
440		.memlimit = memlimit,
	402	.mem_limit = mem_limit,
441	403	.cursor = 0,
442	404	.len = 0,
443	405	};
444	406	}
445	407
446		pub fn get(self: Self, index: usize) u8 {
	408	pub fn get(self: CircularBuffer, index: usize) u8 {
447	409	return if (0 <= index and index < self.buf.items.len)
448	410	self.buf.items[index]
449	411	else
450	412	0;
451	413	}
452	414
453		pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
454		if (index >= self.memlimit) {
	415	pub fn set(self: *CircularBuffer, gpa: Allocator, index: usize, value: u8) !void {
	416	if (index >= self.mem_limit) {
455	417	return error.CorruptInput;
456	418	}
457		try self.buf.ensureTotalCapacity(allocator, index + 1);
	419	try self.buf.ensureTotalCapacity(gpa, index + 1);
458	420	while (self.buf.items.len < index) {
459	421	self.buf.appendAssumeCapacity(0);
460	422	}
...	...	@@ -462,7 +424,7 @@ pub const Decode = struct {
462	424	}
463	425
464	426	/// Retrieve the last byte or return a default
465		pub fn lastOr(self: Self, lit: u8) u8 {
	427	pub fn lastOr(self: CircularBuffer, lit: u8) u8 {
466	428	return if (self.len == 0)
467	429	lit
468	430	else
...	...	@@ -470,7 +432,7 @@ pub const Decode = struct {
470	432	}
471	433
472	434	/// Retrieve the n-th last byte
473		pub fn lastN(self: Self, dist: usize) !u8 {
	435	pub fn lastN(self: CircularBuffer, dist: usize) !u8 {
474	436	if (dist > self.dict_size or dist > self.len) {
475	437	return error.CorruptInput;
476	438	}
...	...	@@ -481,12 +443,12 @@ pub const Decode = struct {
481	443
482	444	/// Append a literal
483	445	pub fn appendLiteral(
484		self: *Self,
485		allocator: Allocator,
	446	self: *CircularBuffer,
	447	gpa: Allocator,
486	448	lit: u8,
487		writer: anytype,
	449	writer: *Writer,
488	450	) !void {
489		try self.set(allocator, self.cursor, lit);
	451	try self.set(gpa, self.cursor, lit);
490	452	self.cursor += 1;
491	453	self.len += 1;
492	454
...	...	@@ -499,11 +461,11 @@ pub const Decode = struct {
499	461
500	462	/// Fetch an LZ sequence (length, distance) from inside the buffer
501	463	pub fn appendLz(
502		self: *Self,
503		allocator: Allocator,
	464	self: *CircularBuffer,
	465	gpa: Allocator,
504	466	len: usize,
505	467	dist: usize,
506		writer: anytype,
	468	writer: *Writer,
507	469	) !void {
508	470	if (dist > self.dict_size or dist > self.len) {
509	471	return error.CorruptInput;
...	...	@@ -513,7 +475,7 @@ pub const Decode = struct {
513	475	var i: usize = 0;
514	476	while (i < len) : (i += 1) {
515	477	const x = self.get(offset);
516		try self.appendLiteral(allocator, x, writer);
	478	try self.appendLiteral(gpa, x, writer);
517	479	offset += 1;
518	480	if (offset == self.dict_size) {
519	481	offset = 0;
...	...	@@ -521,15 +483,15 @@ pub const Decode = struct {
521	483	}
522	484	}
523	485
524		pub fn finish(self: *Self, writer: anytype) !void {
	486	pub fn finish(self: CircularBuffer, writer: Writer) !void {
525	487	if (self.cursor > 0) {
526	488	try writer.writeAll(self.buf.items[0..self.cursor]);
527	489	self.cursor = 0;
528	490	}
529	491	}
530	492
531		pub fn deinit(self: *Self, allocator: Allocator) void {
532		self.buf.deinit(allocator);
	493	pub fn deinit(self: *CircularBuffer, gpa: Allocator) void {
	494	self.buf.deinit(gpa);
533	495	self.* = undefined;
534	496	}
535	497	};
...	...	@@ -538,11 +500,9 @@ pub const Decode = struct {
538	500	return struct {
539	501	probs: [1 << num_bits]u16 = @splat(0x400),
540	502
541		const Self = @This();
542
543	503	pub fn parse(
544		self: *Self,
545		reader: anytype,
	504	self: *@This(),
	505	reader: *Reader,
546	506	decoder: *RangeDecoder,
547	507	update: bool,
548	508	) !u32 {
...	...	@@ -550,15 +510,15 @@ pub const Decode = struct {
550	510	}
551	511
552	512	pub fn parseReverse(
553		self: *Self,
554		reader: anytype,
	513	self: *@This(),
	514	reader: *Reader,
555	515	decoder: *RangeDecoder,
556	516	update: bool,
557	517	) !u32 {
558	518	return decoder.parseReverseBitTree(reader, num_bits, &self.probs, 0, update);
559	519	}
560	520
561		pub fn reset(self: *Self) void {
	521	pub fn reset(self: *@This()) void {
562	522	@memset(&self.probs, 0x400);
563	523	}
564	524	};
...	...	@@ -573,7 +533,7 @@ pub const Decode = struct {
573	533
574	534	pub fn decode(
575	535	self: *LenDecoder,
576		reader: anytype,
	536	reader: *Reader,
577	537	decoder: *RangeDecoder,
578	538	pos_state: usize,
579	539	update: bool,
...	...	@@ -600,45 +560,35 @@ pub const Decode = struct {
600	560	data: []u16,
601	561	cols: usize,
602	562
603		const Self = @This();
604
605		pub fn init(allocator: Allocator, value: u16, size: struct { usize, usize }) !Self {
	563	pub fn init(gpa: Allocator, value: u16, size: struct { usize, usize }) !Vec2d {
606	564	const len = try math.mul(usize, size[0], size[1]);
607		const data = try allocator.alloc(u16, len);
	565	const data = try gpa.alloc(u16, len);
608	566	@memset(data, value);
609		return Self{
	567	return .{
610	568	.data = data,
611	569	.cols = size[1],
612	570	};
613	571	}
614	572
615		pub fn deinit(self: *Self, allocator: Allocator) void {
616		allocator.free(self.data);
	573	pub fn deinit(self: *Vec2d, gpa: Allocator) void {
	574	gpa.free(self.data);
617	575	self.* = undefined;
618	576	}
619	577
620		pub fn fill(self: *Self, value: u16) void {
	578	pub fn fill(self: *Vec2d, value: u16) void {
621	579	@memset(self.data, value);
622	580	}
623	581
624		inline fn _get(self: Self, row: usize) ![]u16 {
	582	fn get(self: Vec2d, row: usize) ![]u16 {
625	583	const start_row = try math.mul(usize, row, self.cols);
626	584	const end_row = try math.add(usize, start_row, self.cols);
627	585	return self.data[start_row..end_row];
628	586	}
629
630		pub fn get(self: Self, row: usize) ![]const u16 {
631		return self._get(row);
632		}
633
634		pub fn getMut(self: *Self, row: usize) ![]u16 {
635		return self._get(row);
636		}
637	587	};
638	588
639	589	pub const Options = struct {
640	590	unpacked_size: UnpackedSize = .read_from_header,
641		memlimit: ?usize = null,
	591	mem_limit: ?usize = null,
642	592	allow_incomplete: bool = false,
643	593	};
644	594
...	...	@@ -649,7 +599,7 @@ pub const Decode = struct {
649	599	};
650	600
651	601	const ProcessingStatus = enum {
652		continue_,
	602	more,
653	603	finished,
654	604	};
655	605
...	...	@@ -670,39 +620,34 @@ pub const Decode = struct {
670	620	dict_size: u32,
671	621	unpacked_size: ?u64,
672	622
673		pub fn readHeader(reader: anytype, options: Options) !Params {
674		var props = try reader.readByte();
675		if (props >= 225) {
676		return error.CorruptInput;
677		}
	623	pub fn readHeader(reader: *Reader, options: Options) !Params {
	624	var props = try reader.takeByte();
	625	if (props >= 225) return error.CorruptInput;
678	626
679		const lc = @as(u4, @intCast(props % 9));
	627	const lc: u4 = @intCast(props % 9);
680	628	props /= 9;
681		const lp = @as(u3, @intCast(props % 5));
	629	const lp: u3 = @intCast(props % 5);
682	630	props /= 5;
683		const pb = @as(u3, @intCast(props));
	631	const pb: u3 = @intCast(props);
684	632
685		const dict_size_provided = try reader.readInt(u32, .little);
	633	const dict_size_provided = try reader.takeInt(u32, .little);
686	634	const dict_size = @max(0x1000, dict_size_provided);
687	635
688	636	const unpacked_size = switch (options.unpacked_size) {
689	637	.read_from_header => blk: {
690		const unpacked_size_provided = try reader.readInt(u64, .little);
	638	const unpacked_size_provided = try reader.takeInt(u64, .little);
691	639	const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
692		break :blk if (marker_mandatory)
693		null
694		else
695		unpacked_size_provided;
	640	break :blk if (marker_mandatory) null else unpacked_size_provided;
696	641	},
697	642	.read_header_but_use_provided => \|x\| blk: {
698		_ = try reader.readInt(u64, .little);
	643	_ = try reader.takeInt(u64, .little);
699	644	break :blk x;
700	645	},
701	646	.use_provided => \|x\| x,
702	647	};
703	648
704		return Params{
705		.properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
	649	return .{
	650	.properties = .{ .lc = lc, .lp = lp, .pb = pb },
706	651	.dict_size = dict_size,
707	652	.unpacked_size = unpacked_size,
708	653	};
...	...	@@ -710,84 +655,121 @@ pub const Decode = struct {
710	655	};
711	656	};
712	657
713		pub fn decompress(
714		allocator: Allocator,
715		reader: anytype,
716		) !Decompress(@TypeOf(reader)) {
717		return decompressWithOptions(allocator, reader, .{});
718		}
719
720		pub fn decompressWithOptions(
721		allocator: Allocator,
722		reader: anytype,
723		options: Decode.Options,
724		) !Decompress(@TypeOf(reader)) {
725		const params = try Decode.Params.readHeader(reader, options);
726		return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);
727		}
728
729		pub fn Decompress(comptime ReaderType: type) type {
730		return struct {
731		const Self = @This();
732
733		pub const Error =
734		ReaderType.Error \|\|
735		Allocator.Error \|\|
736		error{ CorruptInput, EndOfStream, Overflow };
737
738		pub const Reader = std.io.GenericReader(*Self, Error, read);
	658	pub const Decompress = struct {
	659	gpa: Allocator,
	660	input: *Reader,
	661	reader: Reader,
	662	buffer: Decode.CircularBuffer,
	663	range_decoder: RangeDecoder,
	664	decode: Decode,
	665	err: ?Error,
	666
	667	pub const Error = error{
	668	OutOfMemory,
	669	ReadFailed,
	670	CorruptInput,
	671	DecompressedSizeMismatch,
	672	EndOfStream,
	673	Overflow,
	674	};
739	675
740		allocator: Allocator,
741		in_reader: ReaderType,
742		to_read: std.ArrayListUnmanaged(u8),
	676	/// Takes ownership of `buffer` which may be resized with `gpa`.
	677	///
	678	/// LZMA was explicitly designed to take advantage of large heap memory
	679	/// being available, with a dictionary size anywhere from 4K to 4G. Thus,
	680	/// this API dynamically allocates the dictionary as-needed.
	681	pub fn initParams(
	682	input: *Reader,
	683	gpa: Allocator,
	684	buffer: []u8,
	685	params: Decode.Params,
	686	mem_limit: usize,
	687	) !Decompress {
	688	return .{
	689	.gpa = gpa,
	690	.input = input,
	691	.buffer = Decode.CircularBuffer.init(params.dict_size, mem_limit),
	692	.range_decoder = try RangeDecoder.init(input),
	693	.decode = try Decode.init(gpa, params.properties, params.unpacked_size),
	694	.reader = .{
	695	.buffer = buffer,
	696	.vtable = &.{
	697	.readVec = readVec,
	698	.stream = stream,
	699	},
	700	.seek = 0,
	701	.end = 0,
	702	},
	703	.err = null,
	704	};
	705	}
743	706
744		buffer: Decode.LzCircularBuffer,
745		decoder: RangeDecoder,
746		state: Decode,
	707	/// Takes ownership of `buffer` which may be resized with `gpa`.
	708	///
	709	/// LZMA was explicitly designed to take advantage of large heap memory
	710	/// being available, with a dictionary size anywhere from 4K to 4G. Thus,
	711	/// this API dynamically allocates the dictionary as-needed.
	712	pub fn initOptions(
	713	input: *Reader,
	714	gpa: Allocator,
	715	buffer: []u8,
	716	options: Decode.Options,
	717	mem_limit: usize,
	718	) !Decompress {
	719	const params = try Decode.Params.readHeader(input, options);
	720	return initParams(input, gpa, buffer, params, mem_limit);
	721	}
747	722
748		pub fn init(allocator: Allocator, source: ReaderType, params: Decode.Params, memlimit: ?usize) !Self {
749		return Self{
750		.allocator = allocator,
751		.in_reader = source,
752		.to_read = .{},
	723	/// Reclaim ownership of the buffer passed to `init`.
	724	pub fn takeBuffer(d: *Decompress) []u8 {
	725	const buffer = d.reader.buffer;
	726	d.reader.buffer = &.{};
	727	return buffer;
	728	}
753	729
754		.buffer = Decode.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),
755		.decoder = try RangeDecoder.init(source),
756		.state = try Decode.init(allocator, params.properties, params.unpacked_size),
757		};
758		}
	730	pub fn deinit(d: *Decompress) void {
	731	const gpa = d.gpa;
	732	gpa.free(d.reader.buffer);
	733	d.buffer.deinit(gpa);
	734	d.decode.deinit(gpa);
	735	d.* = undefined;
	736	}
759	737
760		pub fn reader(self: *Self) Reader {
761		return .{ .context = self };
762		}
	738	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
	739	_ = data;
	740	return readIndirect(r);
	741	}
763	742
764		pub fn deinit(self: *Self) void {
765		self.to_read.deinit(self.allocator);
766		self.buffer.deinit(self.allocator);
767		self.state.deinit(self.allocator);
768		self.* = undefined;
769		}
	743	fn stream(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
	744	_ = w;
	745	_ = limit;
	746	return readIndirect(r);
	747	}
770	748
771		pub fn read(self: *Self, output: []u8) Error!usize {
772		const writer = self.to_read.writer(self.allocator);
773		while (self.to_read.items.len < output.len) {
774		switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {
775		.continue_ => {},
776		.finished => {
777		try self.buffer.finish(writer);
778		break;
779		},
780		}
781		}
782		const input = self.to_read.items;
783		const n = @min(input.len, output.len);
784		@memcpy(output[0..n], input[0..n]);
785		std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);
786		self.to_read.shrinkRetainingCapacity(input.len - n);
787		return n;
788		}
789		};
790		}
	749	fn readIndirect(r: *Reader) Reader.Error!usize {
	750	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	751	const gpa = d.gpa;
	752	var allocating = Writer.Allocating.initOwnedSlice(gpa, r.buffer);
	753	allocating.writer.end = r.end;
	754	defer r.end = allocating.writer.end;
	755	if (d.decode.state == math.maxInt(usize)) return error.EndOfStream;
	756	d.decode.process(d.input, &allocating, &d.buffer, &d.range_decoder) catch \|err\| switch (err) {
	757	error.WriteFailed => {
	758	d.err = error.OutOfMemory;
	759	return error.ReadFailed;
	760	},
	761	error.EndOfStream => {
	762	d.err = error.EndOfStream;
	763	return error.ReadFailed;
	764	},
	765	else => \|e\| {
	766	d.err = e;
	767	return error.ReadFailed;
	768	},
	769	};
	770	return 0;
	771	}
	772	};
791	773
792	774	test {
793	775	_ = @import("lzma/test.zig");

lib/std/compress/lzma/test.zig+14-12

...	...	@@ -1,19 +1,19 @@
1	1	const std = @import("../../std.zig");
2		const lzma = @import("../lzma.zig");
	2	const lzma = std.compress.lzma;
3	3
4	4	fn testDecompress(compressed: []const u8) ![]u8 {
5		const allocator = std.testing.allocator;
6		var stream = std.io.fixedBufferStream(compressed);
7		var decompressor = try lzma.decompress(allocator, stream.reader());
	5	const gpa = std.testing.allocator;
	6	var stream: std.Io.Reader = .fixed(compressed);
	7
	8	var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
8	9	defer decompressor.deinit();
9		const reader = decompressor.reader();
10		return reader.readAllAlloc(allocator, std.math.maxInt(usize));
	10	return decompressor.reader.allocRemaining(gpa, .unlimited);
11	11	}
12	12
13	13	fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
14		const allocator = std.testing.allocator;
	14	const gpa = std.testing.allocator;
15	15	const decomp = try testDecompress(compressed);
16		defer allocator.free(decomp);
	16	defer gpa.free(decomp);
17	17	try std.testing.expectEqualSlices(u8, expected, decomp);
18	18	}
19	19
...	...	@@ -89,11 +89,13 @@ test "too small uncompressed size in header" {
89	89	}
90	90
91	91	test "reading one byte" {
	92	const gpa = std.testing.allocator;
92	93	const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
93		var stream = std.io.fixedBufferStream(compressed);
94		var decompressor = try lzma.decompress(std.testing.allocator, stream.reader());
	94	var stream: std.Io.Reader = .fixed(compressed);
	95	var decompressor = try lzma.Decompress.initOptions(&stream, gpa, &.{}, .{}, std.math.maxInt(u32));
95	96	defer decompressor.deinit();
96	97
97		var buffer = [1]u8{0};
98		_ = try decompressor.read(buffer[0..]);
	98	var buffer: [1]u8 = undefined;
	99	try decompressor.reader.readSliceAll(&buffer);
	100	try std.testing.expectEqual(72, buffer[0]);
99	101	}

lib/std/compress/lzma2.zig+39-44

...	...	@@ -2,6 +2,8 @@ const std = @import("../std.zig");
2	2	const Allocator = std.mem.Allocator;
3	3	const ArrayList = std.ArrayList;
4	4	const lzma = std.compress.lzma;
	5	const Writer = std.Io.Writer;
	6	const Reader = std.Io.Reader;
5	7
6	8	/// An accumulating buffer for LZ sequences
7	9	pub const LzAccumBuffer = struct {
...	...	@@ -14,30 +16,28 @@ pub const LzAccumBuffer = struct {
14	16	/// Total number of bytes sent through the buffer
15	17	len: usize,
16	18
17		const Self = @This();
18
19		pub fn init(memlimit: usize) Self {
20		return Self{
	19	pub fn init(memlimit: usize) LzAccumBuffer {
	20	return .{
21	21	.buf = .{},
22	22	.memlimit = memlimit,
23	23	.len = 0,
24	24	};
25	25	}
26	26
27		pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
	27	pub fn appendByte(self: *LzAccumBuffer, allocator: Allocator, byte: u8) !void {
28	28	try self.buf.append(allocator, byte);
29	29	self.len += 1;
30	30	}
31	31
32	32	/// Reset the internal dictionary
33		pub fn reset(self: *Self, writer: anytype) !void {
	33	pub fn reset(self: LzAccumBuffer, writer: Writer) !void {
34	34	try writer.writeAll(self.buf.items);
35	35	self.buf.clearRetainingCapacity();
36	36	self.len = 0;
37	37	}
38	38
39	39	/// Retrieve the last byte or return a default
40		pub fn lastOr(self: Self, lit: u8) u8 {
	40	pub fn lastOr(self: LzAccumBuffer, lit: u8) u8 {
41	41	const buf_len = self.buf.items.len;
42	42	return if (buf_len == 0)
43	43	lit
...	...	@@ -46,7 +46,7 @@ pub const LzAccumBuffer = struct {
46	46	}
47	47
48	48	/// Retrieve the n-th last byte
49		pub fn lastN(self: Self, dist: usize) !u8 {
	49	pub fn lastN(self: LzAccumBuffer, dist: usize) !u8 {
50	50	const buf_len = self.buf.items.len;
51	51	if (dist > buf_len) {
52	52	return error.CorruptInput;
...	...	@@ -57,10 +57,10 @@ pub const LzAccumBuffer = struct {
57	57
58	58	/// Append a literal
59	59	pub fn appendLiteral(
60		self: *Self,
	60	self: *LzAccumBuffer,
61	61	allocator: Allocator,
62	62	lit: u8,
63		writer: anytype,
	63	writer: *Writer,
64	64	) !void {
65	65	_ = writer;
66	66	if (self.len >= self.memlimit) {
...	...	@@ -72,11 +72,11 @@ pub const LzAccumBuffer = struct {
72	72
73	73	/// Fetch an LZ sequence (length, distance) from inside the buffer
74	74	pub fn appendLz(
75		self: *Self,
	75	self: *LzAccumBuffer,
76	76	allocator: Allocator,
77	77	len: usize,
78	78	dist: usize,
79		writer: anytype,
	79	writer: *Writer,
80	80	) !void {
81	81	_ = writer;
82	82
...	...	@@ -95,23 +95,23 @@ pub const LzAccumBuffer = struct {
95	95	self.len += len;
96	96	}
97	97
98		pub fn finish(self: *Self, writer: anytype) !void {
	98	pub fn finish(self: LzAccumBuffer, writer: Writer) !void {
99	99	try writer.writeAll(self.buf.items);
100	100	self.buf.clearRetainingCapacity();
101	101	}
102	102
103		pub fn deinit(self: *Self, allocator: Allocator) void {
	103	pub fn deinit(self: *LzAccumBuffer, allocator: Allocator) void {
104	104	self.buf.deinit(allocator);
105	105	self.* = undefined;
106	106	}
107	107	};
108	108
109	109	pub const Decode = struct {
110		lzma_state: lzma.Decode,
	110	lzma_decode: lzma.Decode,
111	111
112	112	pub fn init(allocator: Allocator) !Decode {
113	113	return Decode{
114		.lzma_state = try lzma.Decode.init(
	114	.lzma_decode = try lzma.Decode.init(
115	115	allocator,
116	116	.{
117	117	.lc = 0,
...	...	@@ -124,15 +124,15 @@ pub const Decode = struct {
124	124	}
125	125
126	126	pub fn deinit(self: *Decode, allocator: Allocator) void {
127		self.lzma_state.deinit(allocator);
	127	self.lzma_decode.deinit(allocator);
128	128	self.* = undefined;
129	129	}
130	130
131	131	pub fn decompress(
132	132	self: *Decode,
133	133	allocator: Allocator,
134		reader: anytype,
135		writer: anytype,
	134	reader: *Reader,
	135	writer: *Writer,
136	136	) !void {
137	137	var accum = LzAccumBuffer.init(std.math.maxInt(usize));
138	138	defer accum.deinit(allocator);
...	...	@@ -154,8 +154,8 @@ pub const Decode = struct {
154	154	fn parseLzma(
155	155	self: *Decode,
156	156	allocator: Allocator,
157		reader: anytype,
158		writer: anytype,
	157	reader: *Reader,
	158	writer: *Writer,
159	159	accum: *LzAccumBuffer,
160	160	status: u8,
161	161	) !void {
...	...	@@ -210,7 +210,7 @@ pub const Decode = struct {
210	210	}
211	211
212	212	if (reset.state) {
213		var new_props = self.lzma_state.lzma_props;
	213	var new_props = self.lzma_decode.properties;
214	214
215	215	if (reset.props) {
216	216	var props = try reader.readByte();
...	...	@@ -231,16 +231,16 @@ pub const Decode = struct {
231	231	new_props = .{ .lc = lc, .lp = lp, .pb = pb };
232	232	}
233	233
234		try self.lzma_state.resetState(allocator, new_props);
	234	try self.lzma_decode.resetState(allocator, new_props);
235	235	}
236	236
237		self.lzma_state.unpacked_size = unpacked_size + accum.len;
	237	self.lzma_decode.unpacked_size = unpacked_size + accum.len;
238	238
239	239	var counter = std.io.countingReader(reader);
240	240	const counter_reader = counter.reader();
241	241
242	242	var rangecoder = try lzma.RangeDecoder.init(counter_reader);
243		while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
	243	while (try self.lzma_decode.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
244	244
245	245	if (counter.bytes_read != packed_size) {
246	246	return error.CorruptInput;
...	...	@@ -249,8 +249,8 @@ pub const Decode = struct {
249	249
250	250	fn parseUncompressed(
251	251	allocator: Allocator,
252		reader: anytype,
253		writer: anytype,
	252	reader: *Reader,
	253	writer: *Writer,
254	254	accum: *LzAccumBuffer,
255	255	reset_dict: bool,
256	256	) !void {
...	...	@@ -267,24 +267,19 @@ pub const Decode = struct {
267	267	}
268	268	};
269	269
270		pub fn decompress(
271		allocator: Allocator,
272		reader: anytype,
273		writer: anytype,
274		) !void {
275		var decoder = try Decode.init(allocator);
276		defer decoder.deinit(allocator);
277		return decoder.decompress(allocator, reader, writer);
278		}
279
280		test {
	270	test "decompress hello world stream" {
281	271	const expected = "Hello\nWorld!\n";
282	272	const compressed = &[_]u8{ 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02, 0x00, 0x06, 0x57, 0x6F, 0x72, 0x6C, 0x64, 0x21, 0x0A, 0x00 };
283	273
284		const allocator = std.testing.allocator;
285		var decomp = std.array_list.Managed(u8).init(allocator);
286		defer decomp.deinit();
287		var stream = std.io.fixedBufferStream(compressed);
288		try decompress(allocator, stream.reader(), decomp.writer());
289		try std.testing.expectEqualSlices(u8, expected, decomp.items);
	274	const gpa = std.testing.allocator;
	275
	276	var stream: std.Io.Reader = .fixed(compressed);
	277
	278	var decode = try Decode.init(gpa, &stream);
	279	defer decode.deinit(gpa);
	280
	281	const result = try decode.reader.allocRemaining(gpa, .unlimited);
	282	defer gpa.free(result);
	283
	284	try std.testing.expectEqualStrings(expected, result);
290	285	}

lib/std/compress/xz.zig+1-365

...	...	@@ -1,368 +1,4 @@
1		const std = @import("std");
2		const Allocator = std.mem.Allocator;
3		const ArrayList = std.ArrayList;
4		const Crc32 = std.hash.Crc32;
5		const Crc64 = std.hash.crc.Crc64Xz;
6		const Sha256 = std.crypto.hash.sha2.Sha256;
7		const lzma2 = std.compress.lzma2;
8
9		pub const Check = enum(u4) {
10		none = 0x00,
11		crc32 = 0x01,
12		crc64 = 0x04,
13		sha256 = 0x0A,
14		_,
15		};
16
17		fn readStreamFlags(reader: anytype, check: *Check) !void {
18		const reserved1 = try reader.readByte();
19		if (reserved1 != 0) return error.CorruptInput;
20		const byte = try reader.readByte();
21		if ((byte >> 4) != 0) return error.CorruptInput;
22		check.* = @enumFromInt(@as(u4, @truncate(byte)));
23		}
24
25		pub fn decompress(allocator: Allocator, reader: anytype) !Decompress(@TypeOf(reader)) {
26		return Decompress(@TypeOf(reader)).init(allocator, reader);
27		}
28
29		pub fn Decompress(comptime ReaderType: type) type {
30		return struct {
31		const Self = @This();
32
33		pub const Error = ReaderType.Error \|\| Decoder(ReaderType).Error;
34		pub const Reader = std.io.GenericReader(*Self, Error, read);
35
36		allocator: Allocator,
37		block_decoder: Decoder(ReaderType),
38		in_reader: ReaderType,
39
40		fn init(allocator: Allocator, source: ReaderType) !Self {
41		const magic = try source.readBytesNoEof(6);
42		if (!std.mem.eql(u8, &magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
43		return error.BadHeader;
44
45		var check: Check = undefined;
46		const hash_a = blk: {
47		var hasher = hashedReader(source, Crc32.init());
48		try readStreamFlags(hasher.reader(), &check);
49		break :blk hasher.hasher.final();
50		};
51
52		const hash_b = try source.readInt(u32, .little);
53		if (hash_a != hash_b)
54		return error.WrongChecksum;
55
56		return Self{
57		.allocator = allocator,
58		.block_decoder = try decoder(allocator, source, check),
59		.in_reader = source,
60		};
61		}
62
63		pub fn deinit(self: *Self) void {
64		self.block_decoder.deinit();
65		}
66
67		pub fn reader(self: *Self) Reader {
68		return .{ .context = self };
69		}
70
71		pub fn read(self: *Self, buffer: []u8) Error!usize {
72		if (buffer.len == 0)
73		return 0;
74
75		const r = try self.block_decoder.read(buffer);
76		if (r != 0)
77		return r;
78
79		const index_size = blk: {
80		var hasher = hashedReader(self.in_reader, Crc32.init());
81		hasher.hasher.update(&[1]u8{0x00});
82
83		var counter = std.io.countingReader(hasher.reader());
84		counter.bytes_read += 1;
85
86		const counting_reader = counter.reader();
87
88		const record_count = try std.leb.readUleb128(u64, counting_reader);
89		if (record_count != self.block_decoder.block_count)
90		return error.CorruptInput;
91
92		var i: usize = 0;
93		while (i < record_count) : (i += 1) {
94		// TODO: validate records
95		_ = try std.leb.readUleb128(u64, counting_reader);
96		_ = try std.leb.readUleb128(u64, counting_reader);
97		}
98
99		while (counter.bytes_read % 4 != 0) {
100		if (try counting_reader.readByte() != 0)
101		return error.CorruptInput;
102		}
103
104		const hash_a = hasher.hasher.final();
105		const hash_b = try counting_reader.readInt(u32, .little);
106		if (hash_a != hash_b)
107		return error.WrongChecksum;
108
109		break :blk counter.bytes_read;
110		};
111
112		const hash_a = try self.in_reader.readInt(u32, .little);
113
114		const hash_b = blk: {
115		var hasher = hashedReader(self.in_reader, Crc32.init());
116		const hashed_reader = hasher.reader();
117
118		const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
119		if (backward_size != index_size)
120		return error.CorruptInput;
121
122		var check: Check = undefined;
123		try readStreamFlags(hashed_reader, &check);
124
125		break :blk hasher.hasher.final();
126		};
127
128		if (hash_a != hash_b)
129		return error.WrongChecksum;
130
131		const magic = try self.in_reader.readBytesNoEof(2);
132		if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
133		return error.CorruptInput;
134
135		return 0;
136		}
137		};
138		}
139
140		pub fn HashedReader(ReaderType: type, HasherType: type) type {
141		return struct {
142		child_reader: ReaderType,
143		hasher: HasherType,
144
145		pub const Error = ReaderType.Error;
146		pub const Reader = std.io.GenericReader(*@This(), Error, read);
147
148		pub fn read(self: *@This(), buf: []u8) Error!usize {
149		const amt = try self.child_reader.read(buf);
150		self.hasher.update(buf[0..amt]);
151		return amt;
152		}
153
154		pub fn reader(self: *@This()) Reader {
155		return .{ .context = self };
156		}
157		};
158		}
159
160		pub fn hashedReader(
161		reader: anytype,
162		hasher: anytype,
163		) HashedReader(@TypeOf(reader), @TypeOf(hasher)) {
164		return .{ .child_reader = reader, .hasher = hasher };
165		}
166
167		const DecodeError = error{
168		CorruptInput,
169		EndOfStream,
170		EndOfStreamWithNoError,
171		WrongChecksum,
172		Unsupported,
173		Overflow,
174		};
175
176		pub fn decoder(allocator: Allocator, reader: anytype, check: Check) !Decoder(@TypeOf(reader)) {
177		return Decoder(@TypeOf(reader)).init(allocator, reader, check);
178		}
179
180		pub fn Decoder(comptime ReaderType: type) type {
181		return struct {
182		const Self = @This();
183		pub const Error =
184		ReaderType.Error \|\|
185		DecodeError \|\|
186		Allocator.Error;
187		pub const Reader = std.io.GenericReader(*Self, Error, read);
188
189		allocator: Allocator,
190		inner_reader: ReaderType,
191		check: Check,
192		err: ?Error,
193		to_read: ArrayList(u8),
194		read_pos: usize,
195		block_count: usize,
196
197		fn init(allocator: Allocator, in_reader: ReaderType, check: Check) !Self {
198		return Self{
199		.allocator = allocator,
200		.inner_reader = in_reader,
201		.check = check,
202		.err = null,
203		.to_read = .{},
204		.read_pos = 0,
205		.block_count = 0,
206		};
207		}
208
209		pub fn deinit(self: *Self) void {
210		self.to_read.deinit(self.allocator);
211		}
212
213		pub fn reader(self: *Self) Reader {
214		return .{ .context = self };
215		}
216
217		pub fn read(self: *Self, output: []u8) Error!usize {
218		while (true) {
219		const unread_len = self.to_read.items.len - self.read_pos;
220		if (unread_len > 0) {
221		const n = @min(unread_len, output.len);
222		@memcpy(output[0..n], self.to_read.items[self.read_pos..][0..n]);
223		self.read_pos += n;
224		return n;
225		}
226		if (self.err) \|e\| {
227		if (e == DecodeError.EndOfStreamWithNoError) {
228		return 0;
229		}
230		return e;
231		}
232		if (self.read_pos > 0) {
233		self.to_read.shrinkRetainingCapacity(0);
234		self.read_pos = 0;
235		}
236		self.readBlock() catch \|e\| {
237		self.err = e;
238		};
239		}
240		}
241
242		fn readBlock(self: *Self) Error!void {
243		var block_counter = std.io.countingReader(self.inner_reader);
244		const block_reader = block_counter.reader();
245
246		var packed_size: ?u64 = null;
247		var unpacked_size: ?u64 = null;
248
249		// Block Header
250		{
251		var header_hasher = hashedReader(block_reader, Crc32.init());
252		const header_reader = header_hasher.reader();
253
254		const header_size = @as(u64, try header_reader.readByte()) * 4;
255		if (header_size == 0)
256		return error.EndOfStreamWithNoError;
257
258		const Flags = packed struct(u8) {
259		last_filter_index: u2,
260		reserved: u4,
261		has_packed_size: bool,
262		has_unpacked_size: bool,
263		};
264
265		const flags = @as(Flags, @bitCast(try header_reader.readByte()));
266		const filter_count = @as(u3, flags.last_filter_index) + 1;
267		if (filter_count > 1)
268		return error.Unsupported;
269
270		if (flags.has_packed_size)
271		packed_size = try std.leb.readUleb128(u64, header_reader);
272
273		if (flags.has_unpacked_size)
274		unpacked_size = try std.leb.readUleb128(u64, header_reader);
275
276		const FilterId = enum(u64) {
277		lzma2 = 0x21,
278		_,
279		};
280
281		const filter_id = @as(
282		FilterId,
283		@enumFromInt(try std.leb.readUleb128(u64, header_reader)),
284		);
285
286		if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
287		return error.CorruptInput;
288
289		if (filter_id != .lzma2)
290		return error.Unsupported;
291
292		const properties_size = try std.leb.readUleb128(u64, header_reader);
293		if (properties_size != 1)
294		return error.CorruptInput;
295
296		// TODO: use filter properties
297		_ = try header_reader.readByte();
298
299		while (block_counter.bytes_read != header_size) {
300		if (try header_reader.readByte() != 0)
301		return error.CorruptInput;
302		}
303
304		const hash_a = header_hasher.hasher.final();
305		const hash_b = try header_reader.readInt(u32, .little);
306		if (hash_a != hash_b)
307		return error.WrongChecksum;
308		}
309
310		// Compressed Data
311		var packed_counter = std.io.countingReader(block_reader);
312		try lzma2.decompress(
313		self.allocator,
314		packed_counter.reader(),
315		self.to_read.writer(self.allocator),
316		);
317
318		if (packed_size) \|s\| {
319		if (s != packed_counter.bytes_read)
320		return error.CorruptInput;
321		}
322
323		const unpacked_bytes = self.to_read.items;
324		if (unpacked_size) \|s\| {
325		if (s != unpacked_bytes.len)
326		return error.CorruptInput;
327		}
328
329		// Block Padding
330		while (block_counter.bytes_read % 4 != 0) {
331		if (try block_reader.readByte() != 0)
332		return error.CorruptInput;
333		}
334
335		switch (self.check) {
336		.none => {},
337		.crc32 => {
338		const hash_a = Crc32.hash(unpacked_bytes);
339		const hash_b = try self.inner_reader.readInt(u32, .little);
340		if (hash_a != hash_b)
341		return error.WrongChecksum;
342		},
343		.crc64 => {
344		const hash_a = Crc64.hash(unpacked_bytes);
345		const hash_b = try self.inner_reader.readInt(u64, .little);
346		if (hash_a != hash_b)
347		return error.WrongChecksum;
348		},
349		.sha256 => {
350		var hash_a: [Sha256.digest_length]u8 = undefined;
351		Sha256.hash(unpacked_bytes, &hash_a, .{});
352
353		var hash_b: [Sha256.digest_length]u8 = undefined;
354		try self.inner_reader.readNoEof(&hash_b);
355
356		if (!std.mem.eql(u8, &hash_a, &hash_b))
357		return error.WrongChecksum;
358		},
359		else => return error.Unsupported,
360		}
361
362		self.block_count += 1;
363		}
364		};
365		}
	1	pub const Decompress = @import("xz/Decompress.zig");
366	2
367	3	test {
368	4	_ = @import("xz/test.zig");

lib/std/compress/xz/Decompress.zig created+288

...	...	@@ -0,0 +1,288 @@
	1	const Decompress = @This();
	2	const std = @import("../../std.zig");
	3	const Allocator = std.mem.Allocator;
	4	const ArrayList = std.ArrayList;
	5	const Crc32 = std.hash.Crc32;
	6	const Crc64 = std.hash.crc.Crc64Xz;
	7	const Sha256 = std.crypto.hash.sha2.Sha256;
	8	const lzma2 = std.compress.lzma2;
	9	const Writer = std.Io.Writer;
	10	const Reader = std.Io.Reader;
	11
	12	/// Underlying compressed data stream to pull bytes from.
	13	input: *Reader,
	14	/// Uncompressed bytes output by this stream implementation.
	15	reader: Reader,
	16	gpa: Allocator,
	17	check: Check,
	18	block_count: usize,
	19	err: ?Error,
	20
	21	pub const Error = error{
	22	ReadFailed,
	23	OutOfMemory,
	24	CorruptInput,
	25	EndOfStream,
	26	WrongChecksum,
	27	Unsupported,
	28	Overflow,
	29	};
	30
	31	pub const Check = enum(u4) {
	32	none = 0x00,
	33	crc32 = 0x01,
	34	crc64 = 0x04,
	35	sha256 = 0x0A,
	36	_,
	37	};
	38
	39	pub const StreamFlags = packed struct(u16) {
	40	null: u8 = 0,
	41	check: Check,
	42	reserved: u4 = 0,
	43	};
	44
	45	pub const InitError = error{
	46	NotXzStream,
	47	WrongChecksum,
	48	};
	49
	50	/// XZ uses a series of LZMA2 blocks which each specify a dictionary size
	51	/// anywhere from 4K to 4G. Thus, this API dynamically allocates the dictionary
	52	/// as-needed.
	53	pub fn init(
	54	input: *Reader,
	55	gpa: Allocator,
	56	/// Decompress takes ownership of this buffer and resizes it with `gpa`.
	57	buffer: []u8,
	58	) Decompress {
	59	const magic = try input.takeBytes(6);
	60	if (!std.mem.eql(u8, &magic, &.{ 0xFD, '7', 'z', 'X', 'Z', 0x00 }))
	61	return error.NotXzStream;
	62
	63	const actual_hash = Crc32.hash(try input.peek(@sizeOf(StreamFlags)));
	64	const stream_flags = input.takeStruct(StreamFlags, .little) catch unreachable;
	65	const stored_hash = try input.readInt(u32, .little);
	66	if (actual_hash != stored_hash) return error.WrongChecksum;
	67
	68	return .{
	69	.input = input,
	70	.reader = .{
	71	.vtable = &.{
	72	.stream = stream,
	73	.readVec = readVec,
	74	},
	75	.buffer = buffer,
	76	.seek = 0,
	77	.end = 0,
	78	},
	79	.gpa = gpa,
	80	.check = stream_flags.check,
	81	.block_count = 0,
	82	.err = null,
	83	};
	84	}
	85
	86	fn stream(r: Reader, w: Writer, limit: std.Io.Limit) Reader.StreamError!usize {
	87	_ = w;
	88	_ = limit;
	89	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	90	_ = d;
	91	@panic("TODO");
	92	}
	93
	94	fn readVec(r: *Reader, data: [][]u8) Reader.Error!usize {
	95	_ = data;
	96	const d: *Decompress = @alignCast(@fieldParentPtr("reader", r));
	97	_ = d;
	98	@panic("TODO");
	99	}
	100
	101	// if (buffer.len == 0)
	102	// return 0;
	103	//
	104	// const r = try self.block_decode.read(buffer);
	105	// if (r != 0)
	106	// return r;
	107	//
	108	// const index_size = blk: {
	109	// var hasher = hashedReader(self.in_reader, Crc32.init());
	110	// hasher.hasher.update(&[1]u8{0x00});
	111	//
	112	// var counter = std.io.countingReader(hasher.reader());
	113	// counter.bytes_read += 1;
	114	//
	115	// const counting_reader = counter.reader();
	116	//
	117	// const record_count = try std.leb.readUleb128(u64, counting_reader);
	118	// if (record_count != self.block_decode.block_count)
	119	// return error.CorruptInput;
	120	//
	121	// var i: usize = 0;
	122	// while (i < record_count) : (i += 1) {
	123	// // TODO: validate records
	124	// _ = try std.leb.readUleb128(u64, counting_reader);
	125	// _ = try std.leb.readUleb128(u64, counting_reader);
	126	// }
	127	//
	128	// while (counter.bytes_read % 4 != 0) {
	129	// if (try counting_reader.readByte() != 0)
	130	// return error.CorruptInput;
	131	// }
	132	//
	133	// const hash_a = hasher.hasher.final();
	134	// const hash_b = try counting_reader.readInt(u32, .little);
	135	// if (hash_a != hash_b)
	136	// return error.WrongChecksum;
	137	//
	138	// break :blk counter.bytes_read;
	139	// };
	140	//
	141	// const hash_a = try self.in_reader.readInt(u32, .little);
	142	//
	143	// const hash_b = blk: {
	144	// var hasher = hashedReader(self.in_reader, Crc32.init());
	145	// const hashed_reader = hasher.reader();
	146	//
	147	// const backward_size = (@as(u64, try hashed_reader.readInt(u32, .little)) + 1) * 4;
	148	// if (backward_size != index_size)
	149	// return error.CorruptInput;
	150	//
	151	// var check: Check = undefined;
	152	// try readStreamFlags(hashed_reader, &check);
	153	//
	154	// break :blk hasher.hasher.final();
	155	// };
	156	//
	157	// if (hash_a != hash_b)
	158	// return error.WrongChecksum;
	159	//
	160	// const magic = try self.in_reader.readBytesNoEof(2);
	161	// if (!std.mem.eql(u8, &magic, &.{ 'Y', 'Z' }))
	162	// return error.CorruptInput;
	163	//
	164	// return 0;
	165	//}
	166
	167	//fn readBlock(self: *BlockDecode) Error!void {
	168	// var block_counter = std.io.countingReader(self.inner_reader);
	169	// const block_reader = block_counter.reader();
	170	//
	171	// var packed_size: ?u64 = null;
	172	// var unpacked_size: ?u64 = null;
	173	//
	174	// // Block Header
	175	// {
	176	// var header_hasher = hashedReader(block_reader, Crc32.init());
	177	// const header_reader = header_hasher.reader();
	178	//
	179	// const header_size = @as(u64, try header_reader.readByte()) * 4;
	180	// if (header_size == 0)
	181	// return error.EndOfStreamWithNoError;
	182	//
	183	// const Flags = packed struct(u8) {
	184	// last_filter_index: u2,
	185	// reserved: u4,
	186	// has_packed_size: bool,
	187	// has_unpacked_size: bool,
	188	// };
	189	//
	190	// const flags = @as(Flags, @bitCast(try header_reader.readByte()));
	191	// const filter_count = @as(u3, flags.last_filter_index) + 1;
	192	// if (filter_count > 1)
	193	// return error.Unsupported;
	194	//
	195	// if (flags.has_packed_size)
	196	// packed_size = try std.leb.readUleb128(u64, header_reader);
	197	//
	198	// if (flags.has_unpacked_size)
	199	// unpacked_size = try std.leb.readUleb128(u64, header_reader);
	200	//
	201	// const FilterId = enum(u64) {
	202	// lzma2 = 0x21,
	203	// _,
	204	// };
	205	//
	206	// const filter_id = @as(
	207	// FilterId,
	208	// @enumFromInt(try std.leb.readUleb128(u64, header_reader)),
	209	// );
	210	//
	211	// if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
	212	// return error.CorruptInput;
	213	//
	214	// if (filter_id != .lzma2)
	215	// return error.Unsupported;
	216	//
	217	// const properties_size = try std.leb.readUleb128(u64, header_reader);
	218	// if (properties_size != 1)
	219	// return error.CorruptInput;
	220	//
	221	// // TODO: use filter properties
	222	// _ = try header_reader.readByte();
	223	//
	224	// while (block_counter.bytes_read != header_size) {
	225	// if (try header_reader.readByte() != 0)
	226	// return error.CorruptInput;
	227	// }
	228	//
	229	// const hash_a = header_hasher.hasher.final();
	230	// const hash_b = try header_reader.readInt(u32, .little);
	231	// if (hash_a != hash_b)
	232	// return error.WrongChecksum;
	233	// }
	234	//
	235	// // Compressed Data
	236	// var packed_counter = std.io.countingReader(block_reader);
	237	// try lzma2.decompress(
	238	// self.allocator,
	239	// packed_counter.reader(),
	240	// self.to_read.writer(self.allocator),
	241	// );
	242	//
	243	// if (packed_size) \|s\| {
	244	// if (s != packed_counter.bytes_read)
	245	// return error.CorruptInput;
	246	// }
	247	//
	248	// const unpacked_bytes = self.to_read.items;
	249	// if (unpacked_size) \|s\| {
	250	// if (s != unpacked_bytes.len)
	251	// return error.CorruptInput;
	252	// }
	253	//
	254	// // Block Padding
	255	// while (block_counter.bytes_read % 4 != 0) {
	256	// if (try block_reader.readByte() != 0)
	257	// return error.CorruptInput;
	258	// }
	259	//
	260	// switch (self.check) {
	261	// .none => {},
	262	// .crc32 => {
	263	// const hash_a = Crc32.hash(unpacked_bytes);
	264	// const hash_b = try self.inner_reader.readInt(u32, .little);
	265	// if (hash_a != hash_b)
	266	// return error.WrongChecksum;
	267	// },
	268	// .crc64 => {
	269	// const hash_a = Crc64.hash(unpacked_bytes);
	270	// const hash_b = try self.inner_reader.readInt(u64, .little);
	271	// if (hash_a != hash_b)
	272	// return error.WrongChecksum;
	273	// },
	274	// .sha256 => {
	275	// var hash_a: [Sha256.digest_length]u8 = undefined;
	276	// Sha256.hash(unpacked_bytes, &hash_a, .{});
	277	//
	278	// var hash_b: [Sha256.digest_length]u8 = undefined;
	279	// try self.inner_reader.readNoEof(&hash_b);
	280	//
	281	// if (!std.mem.eql(u8, &hash_a, &hash_b))
	282	// return error.WrongChecksum;
	283	// },
	284	// else => return error.Unsupported,
	285	// }
	286	//
	287	// self.block_count += 1;
	288	//}