Commit da303bdaf1ae8717df2d4ede9e7dfb215636ae33 | zig

lib/std/coff.zig+12-14

...	...	@@ -1087,16 +1087,14 @@ pub const Coff = struct {
1087	1087	const pe_pointer_offset = 0x3C;
1088	1088	const pe_magic = "PE\x00\x00";
1089	1089
1090		var stream = std.io.fixedBufferStream(data);
1091		const reader = stream.reader();
1092		try stream.seekTo(pe_pointer_offset);
	1090	var reader: std.io.BufferedReader = undefined;
	1091	reader.initFixed(data[pe_pointer_offset..]);
1093	1092	const coff_header_offset = try reader.readInt(u32, .little);
1094		try stream.seekTo(coff_header_offset);
1095		var buf: [4]u8 = undefined;
1096		try reader.readNoEof(&buf);
1097		const is_image = mem.eql(u8, pe_magic, &buf);
	1093	reader.initFixed(data[coff_header_offset..]);
	1094	const magic = try reader.peek(4);
	1095	const is_image = mem.eql(u8, pe_magic, magic);
1098	1096
1099		var coff = @This(){
	1097	var coff: Coff = .{
1100	1098	.data = data,
1101	1099	.is_image = is_image,
1102	1100	.is_loaded = is_loaded,
...	...	@@ -1123,16 +1121,16 @@ pub const Coff = struct {
1123	1121	if (@intFromEnum(DirectoryEntry.DEBUG) >= data_dirs.len) return null;
1124	1122
1125	1123	const debug_dir = data_dirs[@intFromEnum(DirectoryEntry.DEBUG)];
1126		var stream = std.io.fixedBufferStream(self.data);
1127		const reader = stream.reader();
	1124	var reader: std.io.BufferedReader = undefined;
	1125	reader.initFixed(self.data);
1128	1126
1129	1127	if (self.is_loaded) {
1130		try stream.seekTo(debug_dir.virtual_address);
	1128	reader.initFixed(self.data[debug_dir.virtual_address..]);
1131	1129	} else {
1132	1130	// Find what section the debug_dir is in, in order to convert the RVA to a file offset
1133	1131	for (self.getSectionHeaders()) \|*sect\| {
1134	1132	if (debug_dir.virtual_address >= sect.virtual_address and debug_dir.virtual_address < sect.virtual_address + sect.virtual_size) {
1135		try stream.seekTo(sect.pointer_to_raw_data + (debug_dir.virtual_address - sect.virtual_address));
	1133	reader.initFixed(self.data[sect.pointer_to_raw_data + (debug_dir.virtual_address - sect.virtual_address) ..]);
1136	1134	break;
1137	1135	}
1138	1136	} else return error.InvalidDebugDirectory;
...	...	@@ -1143,10 +1141,10 @@ pub const Coff = struct {
1143	1141	const debug_dir_entry_count = debug_dir.size / @sizeOf(DebugDirectoryEntry);
1144	1142	var i: u32 = 0;
1145	1143	while (i < debug_dir_entry_count) : (i += 1) {
1146		const debug_dir_entry = try reader.readStruct(DebugDirectoryEntry);
	1144	const debug_dir_entry = try reader.takeStruct(DebugDirectoryEntry);
1147	1145	if (debug_dir_entry.type == .CODEVIEW) {
1148	1146	const dir_offset = if (self.is_loaded) debug_dir_entry.address_of_raw_data else debug_dir_entry.pointer_to_raw_data;
1149		try stream.seekTo(dir_offset);
	1147	reader.initFixed(self.data[dir_offset..]);
1150	1148	break;
1151	1149	}
1152	1150	} else return null;

lib/std/compress/flate.zig+11-67

...	...	@@ -68,81 +68,23 @@ pub const Container = enum {
68	68	//
69	69	// CINFO = 7, CM = 8, FLEVEL = 0b10, FDICT = 0, FCHECK = 0b11100
70	70	.zlib => &[_]u8{ 0x78, 0b10_0_11100 },
71		.raw => &{},
	71	.raw => &.{},
72	72	};
73	73	}
74	74
75		pub fn parseHeader(comptime wrap: Container, reader: *std.io.BufferedReader) !void {
76		switch (wrap) {
77		.gzip => try parseGzipHeader(reader),
78		.zlib => try parseZlibHeader(reader),
79		.raw => {},
80		}
81		}
82
83		fn parseGzipHeader(reader: *std.io.BufferedReader) !void {
84		const magic1 = try reader.read(u8);
85		const magic2 = try reader.read(u8);
86		const method = try reader.read(u8);
87		const flags = try reader.read(u8);
88		try reader.skipBytes(6); // mtime(4), xflags, os
89		if (magic1 != 0x1f or magic2 != 0x8b or method != 0x08)
90		return error.BadGzipHeader;
91		// Flags description: https://www.rfc-editor.org/rfc/rfc1952.html#page-5
92		if (flags != 0) {
93		if (flags & 0b0000_0100 != 0) { // FEXTRA
94		const extra_len = try reader.read(u16);
95		try reader.skipBytes(extra_len);
96		}
97		if (flags & 0b0000_1000 != 0) { // FNAME
98		try reader.skipStringZ();
99		}
100		if (flags & 0b0001_0000 != 0) { // FCOMMENT
101		try reader.skipStringZ();
102		}
103		if (flags & 0b0000_0010 != 0) { // FHCRC
104		try reader.skipBytes(2);
105		}
106		}
107		}
108
109		fn parseZlibHeader(reader: *std.io.BufferedReader) !void {
110		const cm = try reader.read(u4);
111		const cinfo = try reader.read(u4);
112		_ = try reader.read(u8);
113		if (cm != 8 or cinfo > 7) {
114		return error.BadZlibHeader;
115		}
116		}
117
118		pub fn parseFooter(comptime wrap: Container, hasher: Hasher(wrap), reader: std.io.BufferedReader) !void {
119		switch (wrap) {
120		.gzip => {
121		try reader.fill(0);
122		if (try reader.read(u32) != hasher.chksum()) return error.WrongGzipChecksum;
123		if (try reader.read(u32) != hasher.bytesRead()) return error.WrongGzipSize;
124		},
125		.zlib => {
126		const chksum: u32 = @byteSwap(hasher.chksum());
127		if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
128		},
129		.raw => {},
130		}
131		}
132
133	75	pub const Hasher = union(Container) {
	76	raw: void,
134	77	gzip: struct {
135	78	crc: std.hash.Crc32 = .init(),
136	79	count: usize = 0,
137	80	},
138	81	zlib: std.hash.Adler32,
139		raw: void,
140	82
141	83	pub fn init(containter: Container) Hasher {
142	84	return switch (containter) {
143	85	.gzip => .{ .gzip = .{} },
144	86	.zlib => .{ .zlib = .init() },
145		.raw => {},
	87	.raw => .raw,
146	88	};
147	89	}
148	90
...	...	@@ -288,15 +230,18 @@ test "compress/decompress" {
288	230	// compress original stream to compressed stream
289	231	{
290	232	var original: std.io.BufferedReader = undefined;
291		original.initFixed(data);
	233	original.initFixed(@constCast(data));
292	234	var compressed: std.io.BufferedWriter = undefined;
293	235	compressed.initFixed(&cmp_buf);
294		try Compress.pump(container, original.reader(), &compressed, .{ .level = level });
	236	var compress: Compress = .init(&original, .raw);
	237	var compress_br = compress.readable(&.{});
	238	const n = try compress_br.readRemaining(&compressed, .{ .level = level });
295	239	if (compressed_size == 0) {
296	240	if (container == .gzip)
297	241	print("case {d} gzip level {} compressed size: {d}\n", .{ case_no, level, compressed.pos });
298	242	compressed_size = compressed.pos;
299	243	}
	244	try testing.expectEqual(compressed_size, n);
300	245	try testing.expectEqual(compressed_size, compressed.pos);
301	246	}
302	247	// decompress compressed stream to decompressed stream
...	...	@@ -688,9 +633,7 @@ pub const match = struct {
688	633	pub const max_distance = 32768;
689	634	};
690	635
691		pub const history = struct {
692		pub const len = match.max_distance;
693		};
	636	pub const history_len = match.max_distance;
694	637
695	638	pub const lookup = struct {
696	639	pub const bits = 15;
...	...	@@ -707,7 +650,8 @@ test "zlib should not overshoot" {
707	650	0x03, 0x00, 0x8b, 0x61, 0x0f, 0xa4, 0x52, 0x5a, 0x94, 0x12,
708	651	};
709	652
710		var stream = std.io.fixedBufferStream(data[0..]);
	653	var stream: std.io.BufferedReader = undefined;
	654	stream.initFixed(&data);
711	655	const reader = stream.reader();
712	656
713	657	var dcp = Decompress.init(reader);

lib/std/compress/flate/Compress.zig+203-130

...	...	@@ -51,9 +51,37 @@ const math = std.math;
51	51	const Compress = @This();
52	52	const Token = @import("Token.zig");
53	53	const BlockWriter = @import("BlockWriter.zig");
54		const Container = std.compress.flate.Container;
	54	const flate = @import("../flate.zig");
	55	const Container = flate.Container;
55	56	const Lookup = @import("Lookup.zig");
56		const huffman = std.compress.flate.huffman;
	57	const huffman = flate.huffman;
	58
	59	lookup: Lookup = .{},
	60	tokens: Tokens = .{},
	61	/// Asserted to have a buffer capacity of at least `flate.max_window_len`.
	62	input: *std.io.BufferedReader,
	63	block_writer: BlockWriter,
	64	level: LevelArgs,
	65	hasher: Container.Hasher,
	66
	67	// Match and literal at the previous position.
	68	// Used for lazy match finding in processWindow.
	69	prev_match: ?Token = null,
	70	prev_literal: ?u8 = null,
	71
	72	pub fn readable(c: *Compress, buffer: []u8) std.io.BufferedReader {
	73	return .{
	74	.unbuffered_reader = .{
	75	.context = c,
	76	.vtable = .{
	77	.read = read,
	78	.readVec = readVec,
	79	.discard = discard,
	80	},
	81	},
	82	.buffer = buffer,
	83	};
	84	}
57	85
58	86	pub const Options = struct {
59	87	level: Level = .default,
...	...	@@ -77,10 +105,10 @@ pub const Level = enum(u4) {
77	105	best = 0xd,
78	106	};
79	107
80		// Number of tokens to accumulate in deflate before starting block encoding.
81		//
82		// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
83		// 8 and max 9 that gives 14 or 15 bits.
	108	/// Number of tokens to accumulate in deflate before starting block encoding.
	109	///
	110	/// In zlib this depends on memlevel: 6 + memlevel, where default memlevel is
	111	/// 8 and max 9 that gives 14 or 15 bits.
84	112	pub const n_tokens = 1 << 15;
85	113
86	114	/// Algorithm knobs for each level.
...	...	@@ -102,85 +130,60 @@ const LevelArgs = struct {
102	130	}
103	131	};
104	132
105		lookup: Lookup = .{},
106		tokens: Tokens = .{},
107		output: *std.io.BufferedWriter,
108		block_writer: BlockWriter,
109		level: LevelArgs,
110		hasher: Container.Hasher,
111
112		// Match and literal at the previous position.
113		// Used for lazy match finding in processWindow.
114		prev_match: ?Token = null,
115		prev_literal: ?u8 = null,
116
117		pub fn init(output: *std.io.BufferedWriter, options: Options) std.io.Writer.Error!Compress {
118		try output.writeAll(options.container.header(output));
	133	pub fn init(input: *std.io.BufferedReader, options: Options) Compress {
119	134	return .{
120		.output = output,
121		.block_writer = .init(output),
	135	.input = input,
	136	.block_writer = undefined,
122	137	.level = .get(options.level),
123	138	.hasher = .init(options.container),
	139	.state = .header,
124	140	};
125	141	}
126	142
127	143	const FlushOption = enum { none, flush, final };
128	144
129		// Process data in window and create tokens. If token buffer is full
130		// flush tokens to the token writer. In the case of `flush` or `final`
131		// option it will process all data from the window. In the `none` case
132		// it will preserve some data for the next match.
133		fn tokenize(self: *Compress, flush_opt: FlushOption) !void {
134		// flush - process all data from window
135		const should_flush = (flush_opt != .none);
136
137		// While there is data in active lookahead buffer.
138		while (self.win.activeLookahead(should_flush)) \|lh\| {
139		var step: u16 = 1; // 1 in the case of literal, match length otherwise
140		const pos: u16 = self.win.pos();
141		const literal = lh[0]; // literal at current position
142		const min_len: u16 = if (self.prev_match) \|m\| m.length() else 0;
143
144		// Try to find match at least min_len long.
145		if (self.findMatch(pos, lh, min_len)) \|match\| {
146		// Found better match than previous.
147		try self.addPrevLiteral();
148
149		// Is found match length good enough?
150		if (match.length() >= self.level.lazy) {
151		// Don't try to lazy find better match, use this.
152		step = try self.addMatch(match);
153		} else {
154		// Store this match.
155		self.prev_literal = literal;
156		self.prev_match = match;
157		}
	145	/// Process data in window and create tokens. If token buffer is full
	146	/// flush tokens to the token writer.
	147	///
	148	/// Returns number of bytes consumed from `lh`.
	149	fn tokenizeSlice(c: Compress, bw: std.io.BufferedWriter, limit: std.io.Limit, lh: []const u8) !usize {
	150	_ = bw;
	151	_ = limit;
	152	if (true) @panic("TODO");
	153	var step: u16 = 1; // 1 in the case of literal, match length otherwise
	154	const pos: u16 = c.win.pos();
	155	const literal = lh[0]; // literal at current position
	156	const min_len: u16 = if (c.prev_match) \|m\| m.length() else 0;
	157
	158	// Try to find match at least min_len long.
	159	if (c.findMatch(pos, lh, min_len)) \|match\| {
	160	// Found better match than previous.
	161	try c.addPrevLiteral();
	162
	163	// Is found match length good enough?
	164	if (match.length() >= c.level.lazy) {
	165	// Don't try to lazy find better match, use this.
	166	step = try c.addMatch(match);
158	167	} else {
159		// There is no better match at current pos then it was previous.
160		// Write previous match or literal.
161		if (self.prev_match) \|m\| {
162		// Write match from previous position.
163		step = try self.addMatch(m) - 1; // we already advanced 1 from previous position
164		} else {
165		// No match at previous position.
166		// Write previous literal if any, and remember this literal.
167		try self.addPrevLiteral();
168		self.prev_literal = literal;
169		}
	168	// Store this match.
	169	c.prev_literal = literal;
	170	c.prev_match = match;
	171	}
	172	} else {
	173	// There is no better match at current pos then it was previous.
	174	// Write previous match or literal.
	175	if (c.prev_match) \|m\| {
	176	// Write match from previous position.
	177	step = try c.addMatch(m) - 1; // we already advanced 1 from previous position
	178	} else {
	179	// No match at previous position.
	180	// Write previous literal if any, and remember this literal.
	181	try c.addPrevLiteral();
	182	c.prev_literal = literal;
170	183	}
171		// Advance window and add hashes.
172		self.windowAdvance(step, lh, pos);
173		}
174
175		if (should_flush) {
176		// In the case of flushing, last few lookahead buffers were smaller then min match len.
177		// So only last literal can be unwritten.
178		assert(self.prev_match == null);
179		try self.addPrevLiteral();
180		self.prev_literal = null;
181
182		try self.flushTokens(flush_opt);
183	184	}
	185	// Advance window and add hashes.
	186	c.windowAdvance(step, lh, pos);
184	187	}
185	188
186	189	fn windowAdvance(self: *Compress, step: u16, lh: []const u8, pos: u16) void {
...	...	@@ -226,7 +229,7 @@ fn findMatch(self: *Compress, pos: u16, lh: []const u8, min_len: u16) ?Token {
226	229	// Hot path loop!
227	230	while (prev_pos > 0 and chain > 0) : (chain -= 1) {
228	231	const distance = pos - prev_pos;
229		if (distance > std.compress.flate.match.max_distance)
	232	if (distance > flate.match.max_distance)
230	233	break;
231	234
232	235	const new_len = self.win.match(prev_pos, pos, len);
...	...	@@ -272,33 +275,6 @@ fn slide(self: *Compress) void {
272	275	self.lookup.slide(n);
273	276	}
274	277
275		/// Compresses as much data as possible, stops when the reader becomes
276		/// empty. It will introduce some output latency (reading input without
277		/// producing all output) because some data are still in internal
278		/// buffers.
279		///
280		/// It is up to the caller to call flush (if needed) or finish (required)
281		/// when is need to output any pending data or complete stream.
282		///
283		pub fn compress(self: *Compress, reader: anytype) !void {
284		while (true) {
285		// Fill window from reader
286		const buf = self.win.writable();
287		if (buf.len == 0) {
288		try self.tokenize(.none);
289		self.slide();
290		continue;
291		}
292		const n = try reader.readAll(buf);
293		self.hasher.update(buf[0..n]);
294		self.win.written(n);
295		// Process window
296		try self.tokenize(.none);
297		// Exit when no more data in reader
298		if (n < buf.len) break;
299		}
300		}
301
302	278	/// Flushes internal buffers to the output writer. Outputs empty stored
303	279	/// block to sync bit stream to the byte boundary, so that the
304	280	/// decompressor can get all input data available so far.
...	...	@@ -311,8 +287,8 @@ pub fn compress(self: *Compress, reader: anytype) !void {
311	287	/// stored block that is three zero bits plus filler bits to the next
312	288	/// byte, followed by four bytes (00 00 ff ff).
313	289	///
314		pub fn flush(self: *Compress) !void {
315		try self.tokenize(.flush);
	290	pub fn flush(c: *Compress) !void {
	291	try c.tokenize(.flush);
316	292	}
317	293
318	294	/// Completes deflate bit stream by writing any pending data as deflate
...	...	@@ -320,9 +296,9 @@ pub fn flush(self: *Compress) !void {
320	296	/// the compressor as a signal that next block has to have final bit
321	297	/// set.
322	298	///
323		pub fn finish(self: *Compress) !void {
324		try self.tokenize(.final);
325		try self.hasher.writeFooter(self.output);
	299	pub fn finish(c: *Compress) !void {
	300	_ = c;
	301	@panic("TODO");
326	302	}
327	303
328	304	/// Use another writer while preserving history. Most probably flush
...	...	@@ -437,24 +413,6 @@ fn SimpleCompressor(
437	413	}
438	414	self.wp = 0;
439	415	}
440
441		// Writes all data from the input reader of uncompressed data.
442		// It is up to the caller to call flush or finish if there is need to
443		// output compressed blocks.
444		pub fn compress(self: *Self, reader: anytype) !void {
445		while (true) {
446		// read from rdr into buffer
447		const buf = self.buffer[self.wp..];
448		if (buf.len == 0) {
449		try self.flushBuffer(false);
450		continue;
451		}
452		const n = try reader.readAll(buf);
453		self.hasher.update(buf[0..n]);
454		self.wp += n;
455		if (n < buf.len) break; // no more data in reader
456		}
457		}
458	416	};
459	417	}
460	418
...	...	@@ -811,6 +769,119 @@ fn byFreq(context: void, a: LiteralNode, b: LiteralNode) bool {
811	769	return a.freq < b.freq;
812	770	}
813	771
	772	fn read(
	773	context: ?*anyopaque,
	774	bw: *std.io.BufferedWriter,
	775	limit: std.io.Reader.Limit,
	776	) std.io.Reader.RwError!usize {
	777	const c: *Compress = @ptrCast(@alignCast(context));
	778	switch (c.state) {
	779	.header => \|i\| {
	780	const header = c.hasher.container().header();
	781	const n = try bw.write(header[i..]);
	782	if (header.len - i - n == 0) {
	783	c.state = .middle;
	784	} else {
	785	c.state.header += n;
	786	}
	787	return n;
	788	},
	789	.middle => {
	790	c.input.fillMore() catch \|err\| switch (err) {
	791	error.EndOfStream => {
	792	c.state = .final;
	793	return 0;
	794	},
	795	else => \|e\| return e,
	796	};
	797	const buffer_contents = c.input.bufferContents();
	798	const min_lookahead = flate.match.min_length + flate.match.max_length;
	799	const history_plus_lookahead_len = flate.history_len + min_lookahead;
	800	if (buffer_contents.len < history_plus_lookahead_len) return 0;
	801	const lookahead = buffer_contents[flate.history_len..];
	802	const start = bw.count;
	803	const n = try c.tokenizeSlice(bw, limit, lookahead) catch \|err\| switch (err) {
	804	error.WriteFailed => return error.WriteFailed,
	805	};
	806	c.hasher.update(lookahead[0..n]);
	807	c.input.toss(n);
	808	return bw.count - start;
	809	},
	810	.final => {
	811	const buffer_contents = c.input.bufferContents();
	812	const start = bw.count;
	813	const n = c.tokenizeSlice(bw, limit, buffer_contents) catch \|err\| switch (err) {
	814	error.WriteFailed => return error.WriteFailed,
	815	};
	816	if (buffer_contents.len - n == 0) {
	817	c.hasher.update(buffer_contents);
	818	c.input.tossAll();
	819	{
	820	// In the case of flushing, last few lookahead buffers were
	821	// smaller than min match len, so only last literal can be
	822	// unwritten.
	823	assert(c.prev_match == null);
	824	try c.addPrevLiteral();
	825	c.prev_literal = null;
	826
	827	try c.flushTokens(.final);
	828	}
	829	switch (c.hasher) {
	830	.gzip => \|*gzip\| {
	831	// GZIP 8 bytes footer
	832	// - 4 bytes, CRC32 (CRC-32)
	833	// - 4 bytes, ISIZE (Input SIZE) - size of the original (uncompressed) input data modulo 2^32
	834	comptime assert(c.footer_buffer.len == 8);
	835	std.mem.writeInt(u32, c.footer_buffer[0..4], gzip.final(), .little);
	836	std.mem.writeInt(u32, c.footer_buffer[4..8], gzip.bytes_read, .little);
	837	c.state = .{ .footer = 0 };
	838	},
	839	.zlib => \|*zlib\| {
	840	// ZLIB (RFC 1950) is big-endian, unlike GZIP (RFC 1952).
	841	// 4 bytes of ADLER32 (Adler-32 checksum)
	842	// Checksum value of the uncompressed data (excluding any
	843	// dictionary data) computed according to Adler-32
	844	// algorithm.
	845	comptime assert(c.footer_buffer.len == 8);
	846	std.mem.writeInt(u32, c.footer_buffer[4..8], zlib.final, .big);
	847	c.state = .{ .footer = 4 };
	848	},
	849	.raw => {
	850	c.state = .ended;
	851	},
	852	}
	853	}
	854	return bw.count - start;
	855	},
	856	.ended => return error.EndOfStream,
	857	.footer => \|i\| {
	858	const remaining = c.footer_buffer[i..];
	859	const n = try bw.write(limit.slice(remaining));
	860	c.state = if (n == remaining) .ended else .{ .footer = i - n };
	861	return n;
	862	},
	863	}
	864	}
	865
	866	fn readVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
	867	var bw: std.io.BufferedWriter = undefined;
	868	bw.initVec(data);
	869	return read(context, &bw, .countVec(data)) catch \|err\| switch (err) {
	870	error.WriteFailed => unreachable, // Prevented by the limit.
	871	else => \|e\| return e,
	872	};
	873	}
	874
	875	fn discard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
	876	var trash_buffer: [64]u8 = undefined;
	877	var null_writer: std.io.Writer.Null = undefined;
	878	var bw = null_writer.writer().buffered(&trash_buffer);
	879	return read(context, &bw, limit) catch \|err\| switch (err) {
	880	error.WriteFailed => unreachable,
	881	else => \|e\| return e,
	882	};
	883	}
	884
814	885	test "generate a Huffman code from an array of frequencies" {
815	886	var freqs: [19]u16 = [_]u16{
816	887	8, // 0
...	...	@@ -1099,7 +1170,8 @@ test "file tokenization" {
1099	1170	const data = case.data;
1100	1171
1101	1172	for (levels, 0..) \|level, i\| { // for each compression level
1102		var original = io.fixedBufferStream(data);
	1173	var original: std.io.BufferedReader = undefined;
	1174	original.initFixed(data);
1103	1175
1104	1176	// buffer for decompressed data
1105	1177	var al = std.ArrayList(u8).init(testing.allocator);
...	...	@@ -1173,21 +1245,22 @@ test "store simple compressor" {
1173	1245	//0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x21,
1174	1246	};
1175	1247
1176		var fbs = std.io.fixedBufferStream(data);
	1248	var fbs: std.io.BufferedReader = undefined;
	1249	fbs.initFixed(data);
1177	1250	var al = std.ArrayList(u8).init(testing.allocator);
1178	1251	defer al.deinit();
1179	1252
1180	1253	var cmp = try store.compressor(.raw, al.writer());
1181		try cmp.compress(fbs.reader());
	1254	try cmp.compress(&fbs);
1182	1255	try cmp.finish();
1183	1256	try testing.expectEqualSlices(u8, &expected, al.items);
1184	1257
1185		fbs.reset();
	1258	fbs.initFixed(data);
1186	1259	try al.resize(0);
1187	1260
1188	1261	// huffman only compresoor will also emit store block for this small sample
1189	1262	var hc = try huffman.compressor(.raw, al.writer());
1190		try hc.compress(fbs.reader());
	1263	try hc.compress(&fbs);
1191	1264	try hc.finish();
1192	1265	try testing.expectEqualSlices(u8, &expected, al.items);
1193	1266	}

lib/std/compress/flate/Decompress.zig+329-273

...	...	@@ -22,32 +22,38 @@ const flate = std.compress.flate;
22	22	const Container = flate.Container;
23	23	const Token = @import("Token.zig");
24	24	const testing = std.testing;
	25	const Decompress = @This();
25	26
26	27	input: *std.io.BufferedReader,
27	28	// Hashes, produces checksum, of uncompressed data for gzip/zlib footer.
28		hasher: Container.Hasher(),
	29	hasher: Container.Hasher,
29	30
30	31	// dynamic block huffman code decoders
31	32	lit_dec: LiteralDecoder,
32	33	dst_dec: DistanceDecoder,
33	34
34	35	// current read state
35		bfinal: u1,
36		block_type: u2,
37		state: ReadState,
	36	final_block: bool,
	37	state: State,
	38
	39	read_err: ?Error,
38	40
39		read_err: Error!void,
	41	const BlockType = enum(u2) {
	42	stored = 0,
	43	fixed = 1,
	44	dynamic = 2,
	45	};
40	46
41		const ReadState = enum {
	47	const State = union(enum) {
42	48	protocol_header,
43	49	block_header,
44		block,
	50	stored_block: u16,
	51	fixed_block,
	52	dynamic_block,
45	53	protocol_footer,
46	54	end,
47	55	};
48	56
49		const Decompress = @This();
50
51	57	pub const Error = Container.Error \|\| error{
52	58	InvalidCode,
53	59	InvalidMatch,
...	...	@@ -61,71 +67,25 @@ pub const Error = Container.Error \|\| error{
61	67	MissingEndOfBlockCode,
62	68	};
63	69
64		pub fn init(input: *std.io.BufferedReader) Decompress {
	70	pub fn init(input: *std.io.BufferedReader, container: Container) Decompress {
65	71	return .{
66	72	.input = input,
67		.hasher = .{},
	73	.hasher = .init(container),
68	74	.lit_dec = .{},
69	75	.dst_dec = .{},
70		.bfinal = 0,
71		.block_type = 0b11,
	76	.final_block = false,
72	77	.state = .protocol_header,
73		.read_err = {},
	78	.read_err = null,
74	79	};
75	80	}
76	81
77		fn blockHeader(self: *Decompress) Error!void {
78		self.bfinal = try self.bits.read(u1);
79		self.block_type = try self.bits.read(u2);
80		}
81
82		fn storedBlock(self: *Decompress) !bool {
83		self.bits.alignToByte(); // skip padding until byte boundary
84		// everything after this is byte aligned in stored block
85		var len = try self.bits.read(u16);
86		const nlen = try self.bits.read(u16);
87		if (len != ~nlen) return error.WrongStoredBlockNlen;
88
89		while (len > 0) {
90		const buf = self.hist.getWritable(len);
91		try self.bits.readAll(buf);
92		len -= @intCast(buf.len);
93		}
94		return true;
95		}
96
97		fn fixedBlock(self: *Decompress) !bool {
98		while (!self.hist.full()) {
99		const code = try self.bits.readFixedCode();
100		switch (code) {
101		0...255 => self.hist.write(@intCast(code)),
102		256 => return true, // end of block
103		257...285 => try self.fixedDistanceCode(@intCast(code - 257)),
104		else => return error.InvalidCode,
105		}
106		}
107		return false;
108		}
109
110		// Handles fixed block non literal (length) code.
111		// Length code is followed by 5 bits of distance code.
112		fn fixedDistanceCode(self: *Decompress, code: u8) !void {
113		try self.bits.fill(5 + 5 + 13);
114		const length = try self.decodeLength(code);
115		const distance = try self.decodeDistance(try self.bits.readF(u5, .{
116		.buffered = true,
117		.reverse = true,
118		}));
119		try self.hist.writeMatch(length, distance);
120		}
121
122	82	fn decodeLength(self: *Decompress, code: u8) !u16 {
123	83	if (code > 28) return error.InvalidCode;
124	84	const ml = Token.matchLength(code);
125	85	return if (ml.extra_bits == 0) // 0 - 5 extra bits
126	86	ml.base
127	87	else
128		ml.base + try self.bits.readN(ml.extra_bits, .{ .buffered = true });
	88	ml.base + try self.takeNBitsBuffered(ml.extra_bits);
129	89	}
130	90
131	91	fn decodeDistance(self: *Decompress, code: u8) !u16 {
...	...	@@ -134,42 +94,7 @@ fn decodeDistance(self: *Decompress, code: u8) !u16 {
134	94	return if (md.extra_bits == 0) // 0 - 13 extra bits
135	95	md.base
136	96	else
137		md.base + try self.bits.readN(md.extra_bits, .{ .buffered = true });
138		}
139
140		fn dynamicBlockHeader(self: *Decompress) !void {
141		const hlit: u16 = @as(u16, try self.bits.read(u5)) + 257; // number of ll code entries present - 257
142		const hdist: u16 = @as(u16, try self.bits.read(u5)) + 1; // number of distance code entries - 1
143		const hclen: u8 = @as(u8, try self.bits.read(u4)) + 4; // hclen + 4 code lengths are encoded
144
145		if (hlit > 286 or hdist > 30)
146		return error.InvalidDynamicBlockHeader;
147
148		// lengths for code lengths
149		var cl_lens = [_]u4{0} ** 19;
150		for (0..hclen) \|i\| {
151		cl_lens[flate.huffman.codegen_order[i]] = try self.bits.read(u3);
152		}
153		var cl_dec: CodegenDecoder = .{};
154		try cl_dec.generate(&cl_lens);
155
156		// decoded code lengths
157		var dec_lens = [_]u4{0} ** (286 + 30);
158		var pos: usize = 0;
159		while (pos < hlit + hdist) {
160		const sym = try cl_dec.find(try self.bits.peekF(u7, .{ .reverse = true }));
161		try self.bits.shift(sym.code_bits);
162		pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
163		}
164		if (pos > hlit + hdist) {
165		return error.InvalidDynamicBlockHeader;
166		}
167
168		// literal code lengths to literal decoder
169		try self.lit_dec.generate(dec_lens[0..hlit]);
170
171		// distance code lengths to distance decoder
172		try self.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
	97	md.base + try self.takeNBitsBuffered(md.extra_bits);
173	98	}
174	99
175	100	// Decode code length symbol to code length. Writes decoded length into
...	...	@@ -188,7 +113,7 @@ fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usiz
188	113	16 => {
189	114	// Copy the previous code length 3 - 6 times.
190	115	// The next 2 bits indicate repeat length
191		const n: u8 = @as(u8, try self.bits.read(u2)) + 3;
	116	const n: u8 = @as(u8, try self.takeBits(u2)) + 3;
192	117	if (pos == 0 or pos + n > lens.len)
193	118	return error.InvalidDynamicBlockHeader;
194	119	for (0..n) \|i\| {
...	...	@@ -197,194 +122,301 @@ fn dynamicCodeLength(self: *Decompress, code: u16, lens: []u4, pos: usize) !usiz
197	122	return n;
198	123	},
199	124	// Repeat a code length of 0 for 3 - 10 times. (3 bits of length)
200		17 => return @as(u8, try self.bits.read(u3)) + 3,
	125	17 => return @as(u8, try self.takeBits(u3)) + 3,
201	126	// Repeat a code length of 0 for 11 - 138 times (7 bits of length)
202		18 => return @as(u8, try self.bits.read(u7)) + 11,
	127	18 => return @as(u8, try self.takeBits(u7)) + 11,
203	128	else => return error.InvalidDynamicBlockHeader,
204	129	}
205	130	}
206	131
207		// In larger archives most blocks are usually dynamic, so decompression
208		// performance depends on this function.
209		fn dynamicBlock(self: *Decompress) !bool {
210		// Hot path loop!
211		while (!self.hist.full()) {
212		// optimization so other bit reads can be buffered (avoiding one `if` in hot path)
213		try self.bits.fill(15);
214		const sym = try self.decodeSymbol(&self.lit_dec);
215
216		switch (sym.kind) {
217		.literal => self.hist.write(sym.symbol),
218		.match => {
219		// Decode match backreference <length, distance>
220		try self.bits.fill(5 + 15 + 13);
221		const length = try self.decodeLength(sym.symbol);
222		const dsm = try self.decodeSymbol(&self.dst_dec);
223		const distance = try self.decodeDistance(dsm.symbol);
224		try self.hist.writeMatch(length, distance);
225		},
226		.end_of_block => return true,
227		}
228		}
229		return false;
230		}
231
232	132	// Peek 15 bits from bits reader (maximum code len is 15 bits). Use
233	133	// decoder to find symbol for that code. We then know how many bits is
234	134	// used. Shift bit reader for that much bits, those bits are used. And
235	135	// return symbol.
236	136	fn decodeSymbol(self: *Decompress, decoder: anytype) !Symbol {
237		const sym = try decoder.find(try self.bits.peekF(u15, .{ .buffered = true, .reverse = true }));
238		try self.bits.shift(sym.code_bits);
	137	const sym = try decoder.find(try self.peekBitsReverseBuffered(u15));
	138	try self.shiftBits(sym.code_bits);
239	139	return sym;
240	140	}
241	141
242		fn step(self: *Decompress) !void {
243		switch (self.state) {
244		.protocol_header => {
245		try self.hasher.container().parseHeader(&self.bits);
246		self.state = .block_header;
	142	pub fn read(
	143	context: ?*anyopaque,
	144	bw: *std.io.BufferedWriter,
	145	limit: std.io.Reader.Limit,
	146	) std.io.Reader.RwError!usize {
	147	const d: *Decompress = @alignCast(@ptrCast(context));
	148	return readInner(d, bw, limit) catch \|err\| switch (err) {
	149	error.EndOfStream => return error.EndOfStream,
	150	error.WriteFailed => return error.WriteFailed,
	151	else => \|e\| {
	152	// In the event of an error, state is unmodified so that it can be
	153	// better used to diagnose the failure.
	154	d.read_err = e;
	155	return error.ReadFailed;
	156	},
	157	};
	158	}
	159
	160	fn readInner(
	161	d: *Decompress,
	162	bw: *std.io.BufferedWriter,
	163	limit: std.io.Reader.Limit,
	164	) (Error \|\| error{ WriteFailed, EndOfStream })!usize {
	165	const in = d.input;
	166	sw: switch (d.state) {
	167	.protocol_header => switch (d.hasher.container()) {
	168	.gzip => {
	169	const Header = extern struct {
	170	magic: u16 align(1),
	171	method: u8,
	172	flags: packed struct(u8) {
	173	text: bool,
	174	hcrc: bool,
	175	extra: bool,
	176	name: bool,
	177	comment: bool,
	178	reserved: u3,
	179	},
	180	mtime: u32 align(1),
	181	xfl: u8,
	182	os: u8,
	183	};
	184	const header = try in.takeStructEndian(Header, .little);
	185	if (header.magic != 0x8b1f or header.method != 0x08)
	186	return error.BadGzipHeader;
	187	if (header.flags.extra) {
	188	const extra_len = try in.takeInt(u16, .little);
	189	try in.discardAll(extra_len);
	190	}
	191	if (header.flags.name) {
	192	try in.discardDelimiterInclusive(0);
	193	}
	194	if (header.flags.comment) {
	195	try in.discardDelimiterInclusive(0);
	196	}
	197	if (header.flags.hcrc) {
	198	try in.discardAll(2);
	199	}
	200	continue :sw .block_header;
	201	},
	202	.zlib => {
	203	const Header = extern struct {
	204	cmf: packed struct(u8) {
	205	cm: u4,
	206	cinfo: u4,
	207	},
	208	flg: u8,
	209	};
	210	const header = try in.takeStruct(Header);
	211	if (header.cmf.cm != 8 or header.cmf.cinfo > 7) return error.BadZlibHeader;
	212	continue :sw .block_header;
	213	},
	214	.raw => continue :sw .block_header,
247	215	},
248	216	.block_header => {
249		try self.blockHeader();
250		self.state = .block;
251		if (self.block_type == 2) try self.dynamicBlockHeader();
	217	d.final_block = (try d.takeBits(u1)) != 0;
	218	const block_type = try d.takeBits(BlockType);
	219	switch (block_type) {
	220	.stored => {
	221	d.alignBitsToByte(); // skip padding until byte boundary
	222	// everything after this is byte aligned in stored block
	223	const len = try in.takeInt(u16, .little);
	224	const nlen = try in.takeInt(u16, .little);
	225	if (len != ~nlen) return error.WrongStoredBlockNlen;
	226	continue :sw .{ .stored_block = len };
	227	},
	228	.fixed => continue :sw .fixed_block,
	229	.dynamic => {
	230	const hlit: u16 = @as(u16, try d.takeBits(u5)) + 257; // number of ll code entries present - 257
	231	const hdist: u16 = @as(u16, try d.takeBits(u5)) + 1; // number of distance code entries - 1
	232	const hclen: u8 = @as(u8, try d.takeBits(u4)) + 4; // hclen + 4 code lengths are encoded
	233
	234	if (hlit > 286 or hdist > 30)
	235	return error.InvalidDynamicBlockHeader;
	236
	237	// lengths for code lengths
	238	var cl_lens = [_]u4{0} ** 19;
	239	for (0..hclen) \|i\| {
	240	cl_lens[flate.huffman.codegen_order[i]] = try d.takeBits(u3);
	241	}
	242	var cl_dec: CodegenDecoder = .{};
	243	try cl_dec.generate(&cl_lens);
	244
	245	// decoded code lengths
	246	var dec_lens = [_]u4{0} ** (286 + 30);
	247	var pos: usize = 0;
	248	while (pos < hlit + hdist) {
	249	const sym = try cl_dec.find(try d.peekBitsReverse(u7));
	250	try d.shiftBits(sym.code_bits);
	251	pos += try d.dynamicCodeLength(sym.symbol, &dec_lens, pos);
	252	}
	253	if (pos > hlit + hdist) {
	254	return error.InvalidDynamicBlockHeader;
	255	}
	256
	257	// literal code lengths to literal decoder
	258	try d.lit_dec.generate(dec_lens[0..hlit]);
	259
	260	// distance code lengths to distance decoder
	261	try d.dst_dec.generate(dec_lens[hlit .. hlit + hdist]);
	262
	263	continue :sw .dynamic_block;
	264	},
	265	}
252	266	},
253		.block => {
254		const done = switch (self.block_type) {
255		0 => try self.storedBlock(),
256		1 => try self.fixedBlock(),
257		2 => try self.dynamicBlock(),
258		else => return error.InvalidBlockType,
259		};
260		if (done) {
261		self.state = if (self.bfinal == 1) .protocol_footer else .block_header;
	267	.stored_block => \|remaining_len\| {
	268	const out = try bw.writableSliceGreedyPreserving(flate.history_len, 1);
	269	const limited_out = limit.min(.limited(remaining_len)).slice(out);
	270	const n = try d.input.readVec(bw, &.{limited_out});
	271	if (remaining_len - n == 0) {
	272	d.state = if (d.final_block) .protocol_footer else .block_header;
	273	} else {
	274	d.state = .{ .stored_block = remaining_len - n };
	275	}
	276	bw.advance(n);
	277	return n;
	278	},
	279	.fixed_block => {
	280	const start = bw.count;
	281	while (@intFromEnum(limit) > bw.count - start) {
	282	const code = try d.readFixedCode();
	283	switch (code) {
	284	0...255 => try bw.writeBytePreserving(flate.history_len, @intCast(code)),
	285	256 => {
	286	d.state = if (d.final_block) .protocol_footer else .block_header;
	287	return bw.count - start;
	288	},
	289	257...285 => {
	290	// Handles fixed block non literal (length) code.
	291	// Length code is followed by 5 bits of distance code.
	292	const rebased_code = code - 257;
	293	const length = try d.decodeLength(rebased_code);
	294	const distance = try d.decodeDistance(try d.takeBitsReverseBuffered(u5));
	295	try writeMatch(bw, length, distance);
	296	},
	297	else => return error.InvalidCode,
	298	}
	299	}
	300	d.state = .fixed_block;
	301	return bw.count - start;
	302	},
	303	.dynamic_block => {
	304	// In larger archives most blocks are usually dynamic, so decompression
	305	// performance depends on this logic.
	306	const start = bw.count;
	307	while (@intFromEnum(limit) > bw.count - start) {
	308	const sym = try d.decodeSymbol(&d.lit_dec);
	309
	310	switch (sym.kind) {
	311	.literal => d.hist.write(sym.symbol),
	312	.match => {
	313	// Decode match backreference <length, distance>
	314	const length = try d.decodeLength(sym.symbol);
	315	const dsm = try d.decodeSymbol(&d.dst_dec);
	316	const distance = try d.decodeDistance(dsm.symbol);
	317	try writeMatch(bw, length, distance);
	318	},
	319	.end_of_block => {
	320	d.state = if (d.final_block) .protocol_footer else .block_header;
	321	return bw.count - start;
	322	},
	323	}
262	324	}
	325	d.state = .dynamic_block;
	326	return bw.count - start;
263	327	},
264	328	.protocol_footer => {
265		self.bits.alignToByte();
266		try self.hasher.container().parseFooter(&self.hasher, &self.bits);
267		self.state = .end;
	329	d.alignBitsToByte();
	330	switch (d.hasher.container()) {
	331	.gzip => \|*gzip\| {
	332	if (try reader.read(u32) != gzip.final()) return error.WrongGzipChecksum;
	333	if (try reader.read(u32) != gzip.count) return error.WrongGzipSize;
	334	},
	335	.zlib => \|*zlib\| {
	336	const chksum: u32 = @byteSwap(zlib.final());
	337	if (try reader.read(u32) != chksum) return error.WrongZlibChecksum;
	338	},
	339	.raw => {},
	340	}
	341	d.state = .end;
	342	return 0;
268	343	},
269		.end => {},
	344	.end => return error.EndOfStream,
270	345	}
271	346	}
272	347
273		/// Replaces the inner reader with new reader.
274		pub fn setReader(self: Decompress, new_reader: std.io.BufferedReader) void {
275		self.bits.forward_reader = new_reader;
276		if (self.state == .end or self.state == .protocol_footer) {
277		self.state = .protocol_header;
278		}
	348	fn readVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
	349	_ = context;
	350	_ = data;
	351	@panic("TODO remove readVec primitive");
279	352	}
280	353
281		// Reads all compressed data from the internal reader and outputs plain
282		// (uncompressed) data to the provided writer.
283		pub fn decompress(self: Decompress, writer: std.io.BufferedWriter) !void {
284		while (try self.next()) \|buf\| {
285		try writer.writeAll(buf);
286		}
	354	fn discard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
	355	_ = context;
	356	_ = limit;
	357	// Problem here is we still need access to the output ring buffer.
	358	@panic("TODO allow discard to be null");
287	359	}
288	360
289		/// Returns the number of bytes that have been read from the internal
290		/// reader but not yet consumed by the decompressor.
291		pub fn unreadBytes(self: Decompress) usize {
292		// There can be no error here: the denominator is not zero, and
293		// overflow is not possible since the type is unsigned.
294		return std.math.divCeil(usize, self.bits.nbits, 8) catch unreachable;
	361	/// Write match (back-reference to the same data slice) starting at `distance`
	362	/// back from current write position, and `length` of bytes.
	363	fn writeMatch(bw: *std.io.BufferedWriter, length: u16, distance: u16) !void {
	364	_ = bw;
	365	_ = length;
	366	_ = distance;
	367	@panic("TODO");
295	368	}
296	369
297		// Iterator interface
	370	pub fn reader(self: *Decompress) std.io.Reader {
	371	return .{
	372	.context = self,
	373	.vtable = &.{
	374	.read = read,
	375	.readVec = readVec,
	376	.discard = discard,
	377	},
	378	};
	379	}
298	380
299		/// Can be used in iterator like loop without memcpy to another buffer:
300		/// while (try inflate.next()) \|buf\| { ... }
301		pub fn next(self: *Decompress) Error!?[]const u8 {
302		const out = try self.get(0);
303		if (out.len == 0) return null;
304		return out;
	381	pub fn readable(self: *Decompress, buffer: []u8) std.io.BufferedReader {
	382	return reader(self).buffered(buffer);
305	383	}
306	384
307		/// Returns decompressed data from internal sliding window buffer.
308		/// Returned buffer can be any length between 0 and `limit` bytes. 0
309		/// returned bytes means end of stream reached. With limit=0 returns as
310		/// much data it can. It newer will be more than 65536 bytes, which is
311		/// size of internal buffer.
312		/// TODO merge this logic into readerRead and readerReadVec
313		pub fn get(self: *Decompress, limit: usize) Error![]const u8 {
314		while (true) {
315		const out = self.hist.readAtMost(limit);
316		if (out.len > 0) {
317		self.hasher.update(out);
318		return out;
319		}
320		if (self.state == .end) return out;
321		try self.step();
322		}
	385	fn takeBits(d: *Decompress, comptime T: type) !T {
	386	_ = d;
	387	@panic("TODO");
323	388	}
324	389
325		fn readerRead(
326		context: ?*anyopaque,
327		bw: *std.io.BufferedWriter,
328		limit: std.io.Reader.Limit,
329		) std.io.Reader.RwError!usize {
330		const self: *Decompress = @alignCast(@ptrCast(context));
331		const out = try bw.writableSliceGreedy(1);
332		const in = self.get(limit.minInt(out.len)) catch \|err\| switch (err) {
333		error.EndOfStream => return error.EndOfStream,
334		error.ReadFailed => return error.ReadFailed,
335		else => \|e\| {
336		self.read_err = e;
337		return error.ReadFailed;
338		},
339		};
340		if (in.len == 0) return error.EndOfStream;
341		@memcpy(out[0..in.len], in);
342		bw.advance(in.len);
343		return in.len;
	390	fn takeNBitsBuffered(d: *Decompress, n: u4) !u16 {
	391	_ = d;
	392	_ = n;
	393	@panic("TODO");
344	394	}
345	395
346		fn readerReadVec(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
347		const self: *Decompress = @alignCast(@ptrCast(context));
348		return readVec(self, data) catch \|err\| switch (err) {
349		error.EndOfStream => return error.EndOfStream,
350		error.ReadFailed => return error.ReadFailed,
351		else => \|e\| {
352		self.read_err = e;
353		return error.ReadFailed;
354		},
355		};
	396	fn peekBitsReverse(d: *Decompress, comptime T: type) !T {
	397	_ = d;
	398	@panic("TODO");
356	399	}
357	400
358		fn readerDiscard(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
359		_ = context;
360		_ = limit;
	401	fn peekBitsReverseBuffered(d: *Decompress, comptime T: type) !T {
	402	_ = d;
361	403	@panic("TODO");
362	404	}
363	405
364		pub fn readVec(self: *Decompress, data: []const []u8) Error!usize {
365		for (data) \|out\| {
366		if (out.len == 0) continue;
367		const in = try self.get(out.len);
368		@memcpy(out[0..in.len], in);
369		if (in.len == 0) return error.EndOfStream;
370		return in.len;
371		}
372		return 0;
	406	fn alignBitsToByte(d: *Decompress) void {
	407	_ = d;
	408	@panic("TODO");
373	409	}
374	410
375		pub fn reader(self: *Decompress) std.io.Reader {
376		return .{
377		.context = self,
378		.vtable = &.{
379		.read = readerRead,
380		.readVec = readerReadVec,
381		.discard = readerDiscard,
382		},
383		};
	411	fn shiftBits(d: *Decompress, n: u6) !void {
	412	_ = d;
	413	_ = n;
	414	@panic("TODO");
384	415	}
385	416
386		pub fn readable(self: *Decompress, buffer: []u8) std.io.BufferedReader {
387		return reader(self).buffered(buffer);
	417	fn readFixedCode(d: *Decompress) !u16 {
	418	_ = d;
	419	@panic("TODO");
388	420	}
389	421
390	422	pub const Symbol = packed struct {
...	...	@@ -712,12 +744,16 @@ test "decompress" {
712	744	},
713	745	};
714	746	for (cases) \|c\| {
715		var fb = std.io.fixedBufferStream(c.in);
716		var al = std.ArrayList(u8).init(testing.allocator);
717		defer al.deinit();
718
719		try decompress(.raw, fb.reader(), al.writer());
720		try testing.expectEqualStrings(c.out, al.items);
	747	var fb: std.io.BufferedReader = undefined;
	748	fb.initFixed(@constCast(c.in));
	749	var aw: std.io.AllocatingWriter = undefined;
	750	aw.init(testing.allocator);
	751	defer aw.deinit();
	752
	753	var decompress: Decompress = .init(&fb, .raw);
	754	var decompress_br = decompress.readable(&.{});
	755	_ = try decompress_br.readRemaining(&aw.buffered_writer);
	756	try testing.expectEqualStrings(c.out, aw.getWritten());
721	757	}
722	758	}
723	759
...	...	@@ -769,12 +805,16 @@ test "gzip decompress" {
769	805	},
770	806	};
771	807	for (cases) \|c\| {
772		var fb = std.io.fixedBufferStream(c.in);
773		var al = std.ArrayList(u8).init(testing.allocator);
774		defer al.deinit();
775
776		try decompress(.gzip, fb.reader(), al.writer());
777		try testing.expectEqualStrings(c.out, al.items);
	808	var fb: std.io.BufferedReader = undefined;
	809	fb.initFixed(@constCast(c.in));
	810	var aw: std.io.AllocatingWriter = undefined;
	811	aw.init(testing.allocator);
	812	defer aw.deinit();
	813
	814	var decompress: Decompress = .init(&fb, .gzip);
	815	var decompress_br = decompress.readable(&.{});
	816	_ = try decompress_br.readRemaining(&aw.buffered_writer);
	817	try testing.expectEqualStrings(c.out, aw.getWritten());
778	818	}
779	819	}
780	820
...	...	@@ -795,12 +835,16 @@ test "zlib decompress" {
795	835	},
796	836	};
797	837	for (cases) \|c\| {
798		var fb = std.io.fixedBufferStream(c.in);
799		var al = std.ArrayList(u8).init(testing.allocator);
800		defer al.deinit();
801
802		try decompress(.zlib, fb.reader(), al.writer());
803		try testing.expectEqualStrings(c.out, al.items);
	838	var fb: std.io.BufferedReader = undefined;
	839	fb.initFixed(@constCast(c.in));
	840	var aw: std.io.AllocatingWriter = undefined;
	841	aw.init(testing.allocator);
	842	defer aw.deinit();
	843
	844	var decompress: Decompress = .init(&fb, .zlib);
	845	var decompress_br = decompress.readable(&.{});
	846	_ = try decompress_br.readRemaining(&aw.buffered_writer);
	847	try testing.expectEqualStrings(c.out, aw.getWritten());
804	848	}
805	849	}
806	850
...	...	@@ -853,16 +897,21 @@ test "fuzzing tests" {
853	897	};
854	898
855	899	inline for (cases, 0..) \|c, case_no\| {
856		var in = std.io.fixedBufferStream(@embedFile("testdata/fuzz/" ++ c.input ++ ".input"));
857		var out = std.ArrayList(u8).init(testing.allocator);
858		defer out.deinit();
	900	var in: std.io.BufferedReader = undefined;
	901	in.initFixed(@constCast(@embedFile("testdata/fuzz/" ++ c.input ++ ".input")));
	902	var aw: std.io.AllocatingWriter = undefined;
	903	aw.init(testing.allocator);
	904	defer aw.deinit();
859	905	errdefer std.debug.print("test case failed {}\n", .{case_no});
860	906
	907	var decompress: Decompress = .init(&in, .raw);
	908	var decompress_br = decompress.readable(&.{});
861	909	if (c.err) \|expected_err\| {
862		try testing.expectError(expected_err, decompress(.raw, in.reader(), out.writer()));
	910	try testing.expectError(error.ReadFailed, decompress_br.readRemaining(&aw.buffered_writer));
	911	try testing.expectError(expected_err, decompress.read_err.?);
863	912	} else {
864		try decompress(.raw, in.reader(), out.writer());
865		try testing.expectEqualStrings(c.out, out.items);
	913	_ = try decompress_br.readRemaining(&aw.buffered_writer);
	914	try testing.expectEqualStrings(c.out, aw.getWritten());
866	915	}
867	916	}
868	917	}
...	...	@@ -871,21 +920,28 @@ test "bug 18966" {
871	920	const input = @embedFile("testdata/fuzz/bug_18966.input");
872	921	const expect = @embedFile("testdata/fuzz/bug_18966.expect");
873	922
874		var in = std.io.fixedBufferStream(input);
875		var out = std.ArrayList(u8).init(testing.allocator);
876		defer out.deinit();
	923	var in: std.io.BufferedReader = undefined;
	924	in.initFixed(@constCast(input));
	925	var aw: std.io.AllocatingWriter = undefined;
	926	aw.init(testing.allocator);
	927	defer aw.deinit();
877	928
878		try decompress(.gzip, in.reader(), out.writer());
879		try testing.expectEqualStrings(expect, out.items);
	929	var decompress: Decompress = .init(&in, .gzip);
	930	var decompress_br = decompress.readable(&.{});
	931	_ = try decompress_br.readRemaining(&aw.buffered_writer);
	932	try testing.expectEqualStrings(expect, aw.getWritten());
880	933	}
881	934
882		test "bug 19895" {
	935	test "reading into empty buffer" {
	936	// Inspired by https://github.com/ziglang/zig/issues/19895
883	937	const input = &[_]u8{
884	938	0b0000_0001, 0b0000_1100, 0x00, 0b1111_0011, 0xff, // deflate fixed buffer header len, nlen
885	939	'H', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd', 0x0a, // non compressed data
886	940	};
887		var in = std.io.fixedBufferStream(input);
888		var decomp = Decompress.init(.raw, in.reader());
	941	var in: std.io.BufferedReader = undefined;
	942	in.initFixed(@constCast(input));
	943	var decomp: Decompress = .init(&in, .raw);
	944	var decompress_br = decomp.readable(&.{});
889	945	var buf: [0]u8 = undefined;
890		try testing.expectEqual(0, try decomp.read(&buf));
	946	try testing.expectEqual(0, try decompress_br.readVec(&.{&buf}));
891	947	}

lib/std/compress/flate/Lookup.zig+1-1

...	...	@@ -10,7 +10,7 @@ const flate = @import("../flate.zig");
10	10	const Lookup = @This();
11	11
12	12	const prime4 = 0x9E3779B1; // 4 bytes prime number 2654435761
13		const chain_len = 2 * flate.history.len;
	13	const chain_len = 2 * flate.history_len;
14	14
15	15	// Maps hash => first position
16	16	head: [flate.lookup.len]u16 = [_]u16{0} ** flate.lookup.len,

lib/std/compress/xz/test.zig+3-2

...	...	@@ -3,9 +3,10 @@ const testing = std.testing;
3	3	const xz = std.compress.xz;
4	4
5	5	fn decompress(data: []const u8) ![]u8 {
6		var in_stream = std.io.fixedBufferStream(data);
	6	var in_stream: std.io.BufferedReader = undefined;
	7	in_stream.initFixed(data);
7	8
8		var xz_stream = try xz.decompress(testing.allocator, in_stream.reader());
	9	var xz_stream = try xz.decompress(testing.allocator, &in_stream);
9	10	defer xz_stream.deinit();
10	11
11	12	return xz_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));

lib/std/compress/zstd/Decompress.zig+2-2

...	...	@@ -267,8 +267,8 @@ fn discard(context: ?*anyopaque, limit: Reader.Limit) Reader.Error!usize {
267	267
268	268	fn readVec(context: ?*anyopaque, data: []const []u8) Reader.Error!usize {
269	269	var bw: BufferedWriter = undefined;
270		bw.initFixed(data[0]);
271		return read(context, &bw, .limited(data[0].len)) catch \|err\| switch (err) {
	270	bw.initVec(data);
	271	return read(context, &bw, .countVec(data)) catch \|err\| switch (err) {
272	272	error.WriteFailed => unreachable,
273	273	else => \|e\| return e,
274	274	};

lib/std/crypto/codecs/asn1.zig+4-3

...	...	@@ -154,7 +154,8 @@ pub const Tag = struct {
154	154
155	155	test Tag {
156	156	const buf = [_]u8{0xa3};
157		var stream = std.io.fixedBufferStream(&buf);
	157	var stream: std.io.BufferedReader = undefined;
	158	stream.initFixed(&buf);
158	159	const t = Tag.decode(stream.reader());
159	160	try std.testing.expectEqual(Tag.init(@enumFromInt(3), true, .context_specific), t);
160	161	}
...	...	@@ -184,8 +185,8 @@ pub const Element = struct {
184	185	/// - Ensures length is within `bytes`
185	186	/// - Ensures length is less than `std.math.maxInt(Index)`
186	187	pub fn decode(bytes: []const u8, index: Index) DecodeError!Element {
187		var stream = std.io.fixedBufferStream(bytes[index..]);
188		var reader = stream.reader();
	188	var reader: std.io.BufferedReader = undefined;
	189	reader.initFixed(bytes[index..]);
189	190
190	191	const tag = try Tag.decode(reader);
191	192	const size_or_len_size = try reader.readByte();

lib/std/crypto/tls/Client.zig+2-2

...	...	@@ -1227,8 +1227,8 @@ fn read(context: ?anyopaque, bw: std.io.BufferedWriter, limit: Reader.Limit) R
1227	1227
1228	1228	fn readVec(context: ?*anyopaque, data: []const []u8) Reader.Error!usize {
1229	1229	var bw: std.io.BufferedWriter = undefined;
1230		bw.initFixed(data[0]);
1231		return read(context, &bw, .limited(data[0].len)) catch \|err\| switch (err) {
	1230	bw.initVec(data);
	1231	return read(context, &bw, .countVec(data)) catch \|err\| switch (err) {
1232	1232	error.WriteFailed => unreachable,
1233	1233	else => \|e\| return e,
1234	1234	};

lib/std/debug/Dwarf.zig+1-1

...	...	@@ -2241,7 +2241,7 @@ pub const ElfModule = struct {
2241	2241	if (chdr.ch_type != .ZLIB) continue;
2242	2242	const ch_size = chdr.ch_size;
2243	2243
2244		var zlib_stream: std.compress.zlib.Decompressor = .init(&section_reader);
	2244	var zlib_stream: std.compress.flate.Decompress = .init(&section_reader, .zlib);
2245	2245
2246	2246	const decompressed_section = zlib_stream.reader().readRemainingAlloc(gpa, .limited(ch_size)) catch continue;
2247	2247	if (decompressed_section.len != ch_size) {

lib/std/debug/Pdb.zig+5-5

...	...	@@ -67,7 +67,7 @@ pub fn parseDbiStream(self: *Pdb) !void {
67	67	return error.InvalidDebugInfo;
68	68	const reader = stream.reader();
69	69
70		const header = try reader.readStruct(std.pdb.DbiStreamHeader);
	70	const header = try reader.takeStruct(std.pdb.DbiStreamHeader);
71	71	if (header.version_header != 19990903) // V70, only value observed by LLVM team
72	72	return error.UnknownPDBVersion;
73	73	// if (header.Age != age)
...	...	@@ -82,7 +82,7 @@ pub fn parseDbiStream(self: *Pdb) !void {
82	82	// Module Info Substream
83	83	var mod_info_offset: usize = 0;
84	84	while (mod_info_offset != mod_info_size) {
85		const mod_info = try reader.readStruct(pdb.ModInfo);
	85	const mod_info = try reader.takeStruct(pdb.ModInfo);
86	86	var this_record_len: usize = @sizeOf(pdb.ModInfo);
87	87
88	88	const module_name = try reader.readUntilDelimiterAlloc(self.allocator, 0, 1024);
...	...	@@ -131,7 +131,7 @@ pub fn parseDbiStream(self: *Pdb) !void {
131	131	}
132	132	while (sect_cont_offset != section_contrib_size) {
133	133	const entry = try sect_contribs.addOne();
134		entry.* = try reader.readStruct(pdb.SectionContribEntry);
	134	entry.* = try reader.takeStruct(pdb.SectionContribEntry);
135	135	sect_cont_offset += @sizeOf(pdb.SectionContribEntry);
136	136
137	137	if (sect_cont_offset > section_contrib_size)
...	...	@@ -175,7 +175,7 @@ pub fn parseInfoStream(self: *Pdb) !void {
175	175	return cap * 2 / 3 + 1;
176	176	}
177	177	};
178		const hash_tbl_hdr = try reader.readStruct(HashTableHeader);
	178	const hash_tbl_hdr = try reader.takeStruct(HashTableHeader);
179	179	if (hash_tbl_hdr.capacity == 0)
180	180	return error.InvalidDebugInfo;
181	181
...	...	@@ -397,7 +397,7 @@ const Msf = struct {
397	397	fn init(allocator: Allocator, file: File) !Msf {
398	398	const in = file.reader();
399	399
400		const superblock = try in.readStruct(pdb.SuperBlock);
	400	const superblock = try in.takeStruct(pdb.SuperBlock);
401	401
402	402	// Sanity checks
403	403	if (!std.mem.eql(u8, &superblock.file_magic, pdb.SuperBlock.expect_magic))

lib/std/elf.zig+106-117

...	...	@@ -494,30 +494,39 @@ pub const Header = struct {
494	494	shnum: u16,
495	495	shstrndx: u16,
496	496
497		pub fn program_header_iterator(self: Header, parse_source: anytype) ProgramHeaderIterator(@TypeOf(parse_source)) {
498		return ProgramHeaderIterator(@TypeOf(parse_source)){
499		.elf_header = self,
500		.parse_source = parse_source,
	497	pub fn iterateProgramHeaders(h: Header, file_reader: *std.fs.File.Reader) ProgramHeaderIterator {
	498	return .{
	499	.elf_header = h,
	500	.file_reader = file_reader,
501	501	};
502	502	}
503	503
504		pub fn section_header_iterator(self: Header, parse_source: anytype) SectionHeaderIterator(@TypeOf(parse_source)) {
505		return SectionHeaderIterator(@TypeOf(parse_source)){
506		.elf_header = self,
507		.parse_source = parse_source,
	504	pub fn iterateSectionHeaders(h: Header, file_reader: *std.fs.File.Reader) SectionHeaderIterator {
	505	return .{
	506	.elf_header = h,
	507	.file_reader = file_reader,
508	508	};
509	509	}
510	510
511		pub fn read(parse_source: anytype) !Header {
512		var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined;
513		try parse_source.seekableStream().seekTo(0);
514		try parse_source.reader().readNoEof(&hdr_buf);
515		return Header.parse(&hdr_buf);
	511	pub const ReadError = std.io.Reader.Error \|\| ParseError;
	512
	513	pub fn read(br: *std.io.BufferedReader) ReadError!Header {
	514	const buf = try br.peek(@sizeOf(Elf64_Ehdr));
	515	const result = try parse(@ptrCast(buf));
	516	br.toss(if (result.is_64) @sizeOf(Elf64_Ehdr) else @sizeOf(Elf32_Ehdr));
	517	return result;
516	518	}
517	519
518		pub fn parse(hdr_buf: *align(@alignOf(Elf64_Ehdr)) const [@sizeOf(Elf64_Ehdr)]u8) !Header {
519		const hdr32 = @as(*const Elf32_Ehdr, @ptrCast(hdr_buf));
520		const hdr64 = @as(*const Elf64_Ehdr, @ptrCast(hdr_buf));
	520	pub const ParseError = error{
	521	InvalidElfMagic,
	522	InvalidElfVersion,
	523	InvalidElfClass,
	524	InvalidElfEndian,
	525	};
	526
	527	pub fn parse(hdr_buf: *align(@alignOf(Elf64_Ehdr)) const [@sizeOf(Elf64_Ehdr)]u8) ParseError!Header {
	528	const hdr32: *const Elf32_Ehdr = @ptrCast(hdr_buf);
	529	const hdr64: *const Elf64_Ehdr = @ptrCast(hdr_buf);
521	530	if (!mem.eql(u8, hdr32.e_ident[0..4], MAGIC)) return error.InvalidElfMagic;
522	531	if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion;
523	532
...	...	@@ -541,19 +550,19 @@ pub const Header = struct {
541	550	// The meaning of this value depends on `os_abi` so just make it available as `u8`.
542	551	const abi_version = hdr32.e_ident[EI_ABIVERSION];
543	552
544		const @"type" = if (need_bswap) blk: {
	553	const @"type": ET = if (need_bswap) blk: {
545	554	comptime assert(!@typeInfo(ET).@"enum".is_exhaustive);
546	555	const value = @intFromEnum(hdr32.e_type);
547		break :blk @as(ET, @enumFromInt(@byteSwap(value)));
	556	break :blk @enumFromInt(@byteSwap(value));
548	557	} else hdr32.e_type;
549	558
550		const machine = if (need_bswap) blk: {
	559	const machine: EM = if (need_bswap) blk: {
551	560	comptime assert(!@typeInfo(EM).@"enum".is_exhaustive);
552	561	const value = @intFromEnum(hdr32.e_machine);
553		break :blk @as(EM, @enumFromInt(@byteSwap(value)));
	562	break :blk @enumFromInt(@byteSwap(value));
554	563	} else hdr32.e_machine;
555	564
556		return @as(Header, .{
	565	return .{
557	566	.is_64 = is_64,
558	567	.endian = endian,
559	568	.os_abi = os_abi,
...	...	@@ -568,111 +577,91 @@ pub const Header = struct {
568	577	.shentsize = int(is_64, need_bswap, hdr32.e_shentsize, hdr64.e_shentsize),
569	578	.shnum = int(is_64, need_bswap, hdr32.e_shnum, hdr64.e_shnum),
570	579	.shstrndx = int(is_64, need_bswap, hdr32.e_shstrndx, hdr64.e_shstrndx),
571		});
	580	};
572	581	}
573	582	};
574	583
575		pub fn ProgramHeaderIterator(comptime ParseSource: anytype) type {
576		return struct {
577		elf_header: Header,
578		parse_source: ParseSource,
579		index: usize = 0,
580
581		pub fn next(self: *@This()) !?Elf64_Phdr {
582		if (self.index >= self.elf_header.phnum) return null;
583		defer self.index += 1;
584
585		if (self.elf_header.is_64) {
586		var phdr: Elf64_Phdr = undefined;
587		const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index;
588		try self.parse_source.seekableStream().seekTo(offset);
589		try self.parse_source.reader().readNoEof(mem.asBytes(&phdr));
590
591		// ELF endianness matches native endianness.
592		if (self.elf_header.endian == native_endian) return phdr;
593
594		// Convert fields to native endianness.
	584	pub const ProgramHeaderIterator = struct {
	585	elf_header: Header,
	586	file_reader: *std.fs.File.Reader,
	587	index: usize = 0,
	588
	589	pub fn next(it: *ProgramHeaderIterator) !?Elf64_Phdr {
	590	if (it.index >= it.elf_header.phnum) return null;
	591	defer it.index += 1;
	592
	593	if (it.elf_header.is_64) {
	594	var phdr: Elf64_Phdr = undefined;
	595	const offset = it.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * it.index;
	596	try it.file_reader.seekTo(offset);
	597	var br = it.file_reader.readable(&.{});
	598	try br.readSlice(@ptrCast(&phdr));
	599	if (it.elf_header.endian != native_endian)
595	600	mem.byteSwapAllFields(Elf64_Phdr, &phdr);
596		return phdr;
597		}
598
599		var phdr: Elf32_Phdr = undefined;
600		const offset = self.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * self.index;
601		try self.parse_source.seekableStream().seekTo(offset);
602		try self.parse_source.reader().readNoEof(mem.asBytes(&phdr));
603
604		// ELF endianness does NOT match native endianness.
605		if (self.elf_header.endian != native_endian) {
606		// Convert fields to native endianness.
607		mem.byteSwapAllFields(Elf32_Phdr, &phdr);
608		}
609
610		// Convert 32-bit header to 64-bit.
611		return Elf64_Phdr{
612		.p_type = phdr.p_type,
613		.p_offset = phdr.p_offset,
614		.p_vaddr = phdr.p_vaddr,
615		.p_paddr = phdr.p_paddr,
616		.p_filesz = phdr.p_filesz,
617		.p_memsz = phdr.p_memsz,
618		.p_flags = phdr.p_flags,
619		.p_align = phdr.p_align,
620		};
	601	return phdr;
621	602	}
622		};
623		}
624
625		pub fn SectionHeaderIterator(comptime ParseSource: anytype) type {
626		return struct {
627		elf_header: Header,
628		parse_source: ParseSource,
629		index: usize = 0,
630	603
631		pub fn next(self: *@This()) !?Elf64_Shdr {
632		if (self.index >= self.elf_header.shnum) return null;
633		defer self.index += 1;
634
635		if (self.elf_header.is_64) {
636		var shdr: Elf64_Shdr = undefined;
637		const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index;
638		try self.parse_source.seekableStream().seekTo(offset);
639		try self.parse_source.reader().readNoEof(mem.asBytes(&shdr));
640
641		// ELF endianness matches native endianness.
642		if (self.elf_header.endian == native_endian) return shdr;
	604	var phdr: Elf32_Phdr = undefined;
	605	const offset = it.elf_header.phoff + @sizeOf(@TypeOf(phdr)) * it.index;
	606	try it.file_reader.seekTo(offset);
	607	var br = it.file_reader.readable(&.{});
	608	try br.readSlice(@ptrCast(&phdr));
	609	if (it.elf_header.endian != native_endian)
	610	mem.byteSwapAllFields(Elf32_Phdr, &phdr);
	611	return .{
	612	.p_type = phdr.p_type,
	613	.p_offset = phdr.p_offset,
	614	.p_vaddr = phdr.p_vaddr,
	615	.p_paddr = phdr.p_paddr,
	616	.p_filesz = phdr.p_filesz,
	617	.p_memsz = phdr.p_memsz,
	618	.p_flags = phdr.p_flags,
	619	.p_align = phdr.p_align,
	620	};
	621	}
	622	};
643	623
644		// Convert fields to native endianness.
	624	pub const SectionHeaderIterator = struct {
	625	elf_header: Header,
	626	file_reader: *std.fs.File.Reader,
	627	index: usize = 0,
	628
	629	pub fn next(it: *SectionHeaderIterator) !?Elf64_Shdr {
	630	if (it.index >= it.elf_header.shnum) return null;
	631	defer it.index += 1;
	632
	633	if (it.elf_header.is_64) {
	634	var shdr: Elf64_Shdr = undefined;
	635	const offset = it.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * it.index;
	636	try it.file_reader.seekTo(offset);
	637	var br = it.file_reader.readable(&.{});
	638	try br.readSlice(@ptrCast(&shdr));
	639	if (it.elf_header.endian != native_endian)
645	640	mem.byteSwapAllFields(Elf64_Shdr, &shdr);
646		return shdr;
647		}
648
649		var shdr: Elf32_Shdr = undefined;
650		const offset = self.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * self.index;
651		try self.parse_source.seekableStream().seekTo(offset);
652		try self.parse_source.reader().readNoEof(mem.asBytes(&shdr));
653
654		// ELF endianness does NOT match native endianness.
655		if (self.elf_header.endian != native_endian) {
656		// Convert fields to native endianness.
657		mem.byteSwapAllFields(Elf32_Shdr, &shdr);
658		}
659
660		// Convert 32-bit header to 64-bit.
661		return Elf64_Shdr{
662		.sh_name = shdr.sh_name,
663		.sh_type = shdr.sh_type,
664		.sh_flags = shdr.sh_flags,
665		.sh_addr = shdr.sh_addr,
666		.sh_offset = shdr.sh_offset,
667		.sh_size = shdr.sh_size,
668		.sh_link = shdr.sh_link,
669		.sh_info = shdr.sh_info,
670		.sh_addralign = shdr.sh_addralign,
671		.sh_entsize = shdr.sh_entsize,
672		};
	641	return shdr;
673	642	}
674		};
675		}
	643
	644	var shdr: Elf32_Shdr = undefined;
	645	const offset = it.elf_header.shoff + @sizeOf(@TypeOf(shdr)) * it.index;
	646	try it.file_reader.seekTo(offset);
	647	var br = it.file_reader.readable(&.{});
	648	try br.readSlice(@ptrCast(&shdr));
	649	if (it.elf_header.endian != native_endian)
	650	mem.byteSwapAllFields(Elf32_Shdr, &shdr);
	651	return .{
	652	.sh_name = shdr.sh_name,
	653	.sh_type = shdr.sh_type,
	654	.sh_flags = shdr.sh_flags,
	655	.sh_addr = shdr.sh_addr,
	656	.sh_offset = shdr.sh_offset,
	657	.sh_size = shdr.sh_size,
	658	.sh_link = shdr.sh_link,
	659	.sh_info = shdr.sh_info,
	660	.sh_addralign = shdr.sh_addralign,
	661	.sh_entsize = shdr.sh_entsize,
	662	};
	663	}
	664	};
676	665
677	666	fn int(is_64: bool, need_bswap: bool, int_32: anytype, int_64: anytype) @TypeOf(int_64) {
678	667	if (is_64) {

lib/std/fmt.zig+8-9

...	...	@@ -828,8 +828,7 @@ pub const BufPrintError = error{
828	828	NoSpaceLeft,
829	829	};
830	830
831		/// Print a Formatter string into `buf`. Actually just a thin wrapper around `format` and `fixedBufferStream`.
832		/// Returns a slice of the bytes printed to.
	831	/// Print a Formatter string into `buf`. Returns a slice of the bytes printed.
833	832	pub fn bufPrint(buf: []u8, comptime fmt: []const u8, args: anytype) BufPrintError![]u8 {
834	833	var bw: std.io.BufferedWriter = undefined;
835	834	bw.initFixed(buf);
...	...	@@ -1015,18 +1014,18 @@ test "int.padded" {
1015	1014	test "buffer" {
1016	1015	{
1017	1016	var buf1: [32]u8 = undefined;
1018		var fbs = std.io.fixedBufferStream(&buf1);
1019		var bw = fbs.writer();
	1017	var bw: std.io.BufferedWriter = undefined;
	1018	bw.initFixed(&buf1);
1020	1019	try bw.printValue("", .{}, 1234, std.options.fmt_max_depth);
1021		try std.testing.expectEqualStrings("1234", fbs.getWritten());
	1020	try std.testing.expectEqualStrings("1234", bw.getWritten());
1022	1021
1023		fbs.reset();
	1022	bw.initFixed(&buf1);
1024	1023	try bw.printValue("c", .{}, 'a', std.options.fmt_max_depth);
1025		try std.testing.expectEqualStrings("a", fbs.getWritten());
	1024	try std.testing.expectEqualStrings("a", bw.getWritten());
1026	1025
1027		fbs.reset();
	1026	bw.initFixed(&buf1);
1028	1027	try bw.printValue("b", .{}, 0b1100, std.options.fmt_max_depth);
1029		try std.testing.expectEqualStrings("1100", fbs.getWritten());
	1028	try std.testing.expectEqualStrings("1100", bw.getWritten());
1030	1029	}
1031	1030	}
1032	1031

lib/std/io.zig+60

...	...	@@ -9,6 +9,66 @@ const assert = std.debug.assert;
9	9	const Allocator = std.mem.Allocator;
10	10	const Alignment = std.mem.Alignment;
11	11
	12	pub const Limit = enum(usize) {
	13	nothing = 0,
	14	unlimited = std.math.maxInt(usize),
	15	_,
	16
	17	/// `std.math.maxInt(usize)` is interpreted to mean `.unlimited`.
	18	pub fn limited(n: usize) Limit {
	19	return @enumFromInt(n);
	20	}
	21
	22	pub fn countVec(data: []const []const u8) Limit {
	23	var total: usize = 0;
	24	for (data) \|d\| total += d.len;
	25	return .limited(total);
	26	}
	27
	28	pub fn min(a: Limit, b: Limit) Limit {
	29	return @enumFromInt(@min(@intFromEnum(a), @intFromEnum(b)));
	30	}
	31
	32	pub fn minInt(l: Limit, n: usize) usize {
	33	return @min(n, @intFromEnum(l));
	34	}
	35
	36	pub fn slice(l: Limit, s: []u8) []u8 {
	37	return s[0..l.minInt(s.len)];
	38	}
	39
	40	pub fn sliceConst(l: Limit, s: []const u8) []const u8 {
	41	return s[0..l.minInt(s.len)];
	42	}
	43
	44	pub fn toInt(l: Limit) ?usize {
	45	return switch (l) {
	46	else => @intFromEnum(l),
	47	.unlimited => null,
	48	};
	49	}
	50
	51	/// Reduces a slice to account for the limit, leaving room for one extra
	52	/// byte above the limit, allowing for the use case of differentiating
	53	/// between end-of-stream and reaching the limit.
	54	pub fn slice1(l: Limit, non_empty_buffer: []u8) []u8 {
	55	assert(non_empty_buffer.len >= 1);
	56	return non_empty_buffer[0..@min(@intFromEnum(l) +\| 1, non_empty_buffer.len)];
	57	}
	58
	59	pub fn nonzero(l: Limit) bool {
	60	return @intFromEnum(l) > 0;
	61	}
	62
	63	/// Return a new limit reduced by `amount` or return `null` indicating
	64	/// limit would be exceeded.
	65	pub fn subtract(l: Limit, amount: usize) ?Limit {
	66	if (l == .unlimited) return .unlimited;
	67	if (amount > @intFromEnum(l)) return null;
	68	return @enumFromInt(@intFromEnum(l) - amount);
	69	}
	70	};
	71
12	72	pub const Reader = @import("io/Reader.zig");
13	73	pub const Writer = @import("io/Writer.zig");
14	74

lib/std/io/AllocatingWriter.zig+1-1

...	...	@@ -130,7 +130,7 @@ pub fn clearRetainingCapacity(aw: *AllocatingWriter) void {
130	130	}
131	131
132	132	fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize {
133		if (data.len == 0 and splat == 0) return 0;
	133	assert(data.len != 0);
134	134	const aw: *AllocatingWriter = @alignCast(@ptrCast(context));
135	135	const start_len = aw.written.len;
136	136	const bw = &aw.buffered_writer;

lib/std/io/BufferedReader.zig+9-2

...	...	@@ -253,6 +253,12 @@ pub fn toss(br: *BufferedReader, n: usize) void {
253	253	assert(br.seek <= br.end);
254	254	}
255	255
	256	/// Equivalent to `toss(br.bufferedLen())`.
	257	pub fn tossAll(br: *BufferedReader) void {
	258	br.seek = 0;
	259	br.end = 0;
	260	}
	261
256	262	/// Equivalent to `peek` followed by `toss`.
257	263	///
258	264	/// The data returned is invalidated by the next call to `take`, `peek`,
...	...	@@ -791,8 +797,9 @@ pub fn fill(br: *BufferedReader, n: usize) Reader.Error!void {
791	797	}
792	798	}
793	799
794		/// Fills the buffer with at least one more byte of data, without advancing the
795		/// seek position, doing exactly one underlying read.
	800	/// Without advancing the seek position, does exactly one underlying read, filling the buffer as
	801	/// much as possible. This may result in zero bytes added to the buffer, which is not an end of
	802	/// stream condition. End of stream is communicated via returning `error.EndOfStream`.
796	803	///
797	804	/// Asserts buffer capacity is at least 1.
798	805	pub fn fillMore(br: *BufferedReader) Reader.Error!void {

lib/std/io/Reader.zig+21-65

...	...	@@ -14,14 +14,17 @@ vtable: *const VTable,
14	14	pub const VTable = struct {
15	15	/// Writes bytes from the internally tracked stream position to `bw`.
16	16	///
17		/// Returns the number of bytes written, which will be at minimum `0` and at
18		/// most `limit`. The number of bytes read, including zero, does not
19		/// indicate end of stream.
	17	/// Returns the number of bytes written, which will be at minimum `0` and
	18	/// at most `limit`. The number returned, including zero, does not indicate
	19	/// end of stream. `limit` is guaranteed to be at least as large as the
	20	/// buffer capacity of `bw`.
20	21	///
21		/// If the reader has an internal seek position, it moves forward in
22		/// accordance with the number of bytes return from this function.
	22	/// The reader's internal logical seek position moves forward in accordance
	23	/// with the number of bytes returned from this function.
23	24	///
24		/// The implementation should do a maximum of one underlying read call.
	25	/// Implementations are encouraged to utilize mandatory minimum buffer
	26	/// sizes combined with short reads (returning a value less than `limit`)
	27	/// in order to minimize complexity.
25	28	read: const fn (context: ?anyopaque, bw: *BufferedWriter, limit: Limit) RwError!usize,
26	29
27	30	/// Writes bytes from the internally tracked stream position to `data`.
...	...	@@ -30,10 +33,12 @@ pub const VTable = struct {
30	33	/// at most the sum of each data slice length. The number of bytes read,
31	34	/// including zero, does not indicate end of stream.
32	35	///
33		/// If the reader has an internal seek position, it moves forward in
34		/// accordance with the number of bytes return from this function.
	36	/// The reader's internal logical seek position moves forward in accordance
	37	/// with the number of bytes returned from this function.
35	38	///
36		/// The implementation should do a maximum of one underlying read call.
	39	/// Implementations are encouraged to utilize mandatory minimum buffer
	40	/// sizes combined with short reads (returning a value less than the total
	41	/// buffer capacity inside `data`) in order to minimize complexity.
37	42	readVec: const fn (context: ?anyopaque, data: []const []u8) Error!usize,
38	43
39	44	/// Consumes bytes from the internally tracked stream position without
...	...	@@ -43,10 +48,12 @@ pub const VTable = struct {
43	48	/// at most `limit`. The number of bytes returned, including zero, does not
44	49	/// indicate end of stream.
45	50	///
46		/// If the reader has an internal seek position, it moves forward in
47		/// accordance with the number of bytes return from this function.
	51	/// The reader's internal logical seek position moves forward in accordance
	52	/// with the number of bytes returned from this function.
48	53	///
49		/// The implementation should do a maximum of one underlying read call.
	54	/// Implementations are encouraged to utilize mandatory minimum buffer
	55	/// sizes combined with short reads (returning a value less than `limit`)
	56	/// in order to minimize complexity.
50	57	discard: const fn (context: ?anyopaque, limit: Limit) Error!usize,
51	58	};
52	59
...	...	@@ -78,59 +85,8 @@ pub const ShortError = error{
78	85	ReadFailed,
79	86	};
80	87
81		pub const Limit = enum(usize) {
82		nothing = 0,
83		unlimited = std.math.maxInt(usize),
84		_,
85
86		/// `std.math.maxInt(usize)` is interpreted to mean `.unlimited`.
87		pub fn limited(n: usize) Limit {
88		return @enumFromInt(n);
89		}
90
91		pub fn min(a: Limit, b: Limit) Limit {
92		return @enumFromInt(@min(@intFromEnum(a), @intFromEnum(b)));
93		}
94
95		pub fn minInt(l: Limit, n: usize) usize {
96		return @min(n, @intFromEnum(l));
97		}
98
99		pub fn slice(l: Limit, s: []u8) []u8 {
100		return s[0..l.minInt(s.len)];
101		}
102
103		pub fn sliceConst(l: Limit, s: []const u8) []const u8 {
104		return s[0..l.minInt(s.len)];
105		}
106
107		pub fn toInt(l: Limit) ?usize {
108		return switch (l) {
109		else => @intFromEnum(l),
110		.unlimited => null,
111		};
112		}
113
114		/// Reduces a slice to account for the limit, leaving room for one extra
115		/// byte above the limit, allowing for the use case of differentiating
116		/// between end-of-stream and reaching the limit.
117		pub fn slice1(l: Limit, non_empty_buffer: []u8) []u8 {
118		assert(non_empty_buffer.len >= 1);
119		return non_empty_buffer[0..@min(@intFromEnum(l) +\| 1, non_empty_buffer.len)];
120		}
121
122		pub fn nonzero(l: Limit) bool {
123		return @intFromEnum(l) > 0;
124		}
125
126		/// Return a new limit reduced by `amount` or return `null` indicating
127		/// limit would be exceeded.
128		pub fn subtract(l: Limit, amount: usize) ?Limit {
129		if (l == .unlimited) return .unlimited;
130		if (amount > @intFromEnum(l)) return null;
131		return @enumFromInt(@intFromEnum(l) - amount);
132		}
133		};
	88	/// TODO: no pub
	89	pub const Limit = std.io.Limit;
134	90
135	91	pub fn read(r: Reader, bw: *BufferedWriter, limit: Limit) RwError!usize {
136	92	const before = bw.count;

lib/std/io/Reader/Limited.zig+7-7

...	...	@@ -22,7 +22,7 @@ fn passthruRead(context: ?anyopaque, bw: BufferedWriter, limit: Reader.Limit)
22	22	const l: *Limited = @alignCast(@ptrCast(context));
23	23	const combined_limit = limit.min(l.remaining);
24	24	const n = try l.unlimited_reader.read(bw, combined_limit);
25		l.remaining.subtract(n);
	25	l.remaining = l.remaining.subtract(n).?;
26	26	return n;
27	27	}
28	28
...	...	@@ -30,24 +30,24 @@ fn passthruDiscard(context: ?*anyopaque, limit: Reader.Limit) Reader.Error!usize
30	30	const l: *Limited = @alignCast(@ptrCast(context));
31	31	const combined_limit = limit.min(l.remaining);
32	32	const n = try l.unlimited_reader.discard(combined_limit);
33		l.remaining.subtract(n);
	33	l.remaining = l.remaining.subtract(n).?;
34	34	return n;
35	35	}
36	36
37	37	fn passthruReadVec(context: ?*anyopaque, data: []const []u8) Reader.Error!usize {
38	38	const l: *Limited = @alignCast(@ptrCast(context));
39	39	if (data.len == 0) return 0;
40		if (data[0].len >= @intFromEnum(l.limit)) {
41		const n = try l.unlimited_reader.readVec(&.{l.limit.slice(data[0])});
42		l.remaining.subtract(n);
	40	if (data[0].len >= @intFromEnum(l.remaining)) {
	41	const n = try l.unlimited_reader.readVec(&.{l.remaining.slice(data[0])});
	42	l.remaining = l.remaining.subtract(n).?;
43	43	return n;
44	44	}
45	45	var total: usize = 0;
46	46	for (data, 0..) \|buf, i\| {
47	47	total += buf.len;
48		if (total > @intFromEnum(l.limit)) {
	48	if (total > @intFromEnum(l.remaining)) {
49	49	const n = try l.unlimited_reader.readVec(data[0..i]);
50		l.remaining.subtract(n);
	50	l.remaining = l.remaining.subtract(n).?;
51	51	return n;
52	52	}
53	53	}

lib/std/io/Writer.zig+2-1

...	...	@@ -68,7 +68,8 @@ pub const FileError = std.fs.File.PReadError \|\| error{
68	68	Unimplemented,
69	69	};
70	70
71		pub const Limit = std.io.Reader.Limit;
	71	/// TODO: no pub
	72	pub const Limit = std.io.Limit;
72	73
73	74	pub const Offset = enum(u64) {
74	75	zero = 0,

lib/std/os/uefi/protocol/file.zig-24

...	...	@@ -79,30 +79,6 @@ pub const File = extern struct {
79	79	VolumeFull,
80	80	};
81	81
82		pub const SeekableStream = io.SeekableStream(
83		*File,
84		SeekError,
85		SeekError,
86		setPosition,
87		seekBy,
88		getPosition,
89		getEndPos,
90		);
91		pub const Reader = io.Reader(*File, ReadError, read);
92		pub const Writer = io.Writer(*File, WriteError, write);
93
94		pub fn seekableStream(self: *File) SeekableStream {
95		return .{ .context = self };
96		}
97
98		pub fn reader(self: *File) Reader {
99		return .{ .context = self };
100		}
101
102		pub fn writer(self: *File) Writer {
103		return .{ .context = self };
104		}
105
106	82	pub fn open(
107	83	self: *const File,
108	84	file_name: [*:0]const u16,

lib/std/testing.zig+5

...	...	@@ -609,6 +609,11 @@ pub const TmpDir = struct {
609	609	self.parent_dir.close();
610	610	self.* = undefined;
611	611	}
	612
	613	pub fn createFile(td: *TmpDir) std.fs.File {
	614	_ = td;
	615	@panic("TODO");
	616	}
612	617	};
613	618
614	619	pub fn tmpDir(opts: std.fs.Dir.OpenOptions) TmpDir {

lib/std/tz.zig+12-9

...	...	@@ -54,8 +54,8 @@ pub const Tz = struct {
54	54	},
55	55	};
56	56
57		pub fn parse(allocator: std.mem.Allocator, reader: anytype) !Tz {
58		var legacy_header = try reader.readStruct(Header);
	57	pub fn parse(allocator: std.mem.Allocator, reader: *std.io.BufferedReader) !Tz {
	58	var legacy_header = try reader.takeStruct(Header);
59	59	if (!std.mem.eql(u8, &legacy_header.magic, "TZif")) return error.BadHeader;
60	60	if (legacy_header.version != 0 and legacy_header.version != '2' and legacy_header.version != '3') return error.BadVersion;
61	61
...	...	@@ -70,7 +70,7 @@ pub const Tz = struct {
70	70	const skipv = legacy_header.counts.timecnt * 5 + legacy_header.counts.typecnt * 6 + legacy_header.counts.charcnt + legacy_header.counts.leapcnt * 8 + legacy_header.counts.isstdcnt + legacy_header.counts.isutcnt;
71	71	try reader.skipBytes(skipv, .{});
72	72
73		var header = try reader.readStruct(Header);
	73	var header = try reader.takeStruct(Header);
74	74	if (!std.mem.eql(u8, &header.magic, "TZif")) return error.BadHeader;
75	75	if (header.version != '2' and header.version != '3') return error.BadVersion;
76	76	if (builtin.target.cpu.arch.endian() != std.builtin.Endian.big) {
...	...	@@ -215,9 +215,10 @@ pub const Tz = struct {
215	215
216	216	test "slim" {
217	217	const data = @embedFile("tz/asia_tokyo.tzif");
218		var in_stream = std.io.fixedBufferStream(data);
	218	var in_stream: std.io.BufferedReader = undefined;
	219	in_stream.initFixed(data);
219	220
220		var tz = try std.Tz.parse(std.testing.allocator, in_stream.reader());
	221	var tz = try std.Tz.parse(std.testing.allocator, &in_stream);
221	222	defer tz.deinit();
222	223
223	224	try std.testing.expectEqual(tz.transitions.len, 9);
...	...	@@ -228,9 +229,10 @@ test "slim" {
228	229
229	230	test "fat" {
230	231	const data = @embedFile("tz/antarctica_davis.tzif");
231		var in_stream = std.io.fixedBufferStream(data);
	232	var in_stream: std.io.BufferedReader = undefined;
	233	in_stream.initFixed(data);
232	234
233		var tz = try std.Tz.parse(std.testing.allocator, in_stream.reader());
	235	var tz = try std.Tz.parse(std.testing.allocator, &in_stream);
234	236	defer tz.deinit();
235	237
236	238	try std.testing.expectEqual(tz.transitions.len, 8);
...	...	@@ -241,9 +243,10 @@ test "fat" {
241	243	test "legacy" {
242	244	// Taken from Slackware 8.0, from 2001
243	245	const data = @embedFile("tz/europe_vatican.tzif");
244		var in_stream = std.io.fixedBufferStream(data);
	246	var in_stream: std.io.BufferedReader = undefined;
	247	in_stream.initFixed(data);
245	248
246		var tz = try std.Tz.parse(std.testing.allocator, in_stream.reader());
	249	var tz = try std.Tz.parse(std.testing.allocator, &in_stream);
247	250	defer tz.deinit();
248	251
249	252	try std.testing.expectEqual(tz.transitions.len, 170);

lib/std/zip.zig+116-61

...	...	@@ -112,7 +112,7 @@ pub const EndRecord = extern struct {
112	112	return record;
113	113	}
114	114
115		pub const FindFileError = File.GetEndPosError \|\| File.SeekError \|\| error{
	115	pub const FindFileError = File.GetEndPosError \|\| File.SeekError \|\| File.ReadError \|\| error{
116	116	ZipNoEndRecord,
117	117	EndOfStream,
118	118	};
...	...	@@ -138,6 +138,7 @@ pub const EndRecord = extern struct {
138	138	var br = fr.interface().unbuffered();
139	139	br.readSlice(read_buf) catch \|err\| switch (err) {
140	140	error.ReadFailed => return fr.err.?,
	141	error.EndOfStream => return error.EndOfStream,
141	142	};
142	143	loaded_len = new_loaded_len;
143	144	}
...	...	@@ -158,45 +159,83 @@ pub const EndRecord = extern struct {
158	159	}
159	160	};
160	161
161		/// Decompresses the given data from `reader` into `writer`. Stops early if more
162		/// than `uncompressed_size` bytes are processed and verifies that exactly that
163		/// number of bytes are decompressed. Returns the CRC-32 of the uncompressed data.
164		/// `writer` can be anything with a `writeAll(self: *Self, chunk: []const u8) anyerror!void` method.
165		pub fn decompress(
166		method: CompressionMethod,
167		uncompressed_size: u64,
168		reader: *std.io.BufferedReader,
169		writer: *std.io.BufferedWriter,
170		compressed_remaining: *u64,
171		) !u32 {
172		var hash = std.hash.Crc32.init();
173		var total_uncompressed: u64 = 0;
174		switch (method) {
175		.store => {
176		reader.writeAll(writer, .limited(compressed_remaining.*)) catch \|err\| switch (err) {
177		error.EndOfStream => return error.ZipDecompressTruncated,
178		else => \|e\| return e,
179		};
180		total_uncompressed += compressed_remaining.*;
181		},
182		.deflate => {
183		var decompressor: std.compress.flate.Decompressor = .init(reader);
184		while (try decompressor.next()) \|chunk\| {
185		try writer.writeAll(chunk);
186		hash.update(chunk);
187		total_uncompressed += @intCast(chunk.len);
188		if (total_uncompressed > uncompressed_size)
189		return error.ZipUncompressSizeTooSmall;
190		compressed_remaining.* -= chunk.len;
191		}
192		},
193		_ => return error.UnsupportedCompressionMethod,
	162	pub const Decompress = union {
	163	inflate: std.compress.flate.Decompress,
	164	store: *std.io.BufferedReader,
	165
	166	fn readable(
	167	d: *Decompress,
	168	reader: *std.io.BufferedReader,
	169	method: CompressionMethod,
	170	buffer: []u8,
	171	) std.io.BufferedReader {
	172	switch (method) {
	173	.store => {
	174	d.* = .{ .store = reader };
	175	return .{
	176	.unbuffered_reader = .{
	177	.context = d,
	178	.vtable = &.{
	179	.read = readStore,
	180	.readVec = readVecUnimplemented,
	181	.discard = discardUnimplemented,
	182	},
	183	},
	184	.buffer = buffer,
	185	.end = 0,
	186	.seek = 0,
	187	};
	188	},
	189	.deflate => {
	190	d.* = .{ .inflate = .init(reader, .raw) };
	191	return .{
	192	.unbuffered_reader = .{
	193	.context = d,
	194	.vtable = &.{
	195	.read = readDeflate,
	196	.readVec = readVecUnimplemented,
	197	.discard = discardUnimplemented,
	198	},
	199	},
	200	.buffer = buffer,
	201	.end = 0,
	202	.seek = 0,
	203	};
	204	},
	205	else => unreachable,
	206	}
194	207	}
195		if (total_uncompressed != uncompressed_size)
196		return error.ZipUncompressSizeMismatch;
197	208
198		return hash.final();
199		}
	209	fn readStore(
	210	context: ?*anyopaque,
	211	writer: *std.io.BufferedWriter,
	212	limit: std.io.Limit,
	213	) std.io.Reader.RwError!usize {
	214	const d: *Decompress = @ptrCast(@alignCast(context));
	215	return d.store.read(writer, limit);
	216	}
	217
	218	fn readDeflate(
	219	context: ?*anyopaque,
	220	writer: *std.io.BufferedWriter,
	221	limit: std.io.Limit,
	222	) std.io.Reader.RwError!usize {
	223	const d: *Decompress = @ptrCast(@alignCast(context));
	224	return std.compress.flate.Decompress.read(&d.inflate, writer, limit);
	225	}
	226
	227	fn readVecUnimplemented(context: ?*anyopaque, data: []const []u8) std.io.Reader.Error!usize {
	228	_ = context;
	229	_ = data;
	230	@panic("TODO remove readVec primitive");
	231	}
	232
	233	fn discardUnimplemented(context: ?*anyopaque, limit: std.io.Reader.Limit) std.io.Reader.Error!usize {
	234	_ = context;
	235	_ = limit;
	236	@panic("TODO allow discard to be null");
	237	}
	238	};
200	239
201	240	fn isBadFilename(filename: []const u8) bool {
202	241	if (filename.len == 0 or filename[0] == '/')
...	...	@@ -299,8 +338,9 @@ pub const Iterator = struct {
299	338	return error.ZipTruncated;
300	339	try input.seekTo(stream_len - locator_end_offset);
301	340	var br = input.interface().unbuffered();
302		const locator = br.readStructEndian(EndLocator64, .little) catch \|err\| switch (err) {
	341	const locator = br.takeStructEndian(EndLocator64, .little) catch \|err\| switch (err) {
303	342	error.ReadFailed => return input.err.?,
	343	error.EndOfStream => return error.EndOfStream,
304	344	};
305	345	if (!std.mem.eql(u8, &locator.signature, &end_locator64_sig))
306	346	return error.ZipBadLocatorSig;
...	...	@@ -311,8 +351,9 @@ pub const Iterator = struct {
311	351
312	352	try input.seekTo(locator.record_file_offset);
313	353
314		const record64 = br.readStructEndian(EndRecord64, .little) catch \|err\| switch (err) {
	354	const record64 = br.takeStructEndian(EndRecord64, .little) catch \|err\| switch (err) {
315	355	error.ReadFailed => return input.err.?,
	356	error.EndOfStream => return error.EndOfStream,
316	357	};
317	358
318	359	if (!std.mem.eql(u8, &record64.signature, &end_record64_sig))
...	...	@@ -367,8 +408,9 @@ pub const Iterator = struct {
367	408	const input = self.input;
368	409	try input.seekTo(header_zip_offset);
369	410	var br = input.interface().unbuffered();
370		const header = br.readStructEndian(CentralDirectoryFileHeader, .little) catch \|err\| switch (err) {
	411	const header = br.takeStructEndian(CentralDirectoryFileHeader, .little) catch \|err\| switch (err) {
371	412	error.ReadFailed => return input.err.?,
	413	error.EndOfStream => return error.EndOfStream,
372	414	};
373	415	if (!std.mem.eql(u8, &header.signature, &central_file_header_sig))
374	416	return error.ZipBadCdOffset;
...	...	@@ -399,6 +441,7 @@ pub const Iterator = struct {
399	441	try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);
400	442	br.readSlice(extra) catch \|err\| switch (err) {
401	443	error.ReadFailed => return input.err.?,
	444	error.EndOfStream => return error.EndOfStream,
402	445	};
403	446
404	447	var extra_offset: usize = 0;
...	...	@@ -454,20 +497,23 @@ pub const Iterator = struct {
454	497	) !u32 {
455	498	if (filename_buf.len < self.filename_len)
456	499	return error.ZipInsufficientBuffer;
	500	switch (self.compression_method) {
	501	.store, .deflate => {},
	502	else => return error.UnsupportedCompressionMethod,
	503	}
457	504	const filename = filename_buf[0..self.filename_len];
458
459		try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));
460
461	505	{
462		const len = try stream.context.reader().readAll(filename);
463		if (len != filename.len)
464		return error.ZipBadFileOffset;
	506	try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));
	507	var stream_br = stream.readable(&.{});
	508	try stream_br.readSlice(filename);
465	509	}
466	510
467	511	const local_data_header_offset: u64 = local_data_header_offset: {
468	512	const local_header = blk: {
469	513	try stream.seekTo(self.file_offset);
470		break :blk try stream.context.reader().readStructEndian(LocalFileHeader, .little);
	514	var read_buffer: [@sizeOf(LocalFileHeader)]u8 = undefined;
	515	var stream_br = stream.readable(&read_buffer);
	516	break :blk try stream_br.takeStructEndian(LocalFileHeader, .little);
471	517	};
472	518	if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
473	519	return error.ZipBadFileOffset;
...	...	@@ -493,9 +539,8 @@ pub const Iterator = struct {
493	539
494	540	{
495	541	try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
496		const len = try stream.context.reader().readAll(extra);
497		if (len != extra.len)
498		return error.ZipTruncated;
	542	var stream_br = stream.readable(&.{});
	543	try stream_br.readSlice(extra);
499	544	}
500	545
501	546	var extra_offset: usize = 0;
...	...	@@ -557,21 +602,31 @@ pub const Iterator = struct {
557	602	break :blk try dest.createFile(filename, .{ .exclusive = true });
558	603	};
559	604	defer out_file.close();
	605	var file_writer = out_file.writer();
	606	var file_bw = file_writer.writable(&.{});
560	607	const local_data_file_offset: u64 =
561	608	@as(u64, self.file_offset) +
562	609	@as(u64, @sizeOf(LocalFileHeader)) +
563	610	local_data_header_offset;
564	611	try stream.seekTo(local_data_file_offset);
565		var compressed_remaining: u64 = self.compressed_size;
566		const crc = try decompress(
567		self.compression_method,
568		self.uncompressed_size,
569		stream.context.reader(),
570		out_file.writer(),
571		&compressed_remaining,
572		);
573		if (compressed_remaining != 0) return error.ZipDecompressTruncated;
574		return crc;
	612	var limited_file_reader = stream.interface().limited(.limited(self.compressed_size));
	613	var file_read_buffer: [1000]u8 = undefined;
	614	var decompress_read_buffer: [1000]u8 = undefined;
	615	var limited_br = limited_file_reader.reader().buffered(&file_read_buffer);
	616	var decompress: Decompress = undefined;
	617	var decompress_br = decompress.readable(&limited_br, self.compression_method, &decompress_read_buffer);
	618	const start_out = file_bw.count;
	619	var hash_writer = file_bw.hashed(std.hash.Crc32.init());
	620	var hash_bw = hash_writer.writable(&.{});
	621	decompress_br.readAll(&hash_bw, .limited(self.uncompressed_size)) catch \|err\| switch (err) {
	622	error.ReadFailed => return stream.err.?,
	623	error.WriteFailed => return file_writer.err.?,
	624	error.EndOfStream => return error.ZipDecompressTruncated,
	625	};
	626	if (limited_file_reader.remaining.nonzero()) return error.ZipDecompressTruncated;
	627	const written = file_bw.count - start_out;
	628	if (written != self.uncompressed_size) return error.ZipUncompressSizeMismatch;
	629	return hash_writer.hasher.final();
575	630	}
576	631	};
577	632	};

lib/std/zip/test.zig+90-42

...	...	@@ -33,8 +33,10 @@ fn expectFiles(
33	33	std.mem.replaceScalar(u8, normalized_sub_path, '\\', '/');
34	34	var file = try dir.openFile(normalized_sub_path, .{});
35	35	defer file.close();
	36	var file_reader = file.reader();
	37	var file_br = file_reader.readable(&.{});
36	38	var content_buf: [4096]u8 = undefined;
37		const n = try file.reader().readAll(&content_buf);
	39	const n = try file_br.readSliceShort(&content_buf);
38	40	try testing.expectEqualStrings(test_file.content, content_buf[0..n]);
39	41	}
40	42	}
...	...	@@ -49,24 +51,21 @@ const FileStore = struct {
49	51	uncompressed_size: usize,
50	52	};
51	53
52		fn makeZip(buf: []u8, files: []const File, options: WriteZipOptions) !std.io.BufferedReader {
	54	fn makeZip(file_writer: *std.fs.File.Writer, files: []const File, options: WriteZipOptions) !std.io.BufferedReader {
53	55	const store = try std.testing.allocator.alloc(FileStore, files.len);
54	56	defer std.testing.allocator.free(store);
55		return makeZipWithStore(buf, files, options, store);
	57	return makeZipWithStore(file_writer, files, options, store);
56	58	}
57	59
58	60	fn makeZipWithStore(
59		buf: []u8,
	61	file_writer: *std.fs.File.Writer,
60	62	files: []const File,
61	63	options: WriteZipOptions,
62	64	store: []FileStore,
63		) !std.io.BufferedReader {
64		var out: std.io.BufferedWriter = undefined;
65		out.initFixed(buf);
66		try writeZip(&out, files, store, options);
67		var result: std.io.BufferedReader = undefined;
68		result.initFixed(buf[0..out.end]);
69		return result;
	65	) !void {
	66	var buffer: [200]u8 = undefined;
	67	var bw = file_writer.writable(&buffer);
	68	try writeZip(&bw, files, store, options);
70	69	}
71	70
72	71	const WriteZipOptions = struct {
...	...	@@ -201,9 +200,12 @@ const Zipper = struct {
201	200	const offset = writer.count;
202	201	var br: std.io.BufferedReader = undefined;
203	202	br.initFixed(@constCast(opt.content));
204		try std.compress.flate.deflate.compress(.raw, &br, writer, .{});
	203	var compress: std.compress.flate.Compress = .init(&br, .{});
	204	var compress_br = compress.readable(&.{});
	205	const n = try compress_br.readRemaining(writer);
205	206	assert(br.seek == opt.content.len);
206		compressed_size = @intCast(writer.count - offset);
	207	try testing.expectEqual(n, writer.count - offset);
	208	compressed_size = @intCast(n);
207	209	},
208	210	else => unreachable,
209	211	}
...	...	@@ -306,21 +308,27 @@ fn testZipWithStore(
306	308	write_opt: WriteZipOptions,
307	309	store: []FileStore,
308	310	) !void {
309		var zip_buf: [4096]u8 = undefined;
310		var fbs = try makeZipWithStore(&zip_buf, test_files, write_opt, store);
311
312	311	var tmp = testing.tmpDir(.{ .no_follow = true });
313	312	defer tmp.cleanup();
314		try zip.extract(tmp.dir, fbs.seekableStream(), options);
	313
	314	var file = tmp.createFile();
	315	defer file.close();
	316	var file_writer = file.writer();
	317	try makeZipWithStore(&file_writer, test_files, write_opt, store);
	318	var file_reader = file_writer.moveToReader();
	319	try zip.extract(tmp.dir, &file_reader, options);
315	320	try expectFiles(test_files, tmp.dir, .{});
316	321	}
317	322	fn testZipError(expected_error: anyerror, file: File, options: zip.ExtractOptions) !void {
318		var zip_buf: [4096]u8 = undefined;
319		var store: [1]FileStore = undefined;
320		var fbs = try makeZipWithStore(&zip_buf, &[_]File{file}, .{}, &store);
321	323	var tmp = testing.tmpDir(.{ .no_follow = true });
322	324	defer tmp.cleanup();
323		try testing.expectError(expected_error, zip.extract(tmp.dir, fbs.seekableStream(), options));
	325	const tmp_file = tmp.createFile();
	326	defer tmp_file.close();
	327	var file_writer = tmp_file.writer();
	328	var store: [1]FileStore = undefined;
	329	try makeZipWithStore(&file_writer, &[_]File{file}, .{}, &store);
	330	var file_reader = file_writer.moveToReader();
	331	try testing.expectError(expected_error, zip.extract(tmp.dir, &file_reader, options));
324	332	}
325	333
326	334	test "zip one file" {
...	...	@@ -416,53 +424,93 @@ test "zip64" {
416	424	test "bad zip files" {
417	425	var tmp = testing.tmpDir(.{ .no_follow = true });
418	426	defer tmp.cleanup();
419		var zip_buf: [4096]u8 = undefined;
	427	var buffer: [4096]u8 = undefined;
420	428
421	429	const file_a = [_]File{.{ .name = "a", .content = "", .compression = .store }};
422	430
423	431	{
424		var fbs = try makeZip(&zip_buf, &.{}, .{ .end = .{ .sig = [_]u8{ 1, 2, 3, 4 } } });
425		try testing.expectError(error.ZipNoEndRecord, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	432	const tmp_file = tmp.createFile();
	433	defer tmp_file.close();
	434	var file_writer = tmp_file.writable(&buffer);
	435	try makeZip(&file_writer, &.{}, .{ .end = .{ .sig = [_]u8{ 1, 2, 3, 4 } } });
	436	var file_reader = file_writer.moveToReader();
	437	try testing.expectError(error.ZipNoEndRecord, zip.extract(tmp.dir, &file_reader, .{}));
426	438	}
427	439	{
428		var fbs = try makeZip(&zip_buf, &.{}, .{ .end = .{ .comment_len = 1 } });
429		try testing.expectError(error.ZipNoEndRecord, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	440	const tmp_file = tmp.createFile();
	441	defer tmp_file.close();
	442	var file_writer = tmp_file.writable(&buffer);
	443	try makeZip(&file_writer, &.{}, .{ .end = .{ .comment_len = 1 } });
	444	var file_reader = file_writer.moveToReader();
	445	try testing.expectError(error.ZipNoEndRecord, zip.extract(tmp.dir, &file_reader, .{}));
430	446	}
431	447	{
432		var fbs = try makeZip(&zip_buf, &.{}, .{ .end = .{ .comment = "a", .comment_len = 0 } });
433		try testing.expectError(error.ZipNoEndRecord, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	448	const tmp_file = tmp.createFile();
	449	defer tmp_file.close();
	450	var file_writer = tmp_file.writable(&buffer);
	451	try makeZip(&file_writer, &.{}, .{ .end = .{ .comment = "a", .comment_len = 0 } });
	452	var file_reader = file_writer.moveToReader();
	453	try testing.expectError(error.ZipNoEndRecord, zip.extract(tmp.dir, &file_reader, .{}));
434	454	}
435	455	{
436		var fbs = try makeZip(&zip_buf, &.{}, .{ .end = .{ .disk_number = 1 } });
437		try testing.expectError(error.ZipMultiDiskUnsupported, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	456	const tmp_file = tmp.createFile();
	457	defer tmp_file.close();
	458	var file_writer = tmp_file.writable(&buffer);
	459	try makeZip(&file_writer, &.{}, .{ .end = .{ .disk_number = 1 } });
	460	var file_reader = file_writer.moveToReader();
	461	try testing.expectError(error.ZipMultiDiskUnsupported, zip.extract(tmp.dir, &file_reader, .{}));
438	462	}
439	463	{
440		var fbs = try makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_disk_number = 1 } });
441		try testing.expectError(error.ZipMultiDiskUnsupported, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	464	const tmp_file = tmp.createFile();
	465	defer tmp_file.close();
	466	var file_writer = tmp_file.writable(&buffer);
	467	try makeZip(&file_writer, &.{}, .{ .end = .{ .central_directory_disk_number = 1 } });
	468	var file_reader = file_writer.moveToReader();
	469	try testing.expectError(error.ZipMultiDiskUnsupported, zip.extract(tmp.dir, &file_reader, .{}));
442	470	}
443	471	{
444		var fbs = try makeZip(&zip_buf, &.{}, .{ .end = .{ .record_count_disk = 1 } });
445		try testing.expectError(error.ZipDiskRecordCountTooLarge, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	472	const tmp_file = tmp.createFile();
	473	defer tmp_file.close();
	474	var file_writer = tmp_file.writable(&buffer);
	475	try makeZip(&file_writer, &.{}, .{ .end = .{ .record_count_disk = 1 } });
	476	var file_reader = file_writer.moveToReader();
	477	try testing.expectError(error.ZipDiskRecordCountTooLarge, zip.extract(tmp.dir, &file_reader, .{}));
446	478	}
447	479	{
448		var fbs = try makeZip(&zip_buf, &.{}, .{ .end = .{ .central_directory_size = 1 } });
449		try testing.expectError(error.ZipCdOversized, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	480	const tmp_file = tmp.createFile();
	481	defer tmp_file.close();
	482	var file_writer = tmp_file.writable(&buffer);
	483	try makeZip(&file_writer, &.{}, .{ .end = .{ .central_directory_size = 1 } });
	484	var file_reader = file_writer.moveToReader();
	485	try testing.expectError(error.ZipCdOversized, zip.extract(tmp.dir, &file_reader, .{}));
450	486	}
451	487	{
452		var fbs = try makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_size = 0 } });
453		try testing.expectError(error.ZipCdUndersized, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	488	const tmp_file = tmp.createFile();
	489	defer tmp_file.close();
	490	var file_writer = tmp_file.writable(&buffer);
	491	try makeZip(&file_writer, &file_a, .{ .end = .{ .central_directory_size = 0 } });
	492	var file_reader = file_writer.moveToReader();
	493	try testing.expectError(error.ZipCdUndersized, zip.extract(tmp.dir, &file_reader, .{}));
454	494	}
455	495	{
456		var fbs = try makeZip(&zip_buf, &file_a, .{ .end = .{ .central_directory_offset = 0 } });
457		try testing.expectError(error.ZipBadCdOffset, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	496	const tmp_file = tmp.createFile();
	497	defer tmp_file.close();
	498	var file_writer = tmp_file.writable(&buffer);
	499	try makeZip(&file_writer, &file_a, .{ .end = .{ .central_directory_offset = 0 } });
	500	var file_reader = file_writer.moveToReader();
	501	try testing.expectError(error.ZipBadCdOffset, zip.extract(tmp.dir, &file_reader, .{}));
458	502	}
459	503	{
460		var fbs = try makeZip(&zip_buf, &file_a, .{
	504	const tmp_file = tmp.createFile();
	505	defer tmp_file.close();
	506	var file_writer = tmp_file.writable(&buffer);
	507	try makeZip(&file_writer, &file_a, .{
461	508	.end = .{
462	509	.zip64 = .{ .locator_sig = [_]u8{ 1, 2, 3, 4 } },
463	510	.central_directory_size = std.math.maxInt(u32), // trigger 64
464	511	},
465	512	});
466		try testing.expectError(error.ZipBadLocatorSig, zip.extract(tmp.dir, fbs.seekableStream(), .{}));
	513	var file_reader = file_writer.moveToReader();
	514	try testing.expectError(error.ZipBadLocatorSig, zip.extract(tmp.dir, &file_reader, .{}));
467	515	}
468	516	}

src/Package/Fetch.zig+1-2

...	...	@@ -1357,8 +1357,7 @@ fn unpackGitPack(f: Fetch, out_dir: fs.Dir, resource: Resource.Git) anyerror!U
1357	1357	defer pack_dir.close();
1358	1358	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
1359	1359	defer pack_file.close();
1360		var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1361		try fifo.pump(resource.fetch_stream.reader(), pack_file.writer());
	1360	_ = try resource.fetch_stream.reader().readRemaining(pack_file.writer());
1362	1361	try pack_file.sync();
1363	1362
1364	1363	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });

author	Andrew Kelley <andrew@ziglang.org> 2025-05-26 20:31:35-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:29-07:00
log	da303bdaf1ae8717df2d4ede9e7dfb215636ae33
tree	29c02e46d0b180e595cdf636c6e0ebc44a3c1271
parent	fd4fb10722beea90ee3a1c9314a10eec36a44346

std: fix a bunch of compilation errors