Commit 6f46570958af8ae27308eb4a9470e05f33aaa522 | zig

lib/std/Build/Cache.zig+29-37

...	...	@@ -800,7 +800,7 @@ pub const Manifest = struct {
800	800	}
801	801
802	802	var actual_digest: BinDigest = undefined;
803		hashFile(this_file, &actual_digest) catch \|err\| {
	803	hashFile(io, this_file, &actual_digest) catch \|err\| {
804	804	self.diagnostic = .{ .file_read = .{
805	805	.file_index = idx,
806	806	.err = err,
...	...	@@ -908,9 +908,11 @@ pub const Manifest = struct {
908	908	}
909	909	}
910	910
911		fn populateFileHashHandle(self: Manifest, ch_file: File, handle: Io.File) !void {
	911	fn populateFileHashHandle(self: Manifest, ch_file: File, io_file: Io.File) !void {
912	912	const io = self.cache.io;
913		const actual_stat = try handle.stat(io);
	913	const gpa = self.cache.gpa;
	914
	915	const actual_stat = try io_file.stat(io);
914	916	ch_file.stat = .{
915	917	.size = actual_stat.size,
916	918	.mtime = actual_stat.mtime,
...	...	@@ -924,19 +926,17 @@ pub const Manifest = struct {
924	926	}
925	927
926	928	if (ch_file.max_file_size) \|max_file_size\| {
927		if (ch_file.stat.size > max_file_size) {
928		return error.FileTooBig;
929		}
	929	if (ch_file.stat.size > max_file_size) return error.FileTooBig;
930	930
931		const contents = try self.cache.gpa.alloc(u8, @as(usize, @intCast(ch_file.stat.size)));
932		errdefer self.cache.gpa.free(contents);
	931	// Hash while reading from disk, to keep the contents in the cpu
	932	// cache while doing hashing.
	933	const contents = try gpa.alloc(u8, @intCast(ch_file.stat.size));
	934	errdefer gpa.free(contents);
933	935
934		// Hash while reading from disk, to keep the contents in the cpu cache while
935		// doing hashing.
936	936	var hasher = hasher_init;
937	937	var off: usize = 0;
938	938	while (true) {
939		const bytes_read = try handle.pread(contents[off..], off);
	939	const bytes_read = try io_file.readPositional(io, &.{contents[off..]}, off);
940	940	if (bytes_read == 0) break;
941	941	hasher.update(contents[off..][0..bytes_read]);
942	942	off += bytes_read;
...	...	@@ -945,7 +945,7 @@ pub const Manifest = struct {
945	945
946	946	ch_file.contents = contents;
947	947	} else {
948		try hashFile(handle, &ch_file.bin_digest);
	948	try hashFile(io, io_file, &ch_file.bin_digest);
949	949	}
950	950
951	951	self.hash.hasher.update(&ch_file.bin_digest);
...	...	@@ -1169,13 +1169,11 @@ pub const Manifest = struct {
1169	1169
1170	1170	fn downgradeToSharedLock(self: *Manifest) !void {
1171	1171	if (!self.have_exclusive_lock) return;
	1172	const io = self.cache.io;
1172	1173
1173		// WASI does not currently support flock, so we bypass it here.
1174		// TODO: If/when flock is supported on WASI, this check should be removed.
1175		// See https://github.com/WebAssembly/wasi-filesystem/issues/2
1176		if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
	1174	if (std.process.can_spawn or !builtin.single_threaded) {
1177	1175	const manifest_file = self.manifest_file.?;
1178		try manifest_file.downgradeLock();
	1176	try manifest_file.downgradeLock(io);
1179	1177	}
1180	1178
1181	1179	self.have_exclusive_lock = false;
...	...	@@ -1184,16 +1182,14 @@ pub const Manifest = struct {
1184	1182	fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool {
1185	1183	if (self.have_exclusive_lock) return false;
1186	1184	assert(self.manifest_file != null);
	1185	const io = self.cache.io;
1187	1186
1188		// WASI does not currently support flock, so we bypass it here.
1189		// TODO: If/when flock is supported on WASI, this check should be removed.
1190		// See https://github.com/WebAssembly/wasi-filesystem/issues/2
1191		if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
	1187	if (std.process.can_spawn or !builtin.single_threaded) {
1192	1188	const manifest_file = self.manifest_file.?;
1193	1189	// Here we intentionally have a period where the lock is released, in case there are
1194	1190	// other processes holding a shared lock.
1195		manifest_file.unlock();
1196		manifest_file.lock(.exclusive) catch \|err\| {
	1191	manifest_file.unlock(io);
	1192	manifest_file.lock(io, .exclusive) catch \|err\| {
1197	1193	self.diagnostic = .{ .manifest_lock = err };
1198	1194	return error.CacheCheckFailed;
1199	1195	};
...	...	@@ -1206,12 +1202,8 @@ pub const Manifest = struct {
1206	1202	/// The `Manifest` remains safe to deinit.
1207	1203	/// Don't forget to call `writeManifest` before this!
1208	1204	pub fn toOwnedLock(self: *Manifest) Lock {
1209		const lock: Lock = .{
1210		.manifest_file = self.manifest_file.?,
1211		};
1212
1213		self.manifest_file = null;
1214		return lock;
	1205	defer self.manifest_file = null;
	1206	return .{ .manifest_file = self.manifest_file.? };
1215	1207	}
1216	1208
1217	1209	/// Releases the manifest file and frees any memory the Manifest was using.
...	...	@@ -1223,7 +1215,7 @@ pub const Manifest = struct {
1223	1215	if (self.manifest_file) \|file\| {
1224	1216	if (builtin.os.tag == .windows) {
1225	1217	// See Lock.release for why this is required on Windows
1226		file.unlock();
	1218	file.unlock(io);
1227	1219	}
1228	1220
1229	1221	file.close(io);
...	...	@@ -1308,15 +1300,15 @@ pub fn writeSmallFile(dir: Io.Dir, sub_path: []const u8, data: []const u8) !void
1308	1300	}
1309	1301	}
1310	1302
1311		fn hashFile(file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.PReadError!void {
1312		var buf: [1024]u8 = undefined;
	1303	fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void {
	1304	var buffer: [2048]u8 = undefined;
1313	1305	var hasher = hasher_init;
1314		var off: u64 = 0;
	1306	var offset: u64 = 0;
1315	1307	while (true) {
1316		const bytes_read = try file.pread(&buf, off);
1317		if (bytes_read == 0) break;
1318		hasher.update(buf[0..bytes_read]);
1319		off += bytes_read;
	1308	const n = try file.readPositional(io, &.{&buffer}, offset);
	1309	if (n == 0) break;
	1310	hasher.update(buffer[0..n]);
	1311	offset += n;
1320	1312	}
1321	1313	hasher.final(bin_digest);
1322	1314	}

lib/std/Build/WebServer.zig+3-3

...	...	@@ -218,9 +218,9 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
218	218	else => {},
219	219	}
220	220	if (@bitSizeOf(usize) != 64) {
221		// Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
222		// being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
223		// on 32-bit platforms.
	221	// Current implementation depends on posix.mmap()'s second
	222	// parameter, `length: usize`, being compatible with file system's
	223	// u64 return value. This is not the case on 32-bit platforms.
224	224	// Affects or affected by issues #5185, #22523, and #22464.
225	225	std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
226	226	}

lib/std/Io.zig+2-2

...	...	@@ -692,9 +692,9 @@ pub const VTable = struct {
692	692	fileWriteFileStreaming: const fn (?anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit) File.Writer.WriteFileError!usize,
693	693	fileWriteFilePositional: const fn (?anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit, offset: u64) File.WriteFilePositionalError!usize,
694	694	/// Returns 0 on end of stream.
695		fileReadStreaming: const fn (?anyopaque, File, data: [][]u8) File.Reader.Error!usize,
	695	fileReadStreaming: const fn (?anyopaque, File, data: []const []u8) File.Reader.Error!usize,
696	696	/// Returns 0 on end of stream.
697		fileReadPositional: const fn (?anyopaque, File, data: [][]u8, offset: u64) File.ReadPositionalError!usize,
	697	fileReadPositional: const fn (?anyopaque, File, data: []const []u8, offset: u64) File.ReadPositionalError!usize,
698	698	fileSeekBy: const fn (?anyopaque, File, relative_offset: i64) File.SeekError!void,
699	699	fileSeekTo: const fn (?anyopaque, File, absolute_offset: u64) File.SeekError!void,
700	700	fileSync: const fn (?anyopaque, File) File.SyncError!void,

lib/std/Io/File.zig+32-2

...	...	@@ -466,13 +466,21 @@ pub fn setTimestampsNow(file: File, io: Io) SetTimestampsError!void {
466	466
467	467	pub const ReadPositionalError = Reader.Error \|\| error{Unseekable};
468	468
469		pub fn readPositional(file: File, io: Io, buffer: [][]u8, offset: u64) ReadPositionalError!usize {
	469	/// Returns 0 on end of stream.
	470	///
	471	/// See also:
	472	/// * `reader`
	473	pub fn readPositional(file: File, io: Io, buffer: []const []u8, offset: u64) ReadPositionalError!usize {
470	474	return io.vtable.fileReadPositional(io.userdata, file, buffer, offset);
471	475	}
472	476
473	477	pub const WritePositionalError = Writer.Error \|\| error{Unseekable};
474	478
475		pub fn writePositional(file: File, io: Io, buffer: [][]const u8, offset: u64) WritePositionalError!usize {
	479	/// Returns 0 on end of stream.
	480	///
	481	/// See also:
	482	/// * `writer`
	483	pub fn writePositional(file: File, io: Io, buffer: []const []const u8, offset: u64) WritePositionalError!usize {
476	484	return io.vtable.fileWritePositional(io.userdata, file, buffer, offset);
477	485	}
478	486
...	...	@@ -501,13 +509,35 @@ pub const WriteFilePositionalError = Writer.WriteFileError \|\| error{Unseekable};
501	509	///
502	510	/// Positional is more threadsafe, since the global seek position is not
503	511	/// affected.
	512	///
	513	/// See also:
	514	/// * `readerStreaming`
504	515	pub fn reader(file: File, io: Io, buffer: []u8) Reader {
505	516	return .init(file, io, buffer);
506	517	}
507	518
	519	/// Equivalent to creating a positional reader and reading multiple times to fill `buffer`.
	520	///
	521	/// Returns number of bytes read into `buffer`. If less than `buffer.len`, end of file occurred.
	522	///
	523	/// See also:
	524	/// * `reader`
	525	pub fn readPositionalAll(file: File, io: Io, buffer: []u8, offset: u64) ReadPositionalError!usize {
	526	var index: usize = 0;
	527	while (index != buffer.len) {
	528	const amt = try file.readPositional(io, &.{buffer[index..]}, offset + index);
	529	if (amt == 0) break;
	530	index += amt;
	531	}
	532	return index;
	533	}
	534
508	535	/// Positional is more threadsafe, since the global seek position is not
509	536	/// affected, but when such syscalls are not available, preemptively
510	537	/// initializing in streaming mode skips a failed syscall.
	538	///
	539	/// See also:
	540	/// * `reader`
511	541	pub fn readerStreaming(file: File, io: Io, buffer: []u8) Reader {
512	542	return .initStreaming(file, io, buffer);
513	543	}

lib/std/fs/test.zig+7-7

...	...	@@ -1455,7 +1455,7 @@ test "writev, readv" {
1455	1455
1456	1456	try writer.interface.writeVecAll(&write_vecs);
1457	1457	try writer.interface.flush();
1458		try testing.expectEqual(@as(u64, line1.len + line2.len), try src_file.getEndPos());
	1458	try testing.expectEqual(@as(u64, line1.len + line2.len), try src_file.length(io));
1459	1459
1460	1460	var reader = writer.moveToReader(io);
1461	1461	try reader.seekTo(0);
...	...	@@ -1486,7 +1486,7 @@ test "pwritev, preadv" {
1486	1486	try writer.seekTo(16);
1487	1487	try writer.interface.writeVecAll(&lines);
1488	1488	try writer.interface.flush();
1489		try testing.expectEqual(@as(u64, 16 + line1.len + line2.len), try src_file.getEndPos());
	1489	try testing.expectEqual(@as(u64, 16 + line1.len + line2.len), try src_file.length(io));
1490	1490
1491	1491	var reader = writer.moveToReader(io);
1492	1492	try reader.seekTo(16);
...	...	@@ -1511,13 +1511,13 @@ test "setEndPos" {
1511	1511	const f = try tmp.dir.openFile(io, file_name, .{ .mode = .read_write });
1512	1512	defer f.close(io);
1513	1513
1514		const initial_size = try f.getEndPos();
	1514	const initial_size = try f.length(io);
1515	1515	var buffer: [32]u8 = undefined;
1516	1516	var reader = f.reader(io, &.{});
1517	1517
1518	1518	{
1519	1519	try f.setEndPos(initial_size);
1520		try testing.expectEqual(initial_size, try f.getEndPos());
	1520	try testing.expectEqual(initial_size, try f.length(io));
1521	1521	try reader.seekTo(0);
1522	1522	try testing.expectEqual(initial_size, try reader.interface.readSliceShort(&buffer));
1523	1523	try testing.expectEqualStrings("ninebytes", buffer[0..@intCast(initial_size)]);
...	...	@@ -1526,7 +1526,7 @@ test "setEndPos" {
1526	1526	{
1527	1527	const larger = initial_size + 4;
1528	1528	try f.setEndPos(larger);
1529		try testing.expectEqual(larger, try f.getEndPos());
	1529	try testing.expectEqual(larger, try f.length(io));
1530	1530	try reader.seekTo(0);
1531	1531	try testing.expectEqual(larger, try reader.interface.readSliceShort(&buffer));
1532	1532	try testing.expectEqualStrings("ninebytes\x00\x00\x00\x00", buffer[0..@intCast(larger)]);
...	...	@@ -1535,14 +1535,14 @@ test "setEndPos" {
1535	1535	{
1536	1536	const smaller = initial_size - 5;
1537	1537	try f.setEndPos(smaller);
1538		try testing.expectEqual(smaller, try f.getEndPos());
	1538	try testing.expectEqual(smaller, try f.length(io));
1539	1539	try reader.seekTo(0);
1540	1540	try testing.expectEqual(smaller, try reader.interface.readSliceShort(&buffer));
1541	1541	try testing.expectEqualStrings("nine", buffer[0..@intCast(smaller)]);
1542	1542	}
1543	1543
1544	1544	try f.setEndPos(0);
1545		try testing.expectEqual(0, try f.getEndPos());
	1545	try testing.expectEqual(0, try f.length(io));
1546	1546	try reader.seekTo(0);
1547	1547	try testing.expectEqual(0, try reader.interface.readSliceShort(&buffer));
1548	1548	}

src/link/MachO/CodeSignature.zig+9-8

...	...	@@ -12,7 +12,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256;
12	12	const Allocator = std.mem.Allocator;
13	13
14	14	const trace = @import("../../tracy.zig").trace;
15		const Hasher = @import("hasher.zig").ParallelHasher;
	15	const ParallelHasher = @import("hasher.zig").ParallelHasher;
16	16	const MachO = @import("../MachO.zig");
17	17
18	18	const hash_size = Sha256.digest_length;
...	...	@@ -268,7 +268,9 @@ pub fn writeAdhocSignature(
268	268	const tracy = trace(@src());
269	269	defer tracy.end();
270	270
271		const allocator = macho_file.base.comp.gpa;
	271	const comp = macho_file.base.comp;
	272	const gpa = comp.gpa;
	273	const io = comp.io;
272	274
273	275	var header: macho.SuperBlob = .{
274	276	.magic = macho.CSMAGIC_EMBEDDED_SIGNATURE,
...	...	@@ -276,7 +278,7 @@ pub fn writeAdhocSignature(
276	278	.count = 0,
277	279	};
278	280
279		var blobs = std.array_list.Managed(Blob).init(allocator);
	281	var blobs = std.array_list.Managed(Blob).init(gpa);
280	282	defer blobs.deinit();
281	283
282	284	self.code_directory.inner.execSegBase = opts.exec_seg_base;
...	...	@@ -286,13 +288,12 @@ pub fn writeAdhocSignature(
286	288
287	289	const total_pages = @as(u32, @intCast(mem.alignForward(usize, opts.file_size, self.page_size) / self.page_size));
288	290
289		try self.code_directory.code_slots.ensureTotalCapacityPrecise(allocator, total_pages);
	291	try self.code_directory.code_slots.ensureTotalCapacityPrecise(gpa, total_pages);
290	292	self.code_directory.code_slots.items.len = total_pages;
291	293	self.code_directory.inner.nCodeSlots = total_pages;
292	294
293	295	// Calculate hash for each page (in file) and write it to the buffer
294		var hasher = Hasher(Sha256){ .allocator = allocator, .io = macho_file.base.comp.io };
295		try hasher.hash(opts.file, self.code_directory.code_slots.items, .{
	296	try ParallelHasher(Sha256).hash(gpa, io, opts.file, self.code_directory.code_slots.items, .{
296	297	.chunk_size = self.page_size,
297	298	.max_file_size = opts.file_size,
298	299	});
...	...	@@ -304,7 +305,7 @@ pub fn writeAdhocSignature(
304	305	var hash: [hash_size]u8 = undefined;
305	306
306	307	if (self.requirements) \|*req\| {
307		var a: std.Io.Writer.Allocating = .init(allocator);
	308	var a: std.Io.Writer.Allocating = .init(gpa);
308	309	defer a.deinit();
309	310	try req.write(&a.writer);
310	311	Sha256.hash(a.written(), &hash, .{});
...	...	@@ -316,7 +317,7 @@ pub fn writeAdhocSignature(
316	317	}
317	318
318	319	if (self.entitlements) \|*ents\| {
319		var a: std.Io.Writer.Allocating = .init(allocator);
	320	var a: std.Io.Writer.Allocating = .init(gpa);
320	321	defer a.deinit();
321	322	try ents.write(&a.writer);
322	323	Sha256.hash(a.written(), &hash, .{});

src/link/MachO/hasher.zig+9-15

...	...	@@ -1,5 +1,6 @@
1	1	const std = @import("std");
2	2	const Io = std.Io;
	3	const assert = std.debug.assert;
3	4	const Allocator = std.mem.Allocator;
4	5
5	6	const trace = @import("../../tracy.zig").trace;
...	...	@@ -8,20 +9,15 @@ pub fn ParallelHasher(comptime Hasher: type) type {
8	9	const hash_size = Hasher.digest_length;
9	10
10	11	return struct {
11		allocator: Allocator,
12		io: std.Io,
13
14		pub fn hash(self: Self, file: Io.File, out: [][hash_size]u8, opts: struct {
	12	pub fn hash(self: Self, io: Io, file: Io.File, out: [][hash_size]u8, opts: struct {
15	13	chunk_size: u64 = 0x4000,
16	14	max_file_size: ?u64 = null,
17	15	}) !void {
18	16	const tracy = trace(@src());
19	17	defer tracy.end();
20	18
21		const io = self.io;
22
23	19	const file_size = blk: {
24		const file_size = opts.max_file_size orelse try file.getEndPos();
	20	const file_size = opts.max_file_size orelse try file.length(io);
25	21	break :blk std.math.cast(usize, file_size) orelse return error.Overflow;
26	22	};
27	23	const chunk_size = std.math.cast(usize, opts.chunk_size) orelse return error.Overflow;
...	...	@@ -29,12 +25,12 @@ pub fn ParallelHasher(comptime Hasher: type) type {
29	25	const buffer = try self.allocator.alloc(u8, chunk_size * out.len);
30	26	defer self.allocator.free(buffer);
31	27
32		const results = try self.allocator.alloc(Io.File.PReadError!usize, out.len);
	28	const results = try self.allocator.alloc(Io.File.ReadPositionalError!usize, out.len);
33	29	defer self.allocator.free(results);
34	30
35	31	{
36		var group: std.Io.Group = .init;
37		errdefer group.cancel(io);
	32	var group: Io.Group = .init;
	33	defer group.cancel(io);
38	34
39	35	for (out, results, 0..) \|out_buf, result, i\| {
40	36	const fstart = i * chunk_size;
...	...	@@ -42,7 +38,7 @@ pub fn ParallelHasher(comptime Hasher: type) type {
42	38	file_size - fstart
43	39	else
44	40	chunk_size;
45		group.async(io, worker, .{
	41	group.async(worker, .{
46	42	file,
47	43	fstart,
48	44	buffer[fstart..][0..fsize],
...	...	@@ -61,11 +57,9 @@ pub fn ParallelHasher(comptime Hasher: type) type {
61	57	fstart: usize,
62	58	buffer: []u8,
63	59	out: *[hash_size]u8,
64		err: *Io.File.PReadError!usize,
	60	err: *Io.File.ReadPositionalError!usize,
65	61	) void {
66		const tracy = trace(@src());
67		defer tracy.end();
68		err.* = file.preadAll(buffer, fstart);
	62	err.* = file.readPositionalAll(buffer, fstart);
69	63	Hasher.hash(buffer, out, .{});
70	64	}
71	65

src/link/MachO/uuid.zig+9-7

...	...	@@ -4,7 +4,7 @@ const Md5 = std.crypto.hash.Md5;
4	4
5	5	const trace = @import("../../tracy.zig").trace;
6	6	const Compilation = @import("../../Compilation.zig");
7		const Hasher = @import("hasher.zig").ParallelHasher;
	7	const ParallelHasher = @import("hasher.zig").ParallelHasher;
8	8
9	9	/// Calculates Md5 hash of each chunk in parallel and then hashes all Md5 hashes to produce
10	10	/// the final digest.
...	...	@@ -16,21 +16,23 @@ pub fn calcUuid(comp: const Compilation, file: Io.File, file_size: u64, out: [
16	16	const tracy = trace(@src());
17	17	defer tracy.end();
18	18
	19	const gpa = comp.gpa;
	20	const io = comp.io;
	21
19	22	const chunk_size: usize = 1024 * 1024;
20	23	const num_chunks: usize = std.math.cast(usize, @divTrunc(file_size, chunk_size)) orelse return error.Overflow;
21	24	const actual_num_chunks = if (@rem(file_size, chunk_size) > 0) num_chunks + 1 else num_chunks;
22	25
23		const hashes = try comp.gpa.alloc([Md5.digest_length]u8, actual_num_chunks);
24		defer comp.gpa.free(hashes);
	26	const hashes = try gpa.alloc([Md5.digest_length]u8, actual_num_chunks);
	27	defer gpa.free(hashes);
25	28
26		var hasher = Hasher(Md5){ .allocator = comp.gpa, .io = comp.io };
27		try hasher.hash(file, hashes, .{
	29	try ParallelHasher(Md5).hash(gpa, io, file, hashes, .{
28	30	.chunk_size = chunk_size,
29	31	.max_file_size = file_size,
30	32	});
31	33
32		const final_buffer = try comp.gpa.alloc(u8, actual_num_chunks * Md5.digest_length);
33		defer comp.gpa.free(final_buffer);
	34	const final_buffer = try gpa.alloc(u8, actual_num_chunks * Md5.digest_length);
	35	defer gpa.free(final_buffer);
34	36
35	37	for (hashes, 0..) \|hash, i\| {
36	38	@memcpy(final_buffer[i * Md5.digest_length ..][0..Md5.digest_length], &hash);

author	Andrew Kelley <andrew@ziglang.org> 2025-12-08 15:23:18-08:00
committer	Andrew Kelley <andrew@ziglang.org> 2025-12-23 22:15:08-08:00
log	6f46570958af8ae27308eb4a9470e05f33aaa522
tree	19aec2fa52364c78ffa9a9d8dc14d335f664fe06
parent	181ac08459f8d4001c504330ee66037135e56908

link.MachO: update parallel hasher to std.Io