authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-09 16:49:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-09 16:49:05-07:00
logfbc6a00b0a939f8752bbd571284c46ab58a8fcc4
treef1543b917d9407611e014e1a79d26b0eed267621
parent04b0ffdd13e32be0ef5cc84983f8bb830db7520f
parent9f8f4464353460825856b848dfe2480de20d8d55

Merge branch 'mlarouche-Fix_6500'

8 files changed, 90 insertions(+), 29 deletions(-)

lib/std/fs.zig+13
...@@ -1490,6 +1490,19 @@ pub const Dir = struct {...@@ -1490,6 +1490,19 @@ pub const Dir = struct {
1490 return os.windows.ReadLink(self.fd, sub_path_w, buffer);1490 return os.windows.ReadLink(self.fd, sub_path_w, buffer);
1491 }1491 }
14921492
1493 /// Read all of file contents using a preallocated buffer.
1494 /// The returned slice has the same pointer as `buffer`. If the length matches `buffer.len`
1495 /// the situation is ambiguous. It could either mean that the entire file was read, and
1496 /// it exactly fits the buffer, or it could mean the buffer was not big enough for the
1497 /// entire file.
1498 pub fn readFile(self: Dir, file_path: []const u8, buffer: []u8) ![]u8 {
1499 var file = try self.openFile(file_path, .{});
1500 defer file.close();
1501
1502 const end_index = try file.readAll(buffer);
1503 return buffer[0..end_index];
1504 }
1505
1493 /// On success, caller owns returned buffer.1506 /// On success, caller owns returned buffer.
1494 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.1507 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
1495 pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 {1508 pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 {
src/Cache.zig+24
...@@ -576,6 +576,30 @@ pub const Manifest = struct {...@@ -576,6 +576,30 @@ pub const Manifest = struct {
576 }576 }
577};577};
578578
579/// On operating systems that support symlinks, does a readlink. On other operating systems,
580/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
581/// it is treated as not supporting symlinks.
582pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
583 if (std.Target.current.os.tag == .windows) {
584 return dir.readFile(sub_path, buffer);
585 } else {
586 return dir.readLink(sub_path, buffer);
587 }
588}
589
590/// On operating systems that support symlinks, does a symlink. On other operating systems,
591/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
592/// it is treated as not supporting symlinks.
593/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
594pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
595 assert(data.len <= 255);
596 if (std.Target.current.os.tag == .windows) {
597 return dir.writeFile(sub_path, data);
598 } else {
599 return dir.symLink(data, sub_path, .{});
600 }
601}
602
579fn hashFile(file: fs.File, bin_digest: []u8) !void {603fn hashFile(file: fs.File, bin_digest: []u8) !void {
580 var buf: [1024]u8 = undefined;604 var buf: [1024]u8 = undefined;
581605
src/Compilation.zig+10-6
...@@ -2605,8 +2605,12 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node...@@ -2605,8 +2605,12 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
26052605
2606 // We use an extra hex-encoded byte here to store some flags.2606 // We use an extra hex-encoded byte here to store some flags.
2607 var prev_digest_buf: [digest.len + 2]u8 = undefined;2607 var prev_digest_buf: [digest.len + 2]u8 = undefined;
2608 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {2608 const prev_digest: []u8 = Cache.readSmallFile(
2609 log.debug("stage1 {} new_digest={} readlink error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });2609 directory.handle,
2610 id_symlink_basename,
2611 &prev_digest_buf,
2612 ) catch |err| blk: {
2613 log.debug("stage1 {} new_digest={} error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) });
2610 // Handle this as a cache miss.2614 // Handle this as a cache miss.
2611 break :blk prev_digest_buf[0..0];2615 break :blk prev_digest_buf[0..0];
2612 };2616 };
...@@ -2777,7 +2781,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node...@@ -2777,7 +2781,7 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
27772781
2778 const digest = man.final();2782 const digest = man.final();
27792783
2780 // Update the dangling symlink with the digest. If it fails we can continue; it only2784 // Update the small file with the digest. If it fails we can continue; it only
2781 // means that the next invocation will have an unnecessary cache miss.2785 // means that the next invocation will have an unnecessary cache miss.
2782 const stage1_flags_byte = @bitCast(u8, mod.stage1_flags);2786 const stage1_flags_byte = @bitCast(u8, mod.stage1_flags);
2783 log.debug("stage1 {} final digest={} flags={x}", .{2787 log.debug("stage1 {} final digest={} flags={x}", .{
...@@ -2792,10 +2796,10 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node...@@ -2792,10 +2796,10 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node
2792 log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{2796 log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{
2793 digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup,2797 digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup,
2794 });2798 });
2795 directory.handle.symLink(&digest_plus_flags, id_symlink_basename, .{}) catch |err| {2799 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest_plus_flags) catch |err| {
2796 log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)});2800 log.warn("failed to save stage1 hash digest file: {}", .{@errorName(err)});
2797 };2801 };
2798 // Again failure here only means an unnecessary cache miss.2802 // Failure here only means an unnecessary cache miss.
2799 man.writeManifest() catch |err| {2803 man.writeManifest() catch |err| {
2800 log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});2804 log.warn("failed to write cache manifest when linking: {}", .{@errorName(err)});
2801 };2805 };
src/link.zig+8-4
...@@ -466,8 +466,12 @@ pub const File = struct {...@@ -466,8 +466,12 @@ pub const File = struct {
466 const digest = ch.final();466 const digest = ch.final();
467467
468 var prev_digest_buf: [digest.len]u8 = undefined;468 var prev_digest_buf: [digest.len]u8 = undefined;
469 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| b: {469 const prev_digest: []u8 = Cache.readSmallFile(
470 log.debug("archive new_digest={} readlink error: {}", .{ digest, @errorName(err) });470 directory.handle,
471 id_symlink_basename,
472 &prev_digest_buf,
473 ) catch |err| b: {
474 log.debug("archive new_digest={} readFile error: {}", .{ digest, @errorName(err) });
471 break :b prev_digest_buf[0..0];475 break :b prev_digest_buf[0..0];
472 };476 };
473 if (mem.eql(u8, prev_digest, &digest)) {477 if (mem.eql(u8, prev_digest, &digest)) {
...@@ -512,8 +516,8 @@ pub const File = struct {...@@ -512,8 +516,8 @@ pub const File = struct {
512 const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type);516 const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type);
513 if (bad) return error.UnableToWriteArchive;517 if (bad) return error.UnableToWriteArchive;
514518
515 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {519 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
516 std.log.warn("failed to save archive hash digest symlink: {}", .{@errorName(err)});520 std.log.warn("failed to save archive hash digest file: {}", .{@errorName(err)});
517 };521 };
518522
519 ch.writeManifest() catch |err| {523 ch.writeManifest() catch |err| {
src/link/Coff.zig+9-5
...@@ -854,8 +854,12 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {...@@ -854,8 +854,12 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
854 _ = try man.hit();854 _ = try man.hit();
855 digest = man.final();855 digest = man.final();
856 var prev_digest_buf: [digest.len]u8 = undefined;856 var prev_digest_buf: [digest.len]u8 = undefined;
857 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {857 const prev_digest: []u8 = Cache.readSmallFile(
858 log.debug("COFF LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) });858 directory.handle,
859 id_symlink_basename,
860 &prev_digest_buf,
861 ) catch |err| blk: {
862 log.debug("COFF LLD new_digest={} error: {}", .{ digest, @errorName(err) });
859 // Handle this as a cache miss.863 // Handle this as a cache miss.
860 break :blk prev_digest_buf[0..0];864 break :blk prev_digest_buf[0..0];
861 };865 };
...@@ -1180,10 +1184,10 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {...@@ -1180,10 +1184,10 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void {
1180 }1184 }
11811185
1182 if (!self.base.options.disable_lld_caching) {1186 if (!self.base.options.disable_lld_caching) {
1183 // Update the dangling symlink with the digest. If it fails we can continue; it only1187 // Update the file with the digest. If it fails we can continue; it only
1184 // means that the next invocation will have an unnecessary cache miss.1188 // means that the next invocation will have an unnecessary cache miss.
1185 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {1189 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
1186 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});1190 std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)});
1187 };1191 };
1188 // Again failure here only means an unnecessary cache miss.1192 // Again failure here only means an unnecessary cache miss.
1189 man.writeManifest() catch |err| {1193 man.writeManifest() catch |err| {
src/link/Elf.zig+9-5
...@@ -1326,8 +1326,12 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1326,8 +1326,12 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
1326 digest = man.final();1326 digest = man.final();
13271327
1328 var prev_digest_buf: [digest.len]u8 = undefined;1328 var prev_digest_buf: [digest.len]u8 = undefined;
1329 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {1329 const prev_digest: []u8 = Cache.readSmallFile(
1330 log.debug("ELF LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) });1330 directory.handle,
1331 id_symlink_basename,
1332 &prev_digest_buf,
1333 ) catch |err| blk: {
1334 log.debug("ELF LLD new_digest={} error: {}", .{ digest, @errorName(err) });
1331 // Handle this as a cache miss.1335 // Handle this as a cache miss.
1332 break :blk prev_digest_buf[0..0];1336 break :blk prev_digest_buf[0..0];
1333 };1337 };
...@@ -1647,10 +1651,10 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {...@@ -1647,10 +1651,10 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void {
1647 }1651 }
16481652
1649 if (!self.base.options.disable_lld_caching) {1653 if (!self.base.options.disable_lld_caching) {
1650 // Update the dangling symlink with the digest. If it fails we can continue; it only1654 // Update the file with the digest. If it fails we can continue; it only
1651 // means that the next invocation will have an unnecessary cache miss.1655 // means that the next invocation will have an unnecessary cache miss.
1652 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {1656 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
1653 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});1657 std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)});
1654 };1658 };
1655 // Again failure here only means an unnecessary cache miss.1659 // Again failure here only means an unnecessary cache miss.
1656 man.writeManifest() catch |err| {1660 man.writeManifest() catch |err| {
src/link/MachO.zig+9-5
...@@ -419,8 +419,12 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {...@@ -419,8 +419,12 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
419 digest = man.final();419 digest = man.final();
420420
421 var prev_digest_buf: [digest.len]u8 = undefined;421 var prev_digest_buf: [digest.len]u8 = undefined;
422 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {422 const prev_digest: []u8 = Cache.readSmallFile(
423 log.debug("MachO LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) });423 directory.handle,
424 id_symlink_basename,
425 &prev_digest_buf,
426 ) catch |err| blk: {
427 log.debug("MachO LLD new_digest={} error: {}", .{ digest, @errorName(err) });
424 // Handle this as a cache miss.428 // Handle this as a cache miss.
425 break :blk prev_digest_buf[0..0];429 break :blk prev_digest_buf[0..0];
426 };430 };
...@@ -674,10 +678,10 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {...@@ -674,10 +678,10 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void {
674 }678 }
675679
676 if (!self.base.options.disable_lld_caching) {680 if (!self.base.options.disable_lld_caching) {
677 // Update the dangling symlink with the digest. If it fails we can continue; it only681 // Update the file with the digest. If it fails we can continue; it only
678 // means that the next invocation will have an unnecessary cache miss.682 // means that the next invocation will have an unnecessary cache miss.
679 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {683 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
680 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});684 std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)});
681 };685 };
682 // Again failure here only means an unnecessary cache miss.686 // Again failure here only means an unnecessary cache miss.
683 man.writeManifest() catch |err| {687 man.writeManifest() catch |err| {
src/link/Wasm.zig+8-4
...@@ -310,8 +310,12 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {...@@ -310,8 +310,12 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
310 digest = man.final();310 digest = man.final();
311311
312 var prev_digest_buf: [digest.len]u8 = undefined;312 var prev_digest_buf: [digest.len]u8 = undefined;
313 const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: {313 const prev_digest: []u8 = Cache.readSmallFile(
314 log.debug("WASM LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) });314 directory.handle,
315 id_symlink_basename,
316 &prev_digest_buf,
317 ) catch |err| blk: {
318 log.debug("WASM LLD new_digest={} error: {}", .{ digest, @errorName(err) });
315 // Handle this as a cache miss.319 // Handle this as a cache miss.
316 break :blk prev_digest_buf[0..0];320 break :blk prev_digest_buf[0..0];
317 };321 };
...@@ -424,9 +428,9 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {...@@ -424,9 +428,9 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void {
424 }428 }
425429
426 if (!self.base.options.disable_lld_caching) {430 if (!self.base.options.disable_lld_caching) {
427 // Update the dangling symlink with the digest. If it fails we can continue; it only431 // Update the file with the digest. If it fails we can continue; it only
428 // means that the next invocation will have an unnecessary cache miss.432 // means that the next invocation will have an unnecessary cache miss.
429 directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| {433 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
430 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});434 std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)});
431 };435 };
432 // Again failure here only means an unnecessary cache miss.436 // Again failure here only means an unnecessary cache miss.