| ... | ... | @@ -56,71 +56,60 @@ pub fn prefixes(cache: *const Cache) []const Directory { |
| 56 | 56 | return cache.prefixes_buffer[0..cache.prefixes_len]; |
| 57 | 57 | } |
| 58 | 58 | |
| 59 | | pub const PrefixedPath = struct { |
| 60 | | prefix: u8, |
| 61 | | sub_path: []const u8, |
| 62 | | |
| 63 | | fn eql(a: PrefixedPath, b: PrefixedPath) bool { |
| 64 | | return a.prefix == b.prefix and mem.eql(u8, a.sub_path, b.sub_path); |
| 65 | | } |
| 59 | const PrefixIndex = u6; |
| 66 | 60 | |
| 67 | | fn hash(pp: PrefixedPath) u32 { |
| 68 | | return @truncate(std.hash.Wyhash.hash(pp.prefix, pp.sub_path)); |
| 69 | | } |
| 61 | const PrefixedPath = struct { |
| 62 | prefix: PrefixIndex, |
| 63 | sub_path: []const u8, |
| 70 | 64 | }; |
| 71 | 65 | |
| 72 | | fn findPrefixPath(cache: *const Cache, path: Path) !PrefixedPath { |
| 73 | | const gpa = cache.gpa; |
| 74 | | const resolved_path = try std.fs.path.resolve(gpa, &.{ |
| 75 | | cache.cwd, path.root_dir.path orelse ".", path.subPathOrDot(), |
| 76 | | }); |
| 77 | | errdefer gpa.free(resolved_path); |
| 78 | | return findPrefixResolved(cache, resolved_path); |
| 79 | | } |
| 80 | | |
| 81 | | fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath { |
| 82 | | const gpa = cache.gpa; |
| 83 | | const resolved_path = try std.fs.path.resolve(gpa, &.{file_path}); |
| 84 | | errdefer gpa.free(resolved_path); |
| 85 | | return findPrefixResolved(cache, resolved_path); |
| 66 | fn appendPrefixedPath(cache: *const Cache, contents: *std.ArrayList(u8), prefixed_path: PrefixedPath) !PrefixIndex { |
| 67 | const end = contents.items.len + prefixed_path.sub_path.len; |
| 68 | const needed_alignment = @alignOf(Manifest.File) - (end % @alignOf(Manifest.File)); |
| 69 | assert(needed_alignment >= 1); // Always need at least a null byte. |
| 70 | try contents.ensureTotalCapacity(cache.gpa, end + needed_alignment); |
| 71 | contents.appendSliceAssumeCapacity(prefixed_path.sub_path); |
| 72 | contents.appendNTimesAssumeCapacity(0, needed_alignment); |
| 73 | return prefixed_path.prefix; |
| 86 | 74 | } |
| 87 | 75 | |
| 88 | | /// Takes ownership of `resolved_path` on success. |
| 89 | | fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath { |
| 76 | fn resolveAppendPath(cache: *const Cache, contents: *std.ArrayList(u8), path: Path) !PrefixIndex { |
| 90 | 77 | const gpa = cache.gpa; |
| 91 | 78 | const cwd = cache.cwd; |
| 79 | const path_start = contents.items.len; |
| 80 | |
| 81 | const resolved_path = try std.fs.path.resolveAlloc(gpa, &.{ |
| 82 | path.root_dir.path orelse cwd, |
| 83 | path.subPathOrDot(), |
| 84 | }); |
| 85 | defer gpa.free(resolved_path); |
| 86 | |
| 92 | 87 | for (cache.prefixes(), 0..) |prefix, i| { |
| 93 | | const p = prefix.path orelse continue; |
| 94 | | const sub_path = getPrefixSubpath(gpa, cwd, p, resolved_path) catch |err| switch (err) { |
| 95 | | error.NotASubPath => continue, |
| 96 | | else => |e| return e, |
| 97 | | }; |
| 98 | | // Free the resolved path since we're not going to return it |
| 99 | | gpa.free(resolved_path); |
| 100 | | return .{ |
| 101 | | .prefix = @intCast(i), |
| 102 | | .sub_path = sub_path, |
| 103 | | }; |
| 104 | | } |
| 88 | const pp = prefix.path orelse continue; |
| 89 | contents.shrinkRetainingCapacity(path_start); |
| 90 | try std.fs.path.relativeAppend(gpa, contents, cwd, null, pp, resolved_path); |
| 91 | const relative = contents.items[path_start..]; |
| 105 | 92 | |
| 106 | | return .{ |
| 107 | | .prefix = 0, |
| 108 | | .sub_path = resolved_path, |
| 109 | | }; |
| 110 | | } |
| 93 | var component_iterator: std.fs.path.NativeComponentIterator = .init(relative); |
| 94 | if (component_iterator.root() != null) continue; |
| 95 | const first_component = component_iterator.first(); |
| 96 | if (first_component != null and mem.eql(u8, first_component.?.name, "..")) continue; |
| 111 | 97 | |
| 112 | | fn getPrefixSubpath(gpa: Allocator, cwd: []const u8, prefix: []const u8, path: []u8) ![]u8 { |
| 113 | | const relative = try std.fs.path.relative(gpa, cwd, null, prefix, path); |
| 114 | | errdefer gpa.free(relative); |
| 115 | | var component_iterator: std.fs.path.NativeComponentIterator = .init(relative); |
| 116 | | if (component_iterator.root() != null) { |
| 117 | | return error.NotASubPath; |
| 118 | | } |
| 119 | | const first_component = component_iterator.first(); |
| 120 | | if (first_component != null and mem.eql(u8, first_component.?.name, "..")) { |
| 121 | | return error.NotASubPath; |
| 98 | const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); |
| 99 | assert(needed_alignment >= 1); // Always need at least a null byte. |
| 100 | try contents.appendNTimes(gpa, 0, needed_alignment); |
| 101 | |
| 102 | return @intCast(i); |
| 122 | 103 | } |
| 123 | | return relative; |
| 104 | |
| 105 | contents.shrinkRetainingCapacity(path_start); |
| 106 | try contents.appendSlice(gpa, resolved_path); |
| 107 | |
| 108 | const needed_alignment = @alignOf(Manifest.File) - (contents.items.len % @alignOf(Manifest.File)); |
| 109 | assert(needed_alignment >= 1); // Always need at least a null byte. |
| 110 | try contents.appendNTimes(gpa, 0, needed_alignment); |
| 111 | |
| 112 | return 0; |
| 124 | 113 | } |
| 125 | 114 | |
| 126 | 115 | /// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 |
| ... | ... | @@ -359,7 +348,10 @@ pub const Manifest = struct { |
| 359 | 348 | _, |
| 360 | 349 | }, |
| 361 | 350 | /// `have_handle` determines whether this is populated. |
| 362 | | handle: Io.File, |
| 351 | handle: union { |
| 352 | file: Io.File, |
| 353 | dir: Io.Dir, |
| 354 | }, |
| 363 | 355 | |
| 364 | 356 | /// Index into `Manifest.input_paths`. |
| 365 | 357 | pub const Index = enum(u32) { |
| ... | ... | @@ -383,7 +375,7 @@ pub const Manifest = struct { |
| 383 | 375 | pub const Flags = packed struct(u8) { |
| 384 | 376 | is_directory: bool, |
| 385 | 377 | metadata_only: bool, |
| 386 | | prefix: u6, |
| 378 | prefix: PrefixIndex, |
| 387 | 379 | }; |
| 388 | 380 | |
| 389 | 381 | /// Prefixes path names in encoded directory contents. Starts numbering |
| ... | ... | @@ -408,40 +400,49 @@ pub const Manifest = struct { |
| 408 | 400 | _, |
| 409 | 401 | |
| 410 | 402 | pub fn get(offset: Offset, contents: []u8) *File { |
| 411 | | return @ptrCast(@alignCast(contents.items[@backingInt(offset)..][0..@sizeOf(File)])); |
| 403 | return @constCast(getConst(offset, contents)); |
| 404 | } |
| 405 | |
| 406 | pub fn getConst(offset: Offset, contents: []const u8) *const File { |
| 407 | return @ptrCast(@alignCast(contents[@backingInt(offset)..][0..@sizeOf(File)])); |
| 412 | 408 | } |
| 413 | 409 | |
| 414 | 410 | pub fn getFallible(offset: Offset, contents: []u8) error{InvalidFormat}!*File { |
| 415 | | if (@backingInt(offset) + @sizeOf(File) >= contents.items.len) return error.InvalidFormat; |
| 411 | if (@backingInt(offset) + @sizeOf(File) >= contents.len) return error.InvalidFormat; |
| 416 | 412 | return get(offset, contents); |
| 417 | 413 | } |
| 418 | 414 | }; |
| 419 | 415 | |
| 416 | /// Intentionally matches if the files are different only by flags other than prefix. |
| 420 | 417 | pub const HashContext = struct { |
| 421 | | manifest: *const Manifest, |
| 418 | contents: []const u8, |
| 422 | 419 | |
| 423 | 420 | pub fn hash(this: @This(), off: Offset) u32 { |
| 424 | | const file = off.get(this.manifest); |
| 425 | | return @truncate(std.hash.Wyhash.hash(file.prefix, file.path())); |
| 421 | const file_prefix = off.getConst(this.contents).flags.prefix; |
| 422 | const file_path = filePath(this.contents, off); |
| 423 | return @truncate(std.hash.Wyhash.hash(file_prefix, file_path)); |
| 426 | 424 | } |
| 427 | 425 | |
| 428 | 426 | pub fn eql(this: @This(), a_off: Offset, b_off: Offset, b_index: usize) bool { |
| 429 | 427 | _ = b_index; |
| 430 | | const a = a_off.get(this.manifest); |
| 431 | | const b = b_off.get(this.manifest); |
| 432 | | return a.prefix == b.prefix and mem.eql(u8, a.path(), b.path()); |
| 428 | const a_prefix = a_off.getConst(this.contents).flags.prefix; |
| 429 | const b_prefix = b_off.getConst(this.contents).flags.prefix; |
| 430 | if (a_prefix != b_prefix) return false; |
| 431 | const a_path = filePath(this.contents, a_off); |
| 432 | const b_path = filePath(this.contents, b_off); |
| 433 | return mem.eql(u8, a_path, b_path); |
| 433 | 434 | } |
| 434 | 435 | }; |
| 435 | 436 | |
| 436 | 437 | fn setStat(file: *File, m: *Manifest, stat: Stat) Io.Cancelable!void { |
| 437 | 438 | file.size = stat.size; |
| 438 | 439 | file.inode = stat.inode; |
| 439 | | file.mtime = stat.mtime; |
| 440 | file.mtime = @intCast(stat.mtime.toNanoseconds()); |
| 440 | 441 | |
| 441 | 442 | if (try m.isProblematicTimestamp(stat.mtime)) { |
| 442 | 443 | // The actual file has an unreliable timestamp; force it to be hashed. |
| 443 | | file.stat.mtime = 0; |
| 444 | | file.stat.inode = 0; |
| 444 | file.mtime = 0; |
| 445 | file.inode = 0; |
| 445 | 446 | } |
| 446 | 447 | } |
| 447 | 448 | |
| ... | ... | @@ -453,7 +454,7 @@ pub const Manifest = struct { |
| 453 | 454 | { |
| 454 | 455 | return false; |
| 455 | 456 | } else { |
| 456 | | setStat(file, m, stat); |
| 457 | try setStat(file, m, stat); |
| 457 | 458 | return true; |
| 458 | 459 | } |
| 459 | 460 | } |
| ... | ... | @@ -489,12 +490,31 @@ pub const Manifest = struct { |
| 489 | 490 | size: u64, |
| 490 | 491 | inode: Io.File.INode, |
| 491 | 492 | mtime: Io.Timestamp, |
| 493 | |
| 494 | pub fn init(other: Io.File.Stat) Stat { |
| 495 | return .{ |
| 496 | .size = other.size, |
| 497 | .inode = other.inode, |
| 498 | .mtime = other.mtime, |
| 499 | }; |
| 500 | } |
| 492 | 501 | }; |
| 493 | 502 | |
| 494 | 503 | pub const PathHandle = union(enum) { |
| 495 | 504 | file: ?Io.File, |
| 496 | 505 | /// If provided, this handle must be opened with iteration capability. |
| 497 | 506 | dir: ?Io.Dir, |
| 507 | |
| 508 | pub fn isDirectory(this: @This()) bool { |
| 509 | return this == .dir; |
| 510 | } |
| 511 | |
| 512 | pub fn have(this: @This()) bool { |
| 513 | return switch (this) { |
| 514 | .file => |opt_file| opt_file != null, |
| 515 | .dir => |opt_dir| opt_dir != null, |
| 516 | }; |
| 517 | } |
| 498 | 518 | }; |
| 499 | 519 | |
| 500 | 520 | pub const AddInputPathOptions = struct { |
| ... | ... | @@ -524,59 +544,73 @@ pub const Manifest = struct { |
| 524 | 544 | /// See also: |
| 525 | 545 | /// * `addPathPost` |
| 526 | 546 | pub fn addInputPath(m: *Manifest, path: Path, options: AddInputPathOptions) Allocator.Error!InputPath.Index { |
| 527 | | const gpa = m.cache.gpa; |
| 528 | | try m.files.ensureUnusedCapacity(gpa, 1); |
| 547 | const cache = m.cache; |
| 548 | const gpa = cache.gpa; |
| 549 | try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); |
| 529 | 550 | try m.input_paths.ensureUnusedCapacity(gpa, 1); |
| 530 | 551 | |
| 531 | 552 | const prev_contents_len = m.contents.items.len; |
| 532 | | const header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))); |
| 553 | const header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)))); |
| 533 | 554 | errdefer m.contents.shrinkRetainingCapacity(prev_contents_len); |
| 534 | 555 | |
| 535 | 556 | header.* = .{ |
| 536 | 557 | .flags = .{ |
| 537 | | .prefix = try m.cache.findAppendPrefixedPath(&m.contents, path), |
| 538 | | .is_directory = options.is_directory, |
| 558 | .prefix = try cache.resolveAppendPath(&m.contents, path), |
| 559 | .is_directory = options.handle.isDirectory(), |
| 539 | 560 | .metadata_only = options.metadata_only, |
| 540 | 561 | }, |
| 541 | 562 | .size = undefined, |
| 542 | 563 | .inode = undefined, |
| 543 | 564 | .mtime = undefined, |
| 544 | 565 | .digest = undefined, |
| 566 | .path_start = .{}, |
| 545 | 567 | }; |
| 546 | | assert(m.contents.items.len % @alignOf(File) == 0); |
| 568 | assert(mem.isAligned(m.contents.items.len, @alignOf(File))); |
| 547 | 569 | |
| 548 | | const gop = try m.files.getOrPutAssumeCapacityContext(@fromBackingInt(prev_contents_len), .{ |
| 549 | | .manifest = m, |
| 570 | const gop = m.files.getOrPutAssumeCapacityContext(@fromBackingInt(@intCast(prev_contents_len)), .{ |
| 571 | .contents = m.contents.items, |
| 550 | 572 | }); |
| 573 | m.files.lockPointers(); |
| 574 | defer m.files.unlockPointers(); |
| 575 | |
| 551 | 576 | if (gop.found_existing) { |
| 552 | 577 | m.contents.shrinkRetainingCapacity(prev_contents_len); |
| 553 | 578 | const existing_input_file = &m.input_paths.items[gop.index]; |
| 554 | | if (options.handle) |handle| { |
| 555 | | existing_input_file.handle = handle; |
| 556 | | existing_input_file.have_handle = true; |
| 579 | switch (options.handle) { |
| 580 | .file => |opt_file| if (opt_file) |file| { |
| 581 | existing_input_file.handle = .{ .file = file }; |
| 582 | existing_input_file.have_handle = true; |
| 583 | }, |
| 584 | .dir => |opt_dir| if (opt_dir) |dir| { |
| 585 | existing_input_file.handle = .{ .dir = dir }; |
| 586 | existing_input_file.have_handle = true; |
| 587 | }, |
| 557 | 588 | } |
| 558 | 589 | if (options.request_contents) switch (existing_input_file.contents) { |
| 559 | 590 | .requested, .not_requested => existing_input_file.contents = .requested, |
| 560 | 591 | _ => {}, |
| 561 | 592 | }; |
| 562 | | const existing_header = &m.files.keys()[gop.index]; |
| 593 | const existing_header = m.files.keys()[gop.index].get(m.contents.items); |
| 563 | 594 | if (options.stat) |stat| { |
| 564 | 595 | existing_input_file.have_stat = true; |
| 565 | 596 | existing_header.size = stat.size; |
| 566 | 597 | existing_header.inode = stat.inode; |
| 567 | | existing_header.mtime = stat.mtime; |
| 598 | existing_header.mtime = @intCast(stat.mtime.toNanoseconds()); |
| 568 | 599 | } |
| 569 | 600 | // If it trips, the same file path has been added to the cache |
| 570 | 601 | // manifest both as a directory and as a normal file, making the |
| 571 | 602 | // intended caching behavior ambiguous. |
| 572 | | assert(existing_header.flags.is_directory == options.is_directory); |
| 603 | assert(existing_header.flags.is_directory == options.handle.isDirectory()); |
| 573 | 604 | if (!options.metadata_only) |
| 574 | 605 | existing_header.flags.metadata_only = false; |
| 575 | 606 | } else { |
| 576 | 607 | m.input_paths.appendAssumeCapacity(.{ |
| 577 | 608 | .request_handle = options.request_handle, |
| 578 | | .have_handle = options.handle != null, |
| 579 | | .handle = if (options.handle) |handle| handle else undefined, |
| 609 | .have_handle = options.handle.have(), |
| 610 | .handle = switch (options.handle) { |
| 611 | .file => |opt_file| if (opt_file) |file| .{ .file = file } else undefined, |
| 612 | .dir => |opt_dir| if (opt_dir) |dir| .{ .dir = dir } else undefined, |
| 613 | }, |
| 580 | 614 | .contents = if (options.request_contents) .requested else .not_requested, |
| 581 | 615 | .have_digest = false, |
| 582 | 616 | .have_stat = options.stat != null, |
| ... | ... | @@ -585,10 +619,10 @@ pub const Manifest = struct { |
| 585 | 619 | if (options.stat) |stat| { |
| 586 | 620 | header.size = stat.size; |
| 587 | 621 | header.inode = stat.inode; |
| 588 | | header.mtime = stat.mtime; |
| 622 | header.mtime = @intCast(stat.mtime.toNanoseconds()); |
| 589 | 623 | } |
| 590 | 624 | } |
| 591 | | return @fromBackingInt(gop.index); |
| 625 | return @fromBackingInt(@intCast(gop.index)); |
| 592 | 626 | } |
| 593 | 627 | |
| 594 | 628 | pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputPathOptions) Allocator.Error!void { |
| ... | ... | @@ -759,8 +793,8 @@ pub const Manifest = struct { |
| 759 | 793 | if (m.files.count() <= m.input_paths.items.len) return; |
| 760 | 794 | const off = m.files.keys()[m.input_paths.items.len]; |
| 761 | 795 | m.contents.shrinkRetainingCapacity(@backingInt(off)); |
| 762 | | assert(m.contents.items.len % @alignOf(File) == 0); |
| 763 | | m.files.shrinkRetainingCapacity(m.input_paths.items.len); |
| 796 | assert(mem.isAligned(m.contents.items.len, @alignOf(File))); |
| 797 | m.files.shrinkRetainingCapacityContext(m.input_paths.items.len, .{ .contents = m.contents.items }); |
| 764 | 798 | } |
| 765 | 799 | |
| 766 | 800 | /// Assumes that `self.hash.hasher` has been updated only with the original digest and that |
| ... | ... | @@ -806,11 +840,13 @@ pub const Manifest = struct { |
| 806 | 840 | |
| 807 | 841 | // Guess number of files based on manifest contents len to reduce allocations. |
| 808 | 842 | // This is not an upper bound; subsequent insertions may potentially allocate. |
| 809 | | try m.files.ensureUnusedCapacity(gpa, contents.len / (@sizeOf(File) + 32)); |
| 843 | try m.files.ensureUnusedCapacityContext(gpa, contents.len / (@sizeOf(File) + 32), .{ |
| 844 | .contents = contents, |
| 845 | }); |
| 810 | 846 | |
| 811 | 847 | // This group we would like to cancel as soon as a cache miss is discovered. |
| 812 | 848 | const PostResult = union(enum) { |
| 813 | | checkFile: Check.Status, |
| 849 | checkFile: CheckFileError!Check.Status, |
| 814 | 850 | }; |
| 815 | 851 | var post_select_buffer: [10]PostResult = undefined; |
| 816 | 852 | var post_select: Io.Select(PostResult) = .init(io, &post_select_buffer); |
| ... | ... | @@ -819,12 +855,12 @@ pub const Manifest = struct { |
| 819 | 855 | |
| 820 | 856 | while (off + 1 < contents.len) { |
| 821 | 857 | const file_off: File.Offset = @fromBackingInt(off); |
| 822 | | const file = try file_off.getFallible(m); |
| 858 | const file = try file_off.getFallible(contents); |
| 823 | 859 | if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat; |
| 824 | 860 | const path = try filePathFallible(contents, file_off); |
| 825 | 861 | if (path.len == 0) return error.InvalidFormat; |
| 826 | 862 | |
| 827 | | try m.files.put(gpa, file_off, {}); |
| 863 | try m.files.putContext(gpa, file_off, {}, .{ .contents = contents }); |
| 828 | 864 | |
| 829 | 865 | post_select.async(.checkFile, checkFile, .{ m, &c, file_off, path }); |
| 830 | 866 | post_select_remaining += 1; |
| ... | ... | @@ -867,19 +903,19 @@ pub const Manifest = struct { |
| 867 | 903 | try input_group.await(io); |
| 868 | 904 | return .miss; |
| 869 | 905 | }, |
| 870 | | .fail => |diagnostic| { |
| 871 | | m.diagnostic = diagnostic; |
| 872 | | return error.CacheCheckFailed; |
| 873 | | }, |
| 874 | 906 | }, |
| 875 | 907 | }; |
| 876 | 908 | } |
| 877 | 909 | |
| 878 | 910 | try input_group.await(io); |
| 879 | 911 | if (c.status == .miss) return .miss; |
| 912 | if (m.diagnostic != .none) return error.CacheCheckFailed; |
| 913 | |
| 914 | // Needed due to the length mutation above. |
| 915 | const refreshed_contents = m.contents.items; |
| 880 | 916 | |
| 881 | 917 | for (m.files.keys()) |file_off| { |
| 882 | | m.hash.hasher.update(&file_off.get(m).digest); |
| 918 | m.hash.hasher.update(&file_off.get(refreshed_contents).digest); |
| 883 | 919 | } |
| 884 | 920 | |
| 885 | 921 | return .hit; |
| ... | ... | @@ -896,20 +932,25 @@ pub const Manifest = struct { |
| 896 | 932 | if (input_path.have_stat) @panic("TODO"); |
| 897 | 933 | if (input_path.contents != .not_requested) @panic("TODO"); |
| 898 | 934 | if (input_path.request_handle) @panic("TODO"); |
| 899 | | switch (try checkFile(m, c, file_off, file_path)) { |
| 935 | if (checkFile(m, c, file_off, file_path)) |status| switch (status) { |
| 900 | 936 | .hit => return, |
| 901 | 937 | .miss => @atomicStore(Check.Status, &c.status, .miss, .unordered), |
| 938 | } else |err| switch (err) { |
| 939 | error.CacheCheckFailed => assert(m.diagnostic != .none), |
| 940 | else => |e| return e, |
| 902 | 941 | } |
| 903 | 942 | } |
| 904 | 943 | |
| 944 | const CheckFileError = error{ Canceled, CacheCheckFailed }; |
| 945 | |
| 905 | 946 | /// Runs concurrently with other `checkFile`. |
| 906 | 947 | fn checkFile( |
| 907 | 948 | m: *Manifest, |
| 908 | 949 | c: *Check, |
| 909 | 950 | file_off: File.Offset, |
| 910 | 951 | file_path: [:0]const u8, |
| 911 | | ) error{ Canceled, CacheCheckFailed }!Check.Status { |
| 912 | | const file = file_off.get(m); |
| 952 | ) CheckFileError!Check.Status { |
| 953 | const file = file_off.get(m.contents.items); |
| 913 | 954 | const cache = m.cache; |
| 914 | 955 | const gpa = cache.gpa; |
| 915 | 956 | const io = cache.io; |
| ... | ... | @@ -928,7 +969,7 @@ pub const Manifest = struct { |
| 928 | 969 | const actual_is_directory = actual_stat.kind == .directory; |
| 929 | 970 | if (actual_is_directory != file.flags.is_directory) return .miss; |
| 930 | 971 | |
| 931 | | if (try file.setStatChanged(m, actual_stat)) return .miss; |
| 972 | if (try file.setStatChanged(m, .init(actual_stat))) return .miss; |
| 932 | 973 | |
| 933 | 974 | return .hit; |
| 934 | 975 | } |
| ... | ... | @@ -954,7 +995,7 @@ pub const Manifest = struct { |
| 954 | 995 | .err = e, |
| 955 | 996 | } }), |
| 956 | 997 | }; |
| 957 | | if (try file.setStatChanged(m, actual_stat)) { |
| 998 | if (try file.setStatChanged(m, .init(actual_stat))) { |
| 958 | 999 | const prev_digest: BinDigest = file.digest; |
| 959 | 1000 | var contents: std.ArrayList(u8) = .empty; |
| 960 | 1001 | defer contents.deinit(gpa); |
| ... | ... | @@ -989,7 +1030,7 @@ pub const Manifest = struct { |
| 989 | 1030 | } }), |
| 990 | 1031 | }; |
| 991 | 1032 | |
| 992 | | if (try file.setStatChanged(m, actual_stat)) { |
| 1033 | if (try file.setStatChanged(m, .init(actual_stat))) { |
| 993 | 1034 | const prev_digest: BinDigest = file.digest; |
| 994 | 1035 | hashFile(io, opened_file, &file.digest) catch |err| switch (err) { |
| 995 | 1036 | error.Canceled => |e| return e, |
| ... | ... | @@ -1015,8 +1056,9 @@ pub const Manifest = struct { |
| 1015 | 1056 | man.hash.hasher = hasher_init; |
| 1016 | 1057 | man.hash.hasher.update(bin_digest); |
| 1017 | 1058 | man.shrinkFilesToInput(); |
| 1059 | const contents = man.contents.items; |
| 1018 | 1060 | for (man.files.keys()) |off| { |
| 1019 | | const file = off.get(man); |
| 1061 | const file = off.get(contents); |
| 1020 | 1062 | man.hash.hasher.update(&file.digest); |
| 1021 | 1063 | } |
| 1022 | 1064 | } |
| ... | ... | @@ -1065,6 +1107,10 @@ pub const Manifest = struct { |
| 1065 | 1107 | } |
| 1066 | 1108 | |
| 1067 | 1109 | pub const AddPathPostOptions = struct { |
| 1110 | path: union(enum) { |
| 1111 | unresolved: Path, |
| 1112 | prefixed: PrefixedPath, |
| 1113 | }, |
| 1068 | 1114 | handle: PathHandle = .{ .file = null }, |
| 1069 | 1115 | stat: ?Stat = null, |
| 1070 | 1116 | /// If it is a directory, there is a special encoding required for contents, which |
| ... | ... | @@ -1073,29 +1119,30 @@ pub const Manifest = struct { |
| 1073 | 1119 | metadata_only: bool = false, |
| 1074 | 1120 | }; |
| 1075 | 1121 | |
| 1076 | | pub const AddPathPostError = Io.Cancelable || Allocator.Error; |
| 1077 | | |
| 1078 | 1122 | /// Add a file as a dependency of process being cached, after cache miss |
| 1079 | 1123 | /// occurs. |
| 1080 | 1124 | /// |
| 1081 | 1125 | /// See also: |
| 1082 | 1126 | /// * `addInputPath` |
| 1083 | | pub fn addPathPost(m: *Manifest, path: Path, options: AddPathPostOptions) AddPathPostError!void { |
| 1127 | pub fn addPathPost(m: *Manifest, options: AddPathPostOptions) !void { |
| 1084 | 1128 | assert(m.manifest_file != null); |
| 1085 | 1129 | const cache = m.cache; |
| 1086 | 1130 | const gpa = cache.gpa; |
| 1087 | 1131 | const io = cache.io; |
| 1088 | 1132 | const is_directory = options.handle == .dir; |
| 1089 | 1133 | |
| 1090 | | try m.files.ensureUnusedCapacity(gpa, 1); |
| 1134 | try m.files.ensureUnusedCapacityContext(gpa, 1, .{ .contents = m.contents.items }); |
| 1091 | 1135 | |
| 1092 | | const prev_contents_len = m.contents.items.len; |
| 1093 | | const new_header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File))); |
| 1094 | | errdefer m.contents.shrinkRetainingCapacity(prev_contents_len); |
| 1136 | const new_file_offset: File.Offset = @fromBackingInt(@intCast(m.contents.items.len)); |
| 1137 | const new_header: *File = @ptrCast(@alignCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)))); |
| 1138 | errdefer m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); |
| 1095 | 1139 | |
| 1096 | 1140 | new_header.* = .{ |
| 1097 | 1141 | .flags = .{ |
| 1098 | | .prefix = try cache.findAppendPrefixedPath(&m.contents, path), |
| 1142 | .prefix = switch (options.path) { |
| 1143 | .unresolved => |unresolved| try cache.resolveAppendPath(&m.contents, unresolved), |
| 1144 | .prefixed => |prefixed| try cache.appendPrefixedPath(&m.contents, prefixed), |
| 1145 | }, |
| 1099 | 1146 | .is_directory = is_directory, |
| 1100 | 1147 | .metadata_only = options.metadata_only, |
| 1101 | 1148 | }, |
| ... | ... | @@ -1103,31 +1150,32 @@ pub const Manifest = struct { |
| 1103 | 1150 | .inode = undefined, |
| 1104 | 1151 | .mtime = undefined, |
| 1105 | 1152 | .digest = @splat(0), |
| 1153 | .path_start = .{}, |
| 1106 | 1154 | }; |
| 1107 | | assert(m.contents.items.len % @alignOf(File) == 0); |
| 1155 | assert(mem.isAligned(m.contents.items.len, @alignOf(File))); |
| 1108 | 1156 | |
| 1109 | | const gop = m.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{ |
| 1110 | | .manifest = m, |
| 1157 | const gop = m.files.getOrPutAssumeCapacityContext(new_file_offset, .{ |
| 1158 | .contents = m.contents.items, |
| 1111 | 1159 | }); |
| 1112 | 1160 | m.files.lockPointers(); |
| 1113 | 1161 | defer m.files.unlockPointers(); |
| 1114 | 1162 | |
| 1115 | | const header = if (gop.found_existing) h: { |
| 1116 | | m.contents.shrinkRetainingCapacity(prev_contents_len); |
| 1163 | const header, const file_offset = if (gop.found_existing) h: { |
| 1164 | m.contents.shrinkRetainingCapacity(@backingInt(new_file_offset)); |
| 1117 | 1165 | const existing_off = gop.key_ptr.*; |
| 1118 | | const header = existing_off.get(m); |
| 1166 | const header = existing_off.get(m.contents.items); |
| 1119 | 1167 | // If it trips, the same file path has been added to the cache |
| 1120 | 1168 | // manifest both as a directory and as a normal file, making the |
| 1121 | 1169 | // intended caching behavior ambiguous. |
| 1122 | 1170 | assert(header.flags.is_directory == is_directory); |
| 1123 | 1171 | if (!options.metadata_only) |
| 1124 | 1172 | header.flags.metadata_only = false; |
| 1125 | | break :h header; |
| 1126 | | } else new_header; |
| 1173 | break :h .{ header, existing_off }; |
| 1174 | } else .{ new_header, new_file_offset }; |
| 1127 | 1175 | |
| 1128 | 1176 | if (options.stat) |stat| { |
| 1129 | 1177 | try header.setStat(m, stat); |
| 1130 | | if (header.metadata_only) { |
| 1178 | if (header.flags.metadata_only) { |
| 1131 | 1179 | return; |
| 1132 | 1180 | } else if (options.contents) |contents| { |
| 1133 | 1181 | var hasher = hasher_init; |
| ... | ... | @@ -1138,27 +1186,31 @@ pub const Manifest = struct { |
| 1138 | 1186 | } |
| 1139 | 1187 | |
| 1140 | 1188 | const need_stat = options.stat == null; |
| 1189 | const metadata_only = header.flags.metadata_only; |
| 1190 | const prefix = header.flags.prefix; |
| 1141 | 1191 | |
| 1142 | 1192 | switch (options.handle) { |
| 1143 | 1193 | .dir => |opt_handle| if (opt_handle) |handle| { |
| 1144 | | try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); |
| 1194 | try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only); |
| 1145 | 1195 | } else { |
| 1146 | | const dir = cache.prefixes()[header.flags.prefix].handle; |
| 1147 | | const handle = try dir.openDir(io, header.path(), .{ |
| 1196 | const dir = cache.prefixes()[prefix].handle; |
| 1197 | const sub_path = filePath(m.contents.items, file_offset); |
| 1198 | const handle = try dir.openDir(io, sub_path, .{ |
| 1148 | 1199 | .access_sub_paths = false, |
| 1149 | 1200 | .iterate = true, |
| 1150 | 1201 | }); |
| 1151 | 1202 | defer handle.close(io); |
| 1152 | | try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only); |
| 1203 | try populateDirectory(m, header, need_stat, handle, options.contents, metadata_only); |
| 1153 | 1204 | }, |
| 1154 | 1205 | |
| 1155 | 1206 | .file => |opt_handle| if (opt_handle) |handle| { |
| 1156 | | try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only); |
| 1207 | try populateFile(m, header, need_stat, handle, options.contents, metadata_only); |
| 1157 | 1208 | } else { |
| 1158 | | const dir = cache.prefixes()[header.flags.prefix].handle; |
| 1159 | | const handle = try dir.openFile(io, header.path(), .{ .mode = .read_only }); |
| 1209 | const dir = cache.prefixes()[prefix].handle; |
| 1210 | const sub_path = filePath(m.contents.items, file_offset); |
| 1211 | const handle = try dir.openFile(io, sub_path, .{ .mode = .read_only }); |
| 1160 | 1212 | defer handle.close(io); |
| 1161 | | try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only); |
| 1213 | try populateFile(m, header, need_stat, handle, options.contents, metadata_only); |
| 1162 | 1214 | }, |
| 1163 | 1215 | } |
| 1164 | 1216 | } |
| ... | ... | @@ -1175,7 +1227,7 @@ pub const Manifest = struct { |
| 1175 | 1227 | |
| 1176 | 1228 | if (need_stat) { |
| 1177 | 1229 | const stat = try handle.stat(io); |
| 1178 | | try file.setStat(m, stat); |
| 1230 | try file.setStat(m, .init(stat)); |
| 1179 | 1231 | } |
| 1180 | 1232 | if (metadata_only) return; |
| 1181 | 1233 | if (contents) |bytes| { |
| ... | ... | @@ -1201,7 +1253,7 @@ pub const Manifest = struct { |
| 1201 | 1253 | |
| 1202 | 1254 | if (need_stat) { |
| 1203 | 1255 | const stat = try handle.stat(io); |
| 1204 | | try file.setStat(m, stat); |
| 1256 | try file.setStat(m, .init(stat)); |
| 1205 | 1257 | } |
| 1206 | 1258 | if (metadata_only) return; |
| 1207 | 1259 | if (contents) |bytes| { |
| ... | ... | @@ -1238,28 +1290,26 @@ pub const Manifest = struct { |
| 1238 | 1290 | defer resolve_buf.deinit(gpa); |
| 1239 | 1291 | |
| 1240 | 1292 | var it: DepTokenizer = .{ .bytes = dep_file_contents }; |
| 1241 | | while (it.next()) |token| { |
| 1242 | | switch (token) { |
| 1243 | | // We don't care about targets, we only want the prereqs |
| 1244 | | // Clang is invoked in single-source mode but other programs may not |
| 1245 | | .target, .target_must_resolve => {}, |
| 1246 | | .prereq => |file_path| if (self.manifest_file == null) { |
| 1247 | | _ = try self.addInputPath(.initCwd(file_path), .{}); |
| 1248 | | } else try self.addPathPost(file_path), |
| 1249 | | .prereq_must_resolve => { |
| 1250 | | resolve_buf.clearRetainingCapacity(); |
| 1251 | | try token.resolve(gpa, &resolve_buf); |
| 1252 | | if (self.manifest_file == null) { |
| 1253 | | _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); |
| 1254 | | } else try self.addPathPost(resolve_buf.items); |
| 1255 | | }, |
| 1256 | | else => |err| { |
| 1257 | | try err.printError(gpa, &error_buf); |
| 1258 | | log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items }); |
| 1259 | | return error.InvalidDepFile; |
| 1260 | | }, |
| 1261 | | } |
| 1262 | | } |
| 1293 | while (it.next()) |token| switch (token) { |
| 1294 | // We don't care about targets, we only want the prereqs |
| 1295 | // Clang is invoked in single-source mode but other programs may not |
| 1296 | .target, .target_must_resolve => {}, |
| 1297 | .prereq => |file_path| if (self.manifest_file == null) { |
| 1298 | _ = try self.addInputPath(.initCwd(file_path), .{}); |
| 1299 | } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(file_path) } }), |
| 1300 | .prereq_must_resolve => { |
| 1301 | resolve_buf.clearRetainingCapacity(); |
| 1302 | try token.resolve(gpa, &resolve_buf); |
| 1303 | if (self.manifest_file == null) { |
| 1304 | _ = try self.addInputPath(.initCwd(resolve_buf.items), .{}); |
| 1305 | } else try self.addPathPost(.{ .path = .{ .unresolved = .initCwd(resolve_buf.items) } }); |
| 1306 | }, |
| 1307 | else => |err| { |
| 1308 | try err.printError(gpa, &error_buf); |
| 1309 | log.err("failed parsing {s}: {s}", .{ dep_file_sub_path, error_buf.items }); |
| 1310 | return error.InvalidDepFile; |
| 1311 | }, |
| 1312 | }; |
| 1263 | 1313 | } |
| 1264 | 1314 | |
| 1265 | 1315 | /// Returns a binary hash of the inputs. |
| ... | ... | @@ -1368,6 +1418,10 @@ pub const Manifest = struct { |
| 1368 | 1418 | pub fn takeFiles(m: *Manifest) SelfContainedFiles { |
| 1369 | 1419 | defer m.files = .empty; |
| 1370 | 1420 | defer m.contents = .empty; |
| 1421 | return borrowFiles(m); |
| 1422 | } |
| 1423 | |
| 1424 | pub fn borrowFiles(m: *const Manifest) SelfContainedFiles { |
| 1371 | 1425 | return .{ |
| 1372 | 1426 | .files = m.files, |
| 1373 | 1427 | .contents = m.contents, |
| ... | ... | @@ -1488,7 +1542,7 @@ pub const Manifest = struct { |
| 1488 | 1542 | while (true) { |
| 1489 | 1543 | const entries = entry_buffer[0..try reader.read(io, &entry_buffer)]; |
| 1490 | 1544 | for (try entries_list.addManyAsSlice(gpa, entries.len), entries) |*off, entry| { |
| 1491 | | off.* = contents.items.len; |
| 1545 | off.* = @intCast(contents.items.len); |
| 1492 | 1546 | // As an optimization, make the reservation also count the duplication |
| 1493 | 1547 | // of the contents buffer that will be required after sorting. |
| 1494 | 1548 | try contents.ensureUnusedCapacity(gpa, (contents.items.len + entry.name.len + 2 - contents_start) * 2); |
Andrew Kelley <andrew@ziglang.org> 2026-09-03 18:34:48-07:00