authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-03-22 01:13:43-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-03-22 01:13:43-07:00
loga2651cbc829d44df4c3773037598b30e8cf0c4da
tree555c74b10683ae9678c68777310116f47142a8aa
parent54c08579e4859673391843182aa2fd44aabbf6cf
parent950359071bca707dbc9763f1bf3ebc79cd52ebca
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #19388 from ziglang/cache-dedup

cache system file deduplication

11 files changed, 450 insertions(+), 372 deletions(-)

lib/std/Build/Cache.zig+151-140
...@@ -2,77 +2,6 @@...@@ -2,77 +2,6 @@
2//! This is not a general-purpose cache. It is designed to be fast and simple,2//! This is not a general-purpose cache. It is designed to be fast and simple,
3//! not to withstand attacks using specially-crafted input.3//! not to withstand attacks using specially-crafted input.
44
5pub const Directory = struct {
6 /// This field is redundant for operations that can act on the open directory handle
7 /// directly, but it is needed when passing the directory to a child process.
8 /// `null` means cwd.
9 path: ?[]const u8,
10 handle: fs.Dir,
11
12 pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory {
13 return .{
14 .path = if (d.path) |p| try arena.dupe(u8, p) else null,
15 .handle = d.handle,
16 };
17 }
18
19 pub fn cwd() Directory {
20 return .{
21 .path = null,
22 .handle = fs.cwd(),
23 };
24 }
25
26 pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
27 if (self.path) |p| {
28 // TODO clean way to do this with only 1 allocation
29 const part2 = try fs.path.join(allocator, paths);
30 defer allocator.free(part2);
31 return fs.path.join(allocator, &[_][]const u8{ p, part2 });
32 } else {
33 return fs.path.join(allocator, paths);
34 }
35 }
36
37 pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
38 if (self.path) |p| {
39 // TODO clean way to do this with only 1 allocation
40 const part2 = try fs.path.join(allocator, paths);
41 defer allocator.free(part2);
42 return fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
43 } else {
44 return fs.path.joinZ(allocator, paths);
45 }
46 }
47
48 /// Whether or not the handle should be closed, or the path should be freed
49 /// is determined by usage, however this function is provided for convenience
50 /// if it happens to be what the caller needs.
51 pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
52 self.handle.close();
53 if (self.path) |p| gpa.free(p);
54 self.* = undefined;
55 }
56
57 pub fn format(
58 self: Directory,
59 comptime fmt_string: []const u8,
60 options: fmt.FormatOptions,
61 writer: anytype,
62 ) !void {
63 _ = options;
64 if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self);
65 if (self.path) |p| {
66 try writer.writeAll(p);
67 try writer.writeAll(fs.path.sep_str);
68 }
69 }
70
71 pub fn eql(self: Directory, other: Directory) bool {
72 return self.handle.fd == other.handle.fd;
73 }
74};
75
76gpa: Allocator,5gpa: Allocator,
77manifest_dir: fs.Dir,6manifest_dir: fs.Dir,
78hash: HashHelper = .{},7hash: HashHelper = .{},
...@@ -88,6 +17,8 @@ mutex: std.Thread.Mutex = .{},...@@ -88,6 +17,8 @@ mutex: std.Thread.Mutex = .{},
88prefixes_buffer: [4]Directory = undefined,17prefixes_buffer: [4]Directory = undefined,
89prefixes_len: usize = 0,18prefixes_len: usize = 0,
9019
20pub const Path = @import("Cache/Path.zig");
21pub const Directory = @import("Cache/Directory.zig");
91pub const DepTokenizer = @import("Cache/DepTokenizer.zig");22pub const DepTokenizer = @import("Cache/DepTokenizer.zig");
9223
93const Cache = @This();24const Cache = @This();
...@@ -124,7 +55,15 @@ pub fn prefixes(cache: *const Cache) []const Directory {...@@ -124,7 +55,15 @@ pub fn prefixes(cache: *const Cache) []const Directory {
12455
125const PrefixedPath = struct {56const PrefixedPath = struct {
126 prefix: u8,57 prefix: u8,
127 sub_path: []u8,58 sub_path: []const u8,
59
60 fn eql(a: PrefixedPath, b: PrefixedPath) bool {
61 return a.prefix == b.prefix and std.mem.eql(u8, a.sub_path, b.sub_path);
62 }
63
64 fn hash(pp: PrefixedPath) u32 {
65 return @truncate(std.hash.Wyhash.hash(pp.prefix, pp.sub_path));
66 }
128};67};
12968
130fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {69fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
...@@ -183,7 +122,7 @@ pub const HexDigest = [hex_digest_len]u8;...@@ -183,7 +122,7 @@ pub const HexDigest = [hex_digest_len]u8;
183122
184/// This is currently just an arbitrary non-empty string that can't match another manifest line.123/// This is currently just an arbitrary non-empty string that can't match another manifest line.
185const manifest_header = "0";124const manifest_header = "0";
186const manifest_file_size_max = 50 * 1024 * 1024;125const manifest_file_size_max = 100 * 1024 * 1024;
187126
188/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it127/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
189/// provides enough collision resistance for the Manifest use cases, while being one of our128/// provides enough collision resistance for the Manifest use cases, while being one of our
...@@ -201,7 +140,7 @@ pub const hasher_init: Hasher = Hasher.init(&[_]u8{...@@ -201,7 +140,7 @@ pub const hasher_init: Hasher = Hasher.init(&[_]u8{
201});140});
202141
203pub const File = struct {142pub const File = struct {
204 prefixed_path: ?PrefixedPath,143 prefixed_path: PrefixedPath,
205 max_file_size: ?usize,144 max_file_size: ?usize,
206 stat: Stat,145 stat: Stat,
207 bin_digest: BinDigest,146 bin_digest: BinDigest,
...@@ -214,16 +153,18 @@ pub const File = struct {...@@ -214,16 +153,18 @@ pub const File = struct {
214 };153 };
215154
216 pub fn deinit(self: *File, gpa: Allocator) void {155 pub fn deinit(self: *File, gpa: Allocator) void {
217 if (self.prefixed_path) |pp| {156 gpa.free(self.prefixed_path.sub_path);
218 gpa.free(pp.sub_path);
219 self.prefixed_path = null;
220 }
221 if (self.contents) |contents| {157 if (self.contents) |contents| {
222 gpa.free(contents);158 gpa.free(contents);
223 self.contents = null;159 self.contents = null;
224 }160 }
225 self.* = undefined;161 self.* = undefined;
226 }162 }
163
164 pub fn updateMaxSize(file: *File, new_max_size: ?usize) void {
165 const new = new_max_size orelse return;
166 file.max_file_size = if (file.max_file_size) |old| @max(old, new) else new;
167 }
227};168};
228169
229pub const HashHelper = struct {170pub const HashHelper = struct {
...@@ -365,7 +306,7 @@ pub const Manifest = struct {...@@ -365,7 +306,7 @@ pub const Manifest = struct {
365 // order to obtain a problematic timestamp for the next call. Calls after that306 // order to obtain a problematic timestamp for the next call. Calls after that
366 // will then use the same timestamp, to avoid unnecessary filesystem writes.307 // will then use the same timestamp, to avoid unnecessary filesystem writes.
367 want_refresh_timestamp: bool = true,308 want_refresh_timestamp: bool = true,
368 files: std.ArrayListUnmanaged(File) = .{},309 files: Files = .{},
369 hex_digest: HexDigest,310 hex_digest: HexDigest,
370 /// Populated when hit() returns an error because of one311 /// Populated when hit() returns an error because of one
371 /// of the files listed in the manifest.312 /// of the files listed in the manifest.
...@@ -374,6 +315,34 @@ pub const Manifest = struct {...@@ -374,6 +315,34 @@ pub const Manifest = struct {
374 /// what time the file system thinks it is, according to its own granularity.315 /// what time the file system thinks it is, according to its own granularity.
375 recent_problematic_timestamp: i128 = 0,316 recent_problematic_timestamp: i128 = 0,
376317
318 pub const Files = std.ArrayHashMapUnmanaged(File, void, FilesContext, false);
319
320 pub const FilesContext = struct {
321 pub fn hash(fc: FilesContext, file: File) u32 {
322 _ = fc;
323 return file.prefixed_path.hash();
324 }
325
326 pub fn eql(fc: FilesContext, a: File, b: File, b_index: usize) bool {
327 _ = fc;
328 _ = b_index;
329 return a.prefixed_path.eql(b.prefixed_path);
330 }
331 };
332
333 const FilesAdapter = struct {
334 pub fn eql(context: @This(), a: PrefixedPath, b: File, b_index: usize) bool {
335 _ = context;
336 _ = b_index;
337 return a.eql(b.prefixed_path);
338 }
339
340 pub fn hash(context: @This(), key: PrefixedPath) u32 {
341 _ = context;
342 return key.hash();
343 }
344 };
345
377 /// Add a file as a dependency of process being cached. When `hit` is346 /// Add a file as a dependency of process being cached. When `hit` is
378 /// called, the file's contents will be checked to ensure that it matches347 /// called, the file's contents will be checked to ensure that it matches
379 /// the contents from previous times.348 /// the contents from previous times.
...@@ -386,7 +355,7 @@ pub const Manifest = struct {...@@ -386,7 +355,7 @@ pub const Manifest = struct {
386 /// to access the contents of the file after calling `hit()` like so:355 /// to access the contents of the file after calling `hit()` like so:
387 ///356 ///
388 /// ```357 /// ```
389 /// var file_contents = cache_hash.files.items[file_index].contents.?;358 /// var file_contents = cache_hash.files.keys()[file_index].contents.?;
390 /// ```359 /// ```
391 pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {360 pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
392 assert(self.manifest_file == null);361 assert(self.manifest_file == null);
...@@ -396,7 +365,12 @@ pub const Manifest = struct {...@@ -396,7 +365,12 @@ pub const Manifest = struct {
396 const prefixed_path = try self.cache.findPrefix(file_path);365 const prefixed_path = try self.cache.findPrefix(file_path);
397 errdefer gpa.free(prefixed_path.sub_path);366 errdefer gpa.free(prefixed_path.sub_path);
398367
399 self.files.addOneAssumeCapacity().* = .{368 const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{});
369 if (gop.found_existing) {
370 gop.key_ptr.updateMaxSize(max_file_size);
371 return gop.index;
372 }
373 gop.key_ptr.* = .{
400 .prefixed_path = prefixed_path,374 .prefixed_path = prefixed_path,
401 .contents = null,375 .contents = null,
402 .max_file_size = max_file_size,376 .max_file_size = max_file_size,
...@@ -407,7 +381,7 @@ pub const Manifest = struct {...@@ -407,7 +381,7 @@ pub const Manifest = struct {
407 self.hash.add(prefixed_path.prefix);381 self.hash.add(prefixed_path.prefix);
408 self.hash.addBytes(prefixed_path.sub_path);382 self.hash.addBytes(prefixed_path.sub_path);
409383
410 return self.files.items.len - 1;384 return gop.index;
411 }385 }
412386
413 pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {387 pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
...@@ -487,7 +461,7 @@ pub const Manifest = struct {...@@ -487,7 +461,7 @@ pub const Manifest = struct {
487461
488 self.want_refresh_timestamp = true;462 self.want_refresh_timestamp = true;
489463
490 const input_file_count = self.files.items.len;464 const input_file_count = self.files.entries.len;
491 while (true) : (self.unhit(bin_digest, input_file_count)) {465 while (true) : (self.unhit(bin_digest, input_file_count)) {
492 const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);466 const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
493 defer gpa.free(file_contents);467 defer gpa.free(file_contents);
...@@ -499,7 +473,7 @@ pub const Manifest = struct {...@@ -499,7 +473,7 @@ pub const Manifest = struct {
499 if (try self.upgradeToExclusiveLock()) continue;473 if (try self.upgradeToExclusiveLock()) continue;
500 self.manifest_dirty = true;474 self.manifest_dirty = true;
501 while (idx < input_file_count) : (idx += 1) {475 while (idx < input_file_count) : (idx += 1) {
502 const ch_file = &self.files.items[idx];476 const ch_file = &self.files.keys()[idx];
503 self.populateFileHash(ch_file) catch |err| {477 self.populateFileHash(ch_file) catch |err| {
504 self.failed_file_index = idx;478 self.failed_file_index = idx;
505 return err;479 return err;
...@@ -510,18 +484,6 @@ pub const Manifest = struct {...@@ -510,18 +484,6 @@ pub const Manifest = struct {
510 while (line_iter.next()) |line| {484 while (line_iter.next()) |line| {
511 defer idx += 1;485 defer idx += 1;
512486
513 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
514 const new = try self.files.addOne(gpa);
515 new.* = .{
516 .prefixed_path = null,
517 .contents = null,
518 .max_file_size = null,
519 .stat = undefined,
520 .bin_digest = undefined,
521 };
522 break :blk new;
523 };
524
525 var iter = mem.tokenizeScalar(u8, line, ' ');487 var iter = mem.tokenizeScalar(u8, line, ' ');
526 const size = iter.next() orelse return error.InvalidFormat;488 const size = iter.next() orelse return error.InvalidFormat;
527 const inode = iter.next() orelse return error.InvalidFormat;489 const inode = iter.next() orelse return error.InvalidFormat;
...@@ -530,30 +492,61 @@ pub const Manifest = struct {...@@ -530,30 +492,61 @@ pub const Manifest = struct {
530 const prefix_str = iter.next() orelse return error.InvalidFormat;492 const prefix_str = iter.next() orelse return error.InvalidFormat;
531 const file_path = iter.rest();493 const file_path = iter.rest();
532494
533 cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;495 const stat_size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
534 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;496 const stat_inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
535 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;497 const stat_mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
536 _ = fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;498 const file_bin_digest = b: {
499 if (digest_str.len != hex_digest_len) return error.InvalidFormat;
500 var bd: BinDigest = undefined;
501 _ = fmt.hexToBytes(&bd, digest_str) catch return error.InvalidFormat;
502 break :b bd;
503 };
504
537 const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;505 const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
538 if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;506 if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
539507
540 if (file_path.len == 0) {508 if (file_path.len == 0) return error.InvalidFormat;
541 return error.InvalidFormat;
542 }
543 if (cache_hash_file.prefixed_path) |pp| {
544 if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
545 return error.InvalidFormat;
546 }
547 }
548509
549 if (cache_hash_file.prefixed_path == null) {510 const cache_hash_file = f: {
550 cache_hash_file.prefixed_path = .{511 const prefixed_path: PrefixedPath = .{
551 .prefix = prefix,512 .prefix = prefix,
552 .sub_path = try gpa.dupe(u8, file_path),513 .sub_path = file_path, // expires with file_contents
553 };514 };
554 }515 if (idx < input_file_count) {
516 const file = &self.files.keys()[idx];
517 if (!file.prefixed_path.eql(prefixed_path))
518 return error.InvalidFormat;
519
520 file.stat = .{
521 .size = stat_size,
522 .inode = stat_inode,
523 .mtime = stat_mtime,
524 };
525 file.bin_digest = file_bin_digest;
526 break :f file;
527 }
528 const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
529 errdefer assert(self.files.popOrNull() != null);
530 if (!gop.found_existing) {
531 gop.key_ptr.* = .{
532 .prefixed_path = .{
533 .prefix = prefix,
534 .sub_path = try gpa.dupe(u8, file_path),
535 },
536 .contents = null,
537 .max_file_size = null,
538 .stat = .{
539 .size = stat_size,
540 .inode = stat_inode,
541 .mtime = stat_mtime,
542 },
543 .bin_digest = file_bin_digest,
544 };
545 }
546 break :f gop.key_ptr;
547 };
555548
556 const pp = cache_hash_file.prefixed_path.?;549 const pp = cache_hash_file.prefixed_path;
557 const dir = self.cache.prefixes()[pp.prefix].handle;550 const dir = self.cache.prefixes()[pp.prefix].handle;
558 const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) {551 const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
559 error.FileNotFound => {552 error.FileNotFound => {
...@@ -617,7 +610,7 @@ pub const Manifest = struct {...@@ -617,7 +610,7 @@ pub const Manifest = struct {
617 if (try self.upgradeToExclusiveLock()) continue;610 if (try self.upgradeToExclusiveLock()) continue;
618 self.manifest_dirty = true;611 self.manifest_dirty = true;
619 while (idx < input_file_count) : (idx += 1) {612 while (idx < input_file_count) : (idx += 1) {
620 const ch_file = &self.files.items[idx];613 const ch_file = &self.files.keys()[idx];
621 self.populateFileHash(ch_file) catch |err| {614 self.populateFileHash(ch_file) catch |err| {
622 self.failed_file_index = idx;615 self.failed_file_index = idx;
623 return err;616 return err;
...@@ -640,12 +633,12 @@ pub const Manifest = struct {...@@ -640,12 +633,12 @@ pub const Manifest = struct {
640 self.hash.hasher.update(&bin_digest);633 self.hash.hasher.update(&bin_digest);
641634
642 // Remove files not in the initial hash.635 // Remove files not in the initial hash.
643 for (self.files.items[input_file_count..]) |*file| {636 for (self.files.keys()[input_file_count..]) |*file| {
644 file.deinit(self.cache.gpa);637 file.deinit(self.cache.gpa);
645 }638 }
646 self.files.shrinkRetainingCapacity(input_file_count);639 self.files.shrinkRetainingCapacity(input_file_count);
647640
648 for (self.files.items) |file| {641 for (self.files.keys()) |file| {
649 self.hash.hasher.update(&file.bin_digest);642 self.hash.hasher.update(&file.bin_digest);
650 }643 }
651 }644 }
...@@ -685,7 +678,7 @@ pub const Manifest = struct {...@@ -685,7 +678,7 @@ pub const Manifest = struct {
685 }678 }
686679
687 fn populateFileHash(self: *Manifest, ch_file: *File) !void {680 fn populateFileHash(self: *Manifest, ch_file: *File) !void {
688 const pp = ch_file.prefixed_path.?;681 const pp = ch_file.prefixed_path;
689 const dir = self.cache.prefixes()[pp.prefix].handle;682 const dir = self.cache.prefixes()[pp.prefix].handle;
690 const file = try dir.openFile(pp.sub_path, .{});683 const file = try dir.openFile(pp.sub_path, .{});
691 defer file.close();684 defer file.close();
...@@ -751,7 +744,7 @@ pub const Manifest = struct {...@@ -751,7 +744,7 @@ pub const Manifest = struct {
751 .bin_digest = undefined,744 .bin_digest = undefined,
752 .contents = null,745 .contents = null,
753 };746 };
754 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);747 errdefer self.files.shrinkRetainingCapacity(self.files.entries.len - 1);
755748
756 try self.populateFileHash(new_ch_file);749 try self.populateFileHash(new_ch_file);
757750
...@@ -759,9 +752,11 @@ pub const Manifest = struct {...@@ -759,9 +752,11 @@ pub const Manifest = struct {
759 }752 }
760753
761 /// Add a file as a dependency of process being cached, after the initial hash has been754 /// Add a file as a dependency of process being cached, after the initial hash has been
762 /// calculated. This is useful for processes that don't know the all the files that755 /// calculated.
763 /// are depended on ahead of time. For example, a source file that can import other files756 ///
764 /// will need to be recompiled if the imported file is changed.757 /// This is useful for processes that don't know the all the files that are
758 /// depended on ahead of time. For example, a source file that can import
759 /// other files will need to be recompiled if the imported file is changed.
765 pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {760 pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
766 assert(self.manifest_file != null);761 assert(self.manifest_file != null);
767762
...@@ -769,17 +764,26 @@ pub const Manifest = struct {...@@ -769,17 +764,26 @@ pub const Manifest = struct {
769 const prefixed_path = try self.cache.findPrefix(file_path);764 const prefixed_path = try self.cache.findPrefix(file_path);
770 errdefer gpa.free(prefixed_path.sub_path);765 errdefer gpa.free(prefixed_path.sub_path);
771766
772 const new_ch_file = try self.files.addOne(gpa);767 const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
773 new_ch_file.* = .{768 errdefer assert(self.files.popOrNull() != null);
769
770 if (gop.found_existing) {
771 gpa.free(prefixed_path.sub_path);
772 return;
773 }
774
775 gop.key_ptr.* = .{
774 .prefixed_path = prefixed_path,776 .prefixed_path = prefixed_path,
775 .max_file_size = null,777 .max_file_size = null,
776 .stat = undefined,778 .stat = undefined,
777 .bin_digest = undefined,779 .bin_digest = undefined,
778 .contents = null,780 .contents = null,
779 };781 };
780 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
781782
782 try self.populateFileHash(new_ch_file);783 self.files.lockPointers();
784 defer self.files.unlockPointers();
785
786 try self.populateFileHash(gop.key_ptr);
783 }787 }
784788
785 /// Like `addFilePost` but when the file contents have already been loaded from disk.789 /// Like `addFilePost` but when the file contents have already been loaded from disk.
...@@ -793,13 +797,20 @@ pub const Manifest = struct {...@@ -793,13 +797,20 @@ pub const Manifest = struct {
793 assert(self.manifest_file != null);797 assert(self.manifest_file != null);
794 const gpa = self.cache.gpa;798 const gpa = self.cache.gpa;
795799
796 const ch_file = try self.files.addOne(gpa);
797 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
798
799 const prefixed_path = try self.cache.findPrefixResolved(resolved_path);800 const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
800 errdefer gpa.free(prefixed_path.sub_path);801 errdefer gpa.free(prefixed_path.sub_path);
801802
802 ch_file.* = .{803 const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
804 errdefer assert(self.files.popOrNull() != null);
805
806 if (gop.found_existing) {
807 gpa.free(prefixed_path.sub_path);
808 return;
809 }
810
811 const new_file = gop.key_ptr;
812
813 new_file.* = .{
803 .prefixed_path = prefixed_path,814 .prefixed_path = prefixed_path,
804 .max_file_size = null,815 .max_file_size = null,
805 .stat = stat,816 .stat = stat,
...@@ -807,19 +818,19 @@ pub const Manifest = struct {...@@ -807,19 +818,19 @@ pub const Manifest = struct {
807 .contents = null,818 .contents = null,
808 };819 };
809820
810 if (self.isProblematicTimestamp(ch_file.stat.mtime)) {821 if (self.isProblematicTimestamp(new_file.stat.mtime)) {
811 // The actual file has an unreliable timestamp, force it to be hashed822 // The actual file has an unreliable timestamp, force it to be hashed
812 ch_file.stat.mtime = 0;823 new_file.stat.mtime = 0;
813 ch_file.stat.inode = 0;824 new_file.stat.inode = 0;
814 }825 }
815826
816 {827 {
817 var hasher = hasher_init;828 var hasher = hasher_init;
818 hasher.update(bytes);829 hasher.update(bytes);
819 hasher.final(&ch_file.bin_digest);830 hasher.final(&new_file.bin_digest);
820 }831 }
821832
822 self.hash.hasher.update(&ch_file.bin_digest);833 self.hash.hasher.update(&new_file.bin_digest);
823 }834 }
824835
825 pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {836 pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
...@@ -885,14 +896,14 @@ pub const Manifest = struct {...@@ -885,14 +896,14 @@ pub const Manifest = struct {
885896
886 const writer = contents.writer();897 const writer = contents.writer();
887 try writer.writeAll(manifest_header ++ "\n");898 try writer.writeAll(manifest_header ++ "\n");
888 for (self.files.items) |file| {899 for (self.files.keys()) |file| {
889 try writer.print("{d} {d} {d} {} {d} {s}\n", .{900 try writer.print("{d} {d} {d} {} {d} {s}\n", .{
890 file.stat.size,901 file.stat.size,
891 file.stat.inode,902 file.stat.inode,
892 file.stat.mtime,903 file.stat.mtime,
893 fmt.fmtSliceHexLower(&file.bin_digest),904 fmt.fmtSliceHexLower(&file.bin_digest),
894 file.prefixed_path.?.prefix,905 file.prefixed_path.prefix,
895 file.prefixed_path.?.sub_path,906 file.prefixed_path.sub_path,
896 });907 });
897 }908 }
898909
...@@ -961,7 +972,7 @@ pub const Manifest = struct {...@@ -961,7 +972,7 @@ pub const Manifest = struct {
961972
962 file.close();973 file.close();
963 }974 }
964 for (self.files.items) |*file| {975 for (self.files.keys()) |*file| {
965 file.deinit(self.cache.gpa);976 file.deinit(self.cache.gpa);
966 }977 }
967 self.files.deinit(self.cache.gpa);978 self.files.deinit(self.cache.gpa);
...@@ -1130,7 +1141,7 @@ test "check that changing a file makes cache fail" {...@@ -1130,7 +1141,7 @@ test "check that changing a file makes cache fail" {
1130 // There should be nothing in the cache1141 // There should be nothing in the cache
1131 try testing.expectEqual(false, try ch.hit());1142 try testing.expectEqual(false, try ch.hit());
11321143
1133 try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));1144 try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.keys()[temp_file_idx].contents.?));
11341145
1135 digest1 = ch.final();1146 digest1 = ch.final();
11361147
...@@ -1150,7 +1161,7 @@ test "check that changing a file makes cache fail" {...@@ -1150,7 +1161,7 @@ test "check that changing a file makes cache fail" {
1150 try testing.expectEqual(false, try ch.hit());1161 try testing.expectEqual(false, try ch.hit());
11511162
1152 // The cache system does not keep the contents of re-hashed input files.1163 // The cache system does not keep the contents of re-hashed input files.
1153 try testing.expect(ch.files.items[temp_file_idx].contents == null);1164 try testing.expect(ch.files.keys()[temp_file_idx].contents == null);
11541165
1155 digest2 = ch.final();1166 digest2 = ch.final();
11561167
lib/std/Build/Cache/Directory.zig created+74
...@@ -0,0 +1,74 @@
1const Directory = @This();
2const std = @import("../../std.zig");
3const fs = std.fs;
4const fmt = std.fmt;
5const Allocator = std.mem.Allocator;
6
7/// This field is redundant for operations that can act on the open directory handle
8/// directly, but it is needed when passing the directory to a child process.
9/// `null` means cwd.
10path: ?[]const u8,
11handle: fs.Dir,
12
13pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory {
14 return .{
15 .path = if (d.path) |p| try arena.dupe(u8, p) else null,
16 .handle = d.handle,
17 };
18}
19
20pub fn cwd() Directory {
21 return .{
22 .path = null,
23 .handle = fs.cwd(),
24 };
25}
26
27pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
28 if (self.path) |p| {
29 // TODO clean way to do this with only 1 allocation
30 const part2 = try fs.path.join(allocator, paths);
31 defer allocator.free(part2);
32 return fs.path.join(allocator, &[_][]const u8{ p, part2 });
33 } else {
34 return fs.path.join(allocator, paths);
35 }
36}
37
38pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
39 if (self.path) |p| {
40 // TODO clean way to do this with only 1 allocation
41 const part2 = try fs.path.join(allocator, paths);
42 defer allocator.free(part2);
43 return fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
44 } else {
45 return fs.path.joinZ(allocator, paths);
46 }
47}
48
49/// Whether or not the handle should be closed, or the path should be freed
50/// is determined by usage, however this function is provided for convenience
51/// if it happens to be what the caller needs.
52pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
53 self.handle.close();
54 if (self.path) |p| gpa.free(p);
55 self.* = undefined;
56}
57
58pub fn format(
59 self: Directory,
60 comptime fmt_string: []const u8,
61 options: fmt.FormatOptions,
62 writer: anytype,
63) !void {
64 _ = options;
65 if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self);
66 if (self.path) |p| {
67 try writer.writeAll(p);
68 try writer.writeAll(fs.path.sep_str);
69 }
70}
71
72pub fn eql(self: Directory, other: Directory) bool {
73 return self.handle.fd == other.handle.fd;
74}
lib/std/Build/Cache/Path.zig created+154
...@@ -0,0 +1,154 @@
1root_dir: Cache.Directory,
2/// The path, relative to the root dir, that this `Path` represents.
3/// Empty string means the root_dir is the path.
4sub_path: []const u8 = "",
5
6pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path {
7 return .{
8 .root_dir = try p.root_dir.clone(arena),
9 .sub_path = try arena.dupe(u8, p.sub_path),
10 };
11}
12
13pub fn cwd() Path {
14 return .{ .root_dir = Cache.Directory.cwd() };
15}
16
17pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
18 if (sub_path.len == 0) return p;
19 const parts: []const []const u8 =
20 if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
21 return .{
22 .root_dir = p.root_dir,
23 .sub_path = try fs.path.join(arena, parts),
24 };
25}
26
27pub fn resolvePosix(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
28 if (sub_path.len == 0) return p;
29 return .{
30 .root_dir = p.root_dir,
31 .sub_path = try fs.path.resolvePosix(arena, &.{ p.sub_path, sub_path }),
32 };
33}
34
35pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
36 const parts: []const []const u8 =
37 if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
38 return p.root_dir.join(allocator, parts);
39}
40
41pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![:0]u8 {
42 const parts: []const []const u8 =
43 if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
44 return p.root_dir.joinZ(allocator, parts);
45}
46
47pub fn openFile(
48 p: Path,
49 sub_path: []const u8,
50 flags: fs.File.OpenFlags,
51) !fs.File {
52 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
53 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
54 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
55 p.sub_path, sub_path,
56 }) catch return error.NameTooLong;
57 };
58 return p.root_dir.handle.openFile(joined_path, flags);
59}
60
61pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
62 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
63 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
64 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
65 p.sub_path, sub_path,
66 }) catch return error.NameTooLong;
67 };
68 return p.root_dir.handle.makeOpenPath(joined_path, opts);
69}
70
71pub fn statFile(p: Path, sub_path: []const u8) !fs.Dir.Stat {
72 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
73 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
74 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
75 p.sub_path, sub_path,
76 }) catch return error.NameTooLong;
77 };
78 return p.root_dir.handle.statFile(joined_path);
79}
80
81pub fn atomicFile(
82 p: Path,
83 sub_path: []const u8,
84 options: fs.Dir.AtomicFileOptions,
85 buf: *[fs.MAX_PATH_BYTES]u8,
86) !fs.AtomicFile {
87 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
88 break :p std.fmt.bufPrint(buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
89 p.sub_path, sub_path,
90 }) catch return error.NameTooLong;
91 };
92 return p.root_dir.handle.atomicFile(joined_path, options);
93}
94
95pub fn access(p: Path, sub_path: []const u8, flags: fs.File.OpenFlags) !void {
96 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
97 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
98 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
99 p.sub_path, sub_path,
100 }) catch return error.NameTooLong;
101 };
102 return p.root_dir.handle.access(joined_path, flags);
103}
104
105pub fn makePath(p: Path, sub_path: []const u8) !void {
106 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
107 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
108 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
109 p.sub_path, sub_path,
110 }) catch return error.NameTooLong;
111 };
112 return p.root_dir.handle.makePath(joined_path);
113}
114
115pub fn format(
116 self: Path,
117 comptime fmt_string: []const u8,
118 options: std.fmt.FormatOptions,
119 writer: anytype,
120) !void {
121 if (fmt_string.len == 1) {
122 // Quote-escape the string.
123 const stringEscape = std.zig.stringEscape;
124 const f = switch (fmt_string[0]) {
125 'q' => "",
126 '\'' => '\'',
127 else => @compileError("unsupported format string: " ++ fmt_string),
128 };
129 if (self.root_dir.path) |p| {
130 try stringEscape(p, f, options, writer);
131 if (self.sub_path.len > 0) try stringEscape(fs.path.sep_str, f, options, writer);
132 }
133 if (self.sub_path.len > 0) {
134 try stringEscape(self.sub_path, f, options, writer);
135 }
136 return;
137 }
138 if (fmt_string.len > 0)
139 std.fmt.invalidFmtError(fmt_string, self);
140 if (self.root_dir.path) |p| {
141 try writer.writeAll(p);
142 try writer.writeAll(fs.path.sep_str);
143 }
144 if (self.sub_path.len > 0) {
145 try writer.writeAll(self.sub_path);
146 try writer.writeAll(fs.path.sep_str);
147 }
148}
149
150const Path = @This();
151const std = @import("../../std.zig");
152const fs = std.fs;
153const Allocator = std.mem.Allocator;
154const Cache = std.Build.Cache;
lib/std/Build/Step.zig+1-1
...@@ -544,7 +544,7 @@ pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool {...@@ -544,7 +544,7 @@ pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool {
544544
545fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror {545fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror {
546 const i = man.failed_file_index orelse return err;546 const i = man.failed_file_index orelse return err;
547 const pp = man.files.items[i].prefixed_path orelse return err;547 const pp = man.files.keys()[i].prefixed_path;
548 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";548 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
549 return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });549 return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });
550}550}
lib/std/array_hash_map.zig+56-58
...@@ -9,23 +9,26 @@ const Wyhash = std.hash.Wyhash;...@@ -9,23 +9,26 @@ const Wyhash = std.hash.Wyhash;
9const Allocator = mem.Allocator;9const Allocator = mem.Allocator;
10const hash_map = @This();10const hash_map = @This();
1111
12/// An ArrayHashMap with default hash and equal functions.12/// An `ArrayHashMap` with default hash and equal functions.
13/// See AutoContext for a description of the hash and equal implementations.13///
14/// See `AutoContext` for a description of the hash and equal implementations.
14pub fn AutoArrayHashMap(comptime K: type, comptime V: type) type {15pub fn AutoArrayHashMap(comptime K: type, comptime V: type) type {
15 return ArrayHashMap(K, V, AutoContext(K), !autoEqlIsCheap(K));16 return ArrayHashMap(K, V, AutoContext(K), !autoEqlIsCheap(K));
16}17}
1718
18/// An ArrayHashMapUnmanaged with default hash and equal functions.19/// An `ArrayHashMapUnmanaged` with default hash and equal functions.
19/// See AutoContext for a description of the hash and equal implementations.20///
21/// See `AutoContext` for a description of the hash and equal implementations.
20pub fn AutoArrayHashMapUnmanaged(comptime K: type, comptime V: type) type {22pub fn AutoArrayHashMapUnmanaged(comptime K: type, comptime V: type) type {
21 return ArrayHashMapUnmanaged(K, V, AutoContext(K), !autoEqlIsCheap(K));23 return ArrayHashMapUnmanaged(K, V, AutoContext(K), !autoEqlIsCheap(K));
22}24}
2325
24/// Builtin hashmap for strings as keys.26/// An `ArrayHashMap` with strings as keys.
25pub fn StringArrayHashMap(comptime V: type) type {27pub fn StringArrayHashMap(comptime V: type) type {
26 return ArrayHashMap([]const u8, V, StringContext, true);28 return ArrayHashMap([]const u8, V, StringContext, true);
27}29}
2830
31/// An `ArrayHashMapUnmanaged` with strings as keys.
29pub fn StringArrayHashMapUnmanaged(comptime V: type) type {32pub fn StringArrayHashMapUnmanaged(comptime V: type) type {
30 return ArrayHashMapUnmanaged([]const u8, V, StringContext, true);33 return ArrayHashMapUnmanaged([]const u8, V, StringContext, true);
31}34}
...@@ -50,29 +53,33 @@ pub fn hashString(s: []const u8) u32 {...@@ -50,29 +53,33 @@ pub fn hashString(s: []const u8) u32 {
50 return @as(u32, @truncate(std.hash.Wyhash.hash(0, s)));53 return @as(u32, @truncate(std.hash.Wyhash.hash(0, s)));
51}54}
5255
53/// Insertion order is preserved.56/// A hash table of keys and values, each stored sequentially.
54/// Deletions perform a "swap removal" on the entries list.57///
58/// Insertion order is preserved. In general, this data structure supports the same
59/// operations as `std.ArrayList`.
60///
61/// Deletion operations:
62/// * `swapRemove` - O(1)
63/// * `orderedRemove` - O(N)
64///
55/// Modifying the hash map while iterating is allowed, however, one must understand65/// Modifying the hash map while iterating is allowed, however, one must understand
56/// the (well defined) behavior when mixing insertions and deletions with iteration.66/// the (well defined) behavior when mixing insertions and deletions with iteration.
57/// For a hash map that can be initialized directly that does not store an Allocator67///
58/// field, see `ArrayHashMapUnmanaged`.68/// See `ArrayHashMapUnmanaged` for a variant of this data structure that accepts an
59/// When `store_hash` is `false`, this data structure is biased towards cheap `eql`69/// `Allocator` as a parameter when needed rather than storing it.
60/// functions. It does not store each item's hash in the table. Setting `store_hash`
61/// to `true` incurs slightly more memory cost by storing each key's hash in the table
62/// but only has to call `eql` for hash collisions.
63/// If typical operations (except iteration over entries) need to be faster, prefer
64/// the alternative `std.HashMap`.
65/// Context must be a struct type with two member functions:
66/// hash(self, K) u32
67/// eql(self, K, K, usize) bool
68/// Adapted variants of many functions are provided. These variants
69/// take a pseudo key instead of a key. Their context must have the functions:
70/// hash(self, PseudoKey) u32
71/// eql(self, PseudoKey, K, usize) bool
72pub fn ArrayHashMap(70pub fn ArrayHashMap(
73 comptime K: type,71 comptime K: type,
74 comptime V: type,72 comptime V: type,
73 /// A namespace that provides these two functions:
74 /// * `pub fn hash(self, K) u32`
75 /// * `pub fn eql(self, K, K) bool`
76 ///
75 comptime Context: type,77 comptime Context: type,
78 /// When `false`, this data structure is biased towards cheap `eql`
79 /// functions and avoids storing each key's hash in the table. Setting
80 /// `store_hash` to `true` incurs more memory cost but limits `eql` to
81 /// being called only once per insertion/deletion (provided there are no
82 /// hash collisions).
76 comptime store_hash: bool,83 comptime store_hash: bool,
77) type {84) type {
78 return struct {85 return struct {
...@@ -472,34 +479,40 @@ pub fn ArrayHashMap(...@@ -472,34 +479,40 @@ pub fn ArrayHashMap(
472 };479 };
473}480}
474481
475/// General purpose hash table.482/// A hash table of keys and values, each stored sequentially.
476/// Insertion order is preserved.483///
477/// Deletions perform a "swap removal" on the entries list.484/// Insertion order is preserved. In general, this data structure supports the same
485/// operations as `std.ArrayListUnmanaged`.
486///
487/// Deletion operations:
488/// * `swapRemove` - O(1)
489/// * `orderedRemove` - O(N)
490///
478/// Modifying the hash map while iterating is allowed, however, one must understand491/// Modifying the hash map while iterating is allowed, however, one must understand
479/// the (well defined) behavior when mixing insertions and deletions with iteration.492/// the (well defined) behavior when mixing insertions and deletions with iteration.
480/// This type does not store an Allocator field - the Allocator must be passed in493///
494/// This type does not store an `Allocator` field - the `Allocator` must be passed in
481/// with each function call that requires it. See `ArrayHashMap` for a type that stores495/// with each function call that requires it. See `ArrayHashMap` for a type that stores
482/// an Allocator field for convenience.496/// an `Allocator` field for convenience.
497///
483/// Can be initialized directly using the default field values.498/// Can be initialized directly using the default field values.
499///
484/// This type is designed to have low overhead for small numbers of entries. When500/// This type is designed to have low overhead for small numbers of entries. When
485/// `store_hash` is `false` and the number of entries in the map is less than 9,501/// `store_hash` is `false` and the number of entries in the map is less than 9,
486/// the overhead cost of using `ArrayHashMapUnmanaged` rather than `std.ArrayList` is502/// the overhead cost of using `ArrayHashMapUnmanaged` rather than `std.ArrayList` is
487/// only a single pointer-sized integer.503/// only a single pointer-sized integer.
488/// When `store_hash` is `false`, this data structure is biased towards cheap `eql`
489/// functions. It does not store each item's hash in the table. Setting `store_hash`
490/// to `true` incurs slightly more memory cost by storing each key's hash in the table
491/// but guarantees only one call to `eql` per insertion/deletion.
492/// Context must be a struct type with two member functions:
493/// hash(self, K) u32
494/// eql(self, K, K) bool
495/// Adapted variants of many functions are provided. These variants
496/// take a pseudo key instead of a key. Their context must have the functions:
497/// hash(self, PseudoKey) u32
498/// eql(self, PseudoKey, K) bool
499pub fn ArrayHashMapUnmanaged(504pub fn ArrayHashMapUnmanaged(
500 comptime K: type,505 comptime K: type,
501 comptime V: type,506 comptime V: type,
507 /// A namespace that provides these two functions:
508 /// * `pub fn hash(self, K) u32`
509 /// * `pub fn eql(self, K, K) bool`
502 comptime Context: type,510 comptime Context: type,
511 /// When `false`, this data structure is biased towards cheap `eql`
512 /// functions and avoids storing each key's hash in the table. Setting
513 /// `store_hash` to `true` incurs more memory cost but limits `eql` to
514 /// being called only once per insertion/deletion (provided there are no
515 /// hash collisions).
503 comptime store_hash: bool,516 comptime store_hash: bool,
504) type {517) type {
505 return struct {518 return struct {
...@@ -516,10 +529,6 @@ pub fn ArrayHashMapUnmanaged(...@@ -516,10 +529,6 @@ pub fn ArrayHashMapUnmanaged(
516 /// Used to detect memory safety violations.529 /// Used to detect memory safety violations.
517 pointer_stability: std.debug.SafetyLock = .{},530 pointer_stability: std.debug.SafetyLock = .{},
518531
519 comptime {
520 std.hash_map.verifyContext(Context, K, K, u32, true);
521 }
522
523 /// Modifying the key is allowed only if it does not change the hash.532 /// Modifying the key is allowed only if it does not change the hash.
524 /// Modifying the value is allowed.533 /// Modifying the value is allowed.
525 /// Entry pointers become invalid whenever this ArrayHashMap is modified,534 /// Entry pointers become invalid whenever this ArrayHashMap is modified,
...@@ -1834,27 +1843,16 @@ pub fn ArrayHashMapUnmanaged(...@@ -1834,27 +1843,16 @@ pub fn ArrayHashMapUnmanaged(
1834 }1843 }
1835 }1844 }
18361845
1837 inline fn checkedHash(ctx: anytype, key: anytype) u32 {1846 fn checkedHash(ctx: anytype, key: anytype) u32 {
1838 comptime std.hash_map.verifyContext(@TypeOf(ctx), @TypeOf(key), K, u32, true);
1839 // If you get a compile error on the next line, it means that your1847 // If you get a compile error on the next line, it means that your
1840 // generic hash function doesn't accept your key.1848 // generic hash function doesn't accept your key.
1841 const hash = ctx.hash(key);1849 return ctx.hash(key);
1842 if (@TypeOf(hash) != u32) {
1843 @compileError("Context " ++ @typeName(@TypeOf(ctx)) ++ " has a generic hash function that returns the wrong type!\n" ++
1844 @typeName(u32) ++ " was expected, but found " ++ @typeName(@TypeOf(hash)));
1845 }
1846 return hash;
1847 }1850 }
1848 inline fn checkedEql(ctx: anytype, a: anytype, b: K, b_index: usize) bool {1851
1849 comptime std.hash_map.verifyContext(@TypeOf(ctx), @TypeOf(a), K, u32, true);1852 fn checkedEql(ctx: anytype, a: anytype, b: K, b_index: usize) bool {
1850 // If you get a compile error on the next line, it means that your1853 // If you get a compile error on the next line, it means that your
1851 // generic eql function doesn't accept (self, adapt key, K, index).1854 // generic eql function doesn't accept (self, adapt key, K, index).
1852 const eql = ctx.eql(a, b, b_index);1855 return ctx.eql(a, b, b_index);
1853 if (@TypeOf(eql) != bool) {
1854 @compileError("Context " ++ @typeName(@TypeOf(ctx)) ++ " has a generic eql function that returns the wrong type!\n" ++
1855 @typeName(bool) ++ " was expected, but found " ++ @typeName(@TypeOf(eql)));
1856 }
1857 return eql;
1858 }1856 }
18591857
1860 fn dumpState(self: Self, comptime keyFmt: []const u8, comptime valueFmt: []const u8) void {1858 fn dumpState(self: Self, comptime keyFmt: []const u8, comptime valueFmt: []const u8) void {
src/Compilation.zig+2-2
...@@ -1999,7 +1999,7 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void...@@ -1999,7 +1999,7 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
19991999
2000 const is_hit = man.hit() catch |err| {2000 const is_hit = man.hit() catch |err| {
2001 const i = man.failed_file_index orelse return err;2001 const i = man.failed_file_index orelse return err;
2002 const pp = man.files.items[i].prefixed_path orelse return err;2002 const pp = man.files.keys()[i].prefixed_path;
2003 const prefix = man.cache.prefixes()[pp.prefix];2003 const prefix = man.cache.prefixes()[pp.prefix];
2004 return comp.setMiscFailure(2004 return comp.setMiscFailure(
2005 .check_whole_cache,2005 .check_whole_cache,
...@@ -4147,7 +4147,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module...@@ -4147,7 +4147,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
4147 const prev_hash_state = man.hash.peekBin();4147 const prev_hash_state = man.hash.peekBin();
4148 const actual_hit = hit: {4148 const actual_hit = hit: {
4149 _ = try man.hit();4149 _ = try man.hit();
4150 if (man.files.items.len == 0) {4150 if (man.files.entries.len == 0) {
4151 man.unhit(prev_hash_state, 0);4151 man.unhit(prev_hash_state, 0);
4152 break :hit false;4152 break :hit false;
4153 }4153 }
src/Package.zig-159
...@@ -2,162 +2,3 @@ pub const Module = @import("Package/Module.zig");...@@ -2,162 +2,3 @@ pub const Module = @import("Package/Module.zig");
2pub const Fetch = @import("Package/Fetch.zig");2pub const Fetch = @import("Package/Fetch.zig");
3pub const build_zig_basename = "build.zig";3pub const build_zig_basename = "build.zig";
4pub const Manifest = @import("Package/Manifest.zig");4pub const Manifest = @import("Package/Manifest.zig");
5
6pub const Path = struct {
7 root_dir: Cache.Directory,
8 /// The path, relative to the root dir, that this `Path` represents.
9 /// Empty string means the root_dir is the path.
10 sub_path: []const u8 = "",
11
12 pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path {
13 return .{
14 .root_dir = try p.root_dir.clone(arena),
15 .sub_path = try arena.dupe(u8, p.sub_path),
16 };
17 }
18
19 pub fn cwd() Path {
20 return .{ .root_dir = Cache.Directory.cwd() };
21 }
22
23 pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
24 if (sub_path.len == 0) return p;
25 const parts: []const []const u8 =
26 if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
27 return .{
28 .root_dir = p.root_dir,
29 .sub_path = try fs.path.join(arena, parts),
30 };
31 }
32
33 pub fn resolvePosix(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
34 if (sub_path.len == 0) return p;
35 return .{
36 .root_dir = p.root_dir,
37 .sub_path = try fs.path.resolvePosix(arena, &.{ p.sub_path, sub_path }),
38 };
39 }
40
41 pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
42 const parts: []const []const u8 =
43 if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
44 return p.root_dir.join(allocator, parts);
45 }
46
47 pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![:0]u8 {
48 const parts: []const []const u8 =
49 if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
50 return p.root_dir.joinZ(allocator, parts);
51 }
52
53 pub fn openFile(
54 p: Path,
55 sub_path: []const u8,
56 flags: fs.File.OpenFlags,
57 ) !fs.File {
58 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
59 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
60 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
61 p.sub_path, sub_path,
62 }) catch return error.NameTooLong;
63 };
64 return p.root_dir.handle.openFile(joined_path, flags);
65 }
66
67 pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
68 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
69 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
70 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
71 p.sub_path, sub_path,
72 }) catch return error.NameTooLong;
73 };
74 return p.root_dir.handle.makeOpenPath(joined_path, opts);
75 }
76
77 pub fn statFile(p: Path, sub_path: []const u8) !fs.Dir.Stat {
78 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
79 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
80 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
81 p.sub_path, sub_path,
82 }) catch return error.NameTooLong;
83 };
84 return p.root_dir.handle.statFile(joined_path);
85 }
86
87 pub fn atomicFile(
88 p: Path,
89 sub_path: []const u8,
90 options: fs.Dir.AtomicFileOptions,
91 buf: *[fs.MAX_PATH_BYTES]u8,
92 ) !fs.AtomicFile {
93 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
94 break :p std.fmt.bufPrint(buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
95 p.sub_path, sub_path,
96 }) catch return error.NameTooLong;
97 };
98 return p.root_dir.handle.atomicFile(joined_path, options);
99 }
100
101 pub fn access(p: Path, sub_path: []const u8, flags: fs.File.OpenFlags) !void {
102 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
103 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
104 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
105 p.sub_path, sub_path,
106 }) catch return error.NameTooLong;
107 };
108 return p.root_dir.handle.access(joined_path, flags);
109 }
110
111 pub fn makePath(p: Path, sub_path: []const u8) !void {
112 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
113 const joined_path = if (p.sub_path.len == 0) sub_path else p: {
114 break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
115 p.sub_path, sub_path,
116 }) catch return error.NameTooLong;
117 };
118 return p.root_dir.handle.makePath(joined_path);
119 }
120
121 pub fn format(
122 self: Path,
123 comptime fmt_string: []const u8,
124 options: std.fmt.FormatOptions,
125 writer: anytype,
126 ) !void {
127 if (fmt_string.len == 1) {
128 // Quote-escape the string.
129 const stringEscape = std.zig.stringEscape;
130 const f = switch (fmt_string[0]) {
131 'q' => "",
132 '\'' => '\'',
133 else => @compileError("unsupported format string: " ++ fmt_string),
134 };
135 if (self.root_dir.path) |p| {
136 try stringEscape(p, f, options, writer);
137 if (self.sub_path.len > 0) try stringEscape(fs.path.sep_str, f, options, writer);
138 }
139 if (self.sub_path.len > 0) {
140 try stringEscape(self.sub_path, f, options, writer);
141 }
142 return;
143 }
144 if (fmt_string.len > 0)
145 std.fmt.invalidFmtError(fmt_string, self);
146 if (self.root_dir.path) |p| {
147 try writer.writeAll(p);
148 try writer.writeAll(fs.path.sep_str);
149 }
150 if (self.sub_path.len > 0) {
151 try writer.writeAll(self.sub_path);
152 try writer.writeAll(fs.path.sep_str);
153 }
154 }
155};
156
157const Package = @This();
158const builtin = @import("builtin");
159const std = @import("std");
160const fs = std.fs;
161const Allocator = std.mem.Allocator;
162const assert = std.debug.assert;
163const Cache = std.Build.Cache;
src/Package/Fetch.zig+6-6
...@@ -33,7 +33,7 @@ location_tok: std.zig.Ast.TokenIndex,...@@ -33,7 +33,7 @@ location_tok: std.zig.Ast.TokenIndex,
33hash_tok: std.zig.Ast.TokenIndex,33hash_tok: std.zig.Ast.TokenIndex,
34name_tok: std.zig.Ast.TokenIndex,34name_tok: std.zig.Ast.TokenIndex,
35lazy_status: LazyStatus,35lazy_status: LazyStatus,
36parent_package_root: Package.Path,36parent_package_root: Cache.Path,
37parent_manifest_ast: ?*const std.zig.Ast,37parent_manifest_ast: ?*const std.zig.Ast,
38prog_node: *std.Progress.Node,38prog_node: *std.Progress.Node,
39job_queue: *JobQueue,39job_queue: *JobQueue,
...@@ -50,7 +50,7 @@ allow_missing_paths_field: bool,...@@ -50,7 +50,7 @@ allow_missing_paths_field: bool,
5050
51/// This will either be relative to `global_cache`, or to the build root of51/// This will either be relative to `global_cache`, or to the build root of
52/// the root package.52/// the root package.
53package_root: Package.Path,53package_root: Cache.Path,
54error_bundle: ErrorBundle.Wip,54error_bundle: ErrorBundle.Wip,
55manifest: ?Manifest,55manifest: ?Manifest,
56manifest_ast: std.zig.Ast,56manifest_ast: std.zig.Ast,
...@@ -263,7 +263,7 @@ pub const JobQueue = struct {...@@ -263,7 +263,7 @@ pub const JobQueue = struct {
263pub const Location = union(enum) {263pub const Location = union(enum) {
264 remote: Remote,264 remote: Remote,
265 /// A directory found inside the parent package.265 /// A directory found inside the parent package.
266 relative_path: Package.Path,266 relative_path: Cache.Path,
267 /// Recursive Fetch tasks will never use this Location, but it may be267 /// Recursive Fetch tasks will never use this Location, but it may be
268 /// passed in by the CLI. Indicates the file contents here should be copied268 /// passed in by the CLI. Indicates the file contents here should be copied
269 /// into the global package cache. It may be a file relative to the cwd or269 /// into the global package cache. It may be a file relative to the cwd or
...@@ -564,7 +564,7 @@ fn checkBuildFileExistence(f: *Fetch) RunError!void {...@@ -564,7 +564,7 @@ fn checkBuildFileExistence(f: *Fetch) RunError!void {
564}564}
565565
566/// This function populates `f.manifest` or leaves it `null`.566/// This function populates `f.manifest` or leaves it `null`.
567fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void {567fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
568 const eb = &f.error_bundle;568 const eb = &f.error_bundle;
569 const arena = f.arena.allocator();569 const arena = f.arena.allocator();
570 const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(570 const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(
...@@ -722,7 +722,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -722,7 +722,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
722}722}
723723
724pub fn relativePathDigest(724pub fn relativePathDigest(
725 pkg_root: Package.Path,725 pkg_root: Cache.Path,
726 cache_root: Cache.Directory,726 cache_root: Cache.Directory,
727) Manifest.MultiHashHexDigest {727) Manifest.MultiHashHexDigest {
728 var hasher = Manifest.Hash.init(.{});728 var hasher = Manifest.Hash.init(.{});
...@@ -1658,7 +1658,7 @@ const Filter = struct {...@@ -1658,7 +1658,7 @@ const Filter = struct {
1658};1658};
16591659
1660pub fn depDigest(1660pub fn depDigest(
1661 pkg_root: Package.Path,1661 pkg_root: Cache.Path,
1662 cache_root: Cache.Directory,1662 cache_root: Cache.Directory,
1663 dep: Manifest.Dependency,1663 dep: Manifest.Dependency,
1664) ?Manifest.MultiHashHexDigest {1664) ?Manifest.MultiHashHexDigest {
src/Package/Module.zig+3-3
...@@ -3,7 +3,7 @@...@@ -3,7 +3,7 @@
3//! to Zcu. https://github.com/ziglang/zig/issues/143073//! to Zcu. https://github.com/ziglang/zig/issues/14307
44
5/// Only files inside this directory can be imported.5/// Only files inside this directory can be imported.
6root: Package.Path,6root: Cache.Path,
7/// Relative to `root`. May contain path separators.7/// Relative to `root`. May contain path separators.
8root_src_path: []const u8,8root_src_path: []const u8,
9/// Name used in compile errors. Looks like "root.foo.bar".9/// Name used in compile errors. Looks like "root.foo.bar".
...@@ -69,7 +69,7 @@ pub const CreateOptions = struct {...@@ -69,7 +69,7 @@ pub const CreateOptions = struct {
69 builtin_modules: ?*std.StringHashMapUnmanaged(*Module),69 builtin_modules: ?*std.StringHashMapUnmanaged(*Module),
7070
71 pub const Paths = struct {71 pub const Paths = struct {
72 root: Package.Path,72 root: Cache.Path,
73 /// Relative to `root`. May contain path separators.73 /// Relative to `root`. May contain path separators.
74 root_src_path: []const u8,74 root_src_path: []const u8,
75 };75 };
...@@ -463,7 +463,7 @@ pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module {...@@ -463,7 +463,7 @@ pub fn create(arena: Allocator, options: CreateOptions) !*Package.Module {
463463
464/// All fields correspond to `CreateOptions`.464/// All fields correspond to `CreateOptions`.
465pub const LimitedOptions = struct {465pub const LimitedOptions = struct {
466 root: Package.Path,466 root: Cache.Path,
467 root_src_path: []const u8,467 root_src_path: []const u8,
468 fully_qualified_name: []const u8,468 fully_qualified_name: []const u8,
469};469};
src/glibc.zig+1-1
...@@ -713,7 +713,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: *std.Progress.Node) !vo...@@ -713,7 +713,7 @@ pub fn buildSharedObjects(comp: *Compilation, prog_node: *std.Progress.Node) !vo
713 };713 };
714 defer o_directory.handle.close();714 defer o_directory.handle.close();
715715
716 const abilists_contents = man.files.items[abilists_index].contents.?;716 const abilists_contents = man.files.keys()[abilists_index].contents.?;
717 const metadata = try loadMetaData(comp.gpa, abilists_contents);717 const metadata = try loadMetaData(comp.gpa, abilists_contents);
718 defer metadata.destroy(comp.gpa);718 defer metadata.destroy(comp.gpa);
719719
src/main.zig+2-2
...@@ -6143,7 +6143,7 @@ fn cmdAstCheck(...@@ -6143,7 +6143,7 @@ fn cmdAstCheck(
6143 }6143 }
61446144
6145 file.mod = try Package.Module.createLimited(arena, .{6145 file.mod = try Package.Module.createLimited(arena, .{
6146 .root = Package.Path.cwd(),6146 .root = Cache.Path.cwd(),
6147 .root_src_path = file.sub_file_path,6147 .root_src_path = file.sub_file_path,
6148 .fully_qualified_name = "root",6148 .fully_qualified_name = "root",
6149 });6149 });
...@@ -6316,7 +6316,7 @@ fn cmdChangelist(...@@ -6316,7 +6316,7 @@ fn cmdChangelist(
6316 };6316 };
63176317
6318 file.mod = try Package.Module.createLimited(arena, .{6318 file.mod = try Package.Module.createLimited(arena, .{
6319 .root = Package.Path.cwd(),6319 .root = Cache.Path.cwd(),
6320 .root_src_path = file.sub_file_path,6320 .root_src_path = file.sub_file_path,
6321 .fully_qualified_name = "root",6321 .fully_qualified_name = "root",
6322 });6322 });