authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-29 17:29:59-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-07-03 15:28:29-07:00
log390a4ded986237381fffbe46719445b6cd9733f3
tree4d92daac7b55c38d9887828a9c9264ccede3f98a
parentb0be0c77d8fb11e1abed578f37a859a2084a934e

Merge pull request #9258 from ziglang/shared-cache-locking

Shared Cache Locking

7 files changed, 487 insertions(+), 157 deletions(-)

doc/langref.html.in+8
......@@ -10516,6 +10516,14 @@ fn readU32Be() u32 {}
1051610516 See the Zig Standard Library for more examples.
1051710517 </p>
1051810518 {#header_close#}
10519 {#header_open|Doc Comment Guidance#}
10520 <ul>
10521 <li>Omit any information that is redundant based on the name of the thing being documented.</li>
10522 <li>Duplicating information onto multiple similar functions is encouraged because it helps IDEs and other tools provide better help text.</li>
10523 <li>Use the word <strong>assume</strong> to indicate invariants that cause {#link|Undefined Behavior#} when violated.</li>
10524 <li>Use the word <strong>assert</strong> to indicate invariants that cause <em>safety-checked</em> {#link|Undefined Behavior#} when violated.</li>
10525 </ul>
10526 {#header_close#}
1051910527 {#header_close#}
1052010528 {#header_open|Source Encoding#}
1052110529 <p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>
lib/std/fs.zig+47-17
......@@ -883,24 +883,39 @@ pub const Dir = struct {
883883 /// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
884884 pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
885885 const w = os.windows;
886 return @as(File, .{
887 .handle = try os.windows.OpenFile(sub_path_w, .{
886 const file: File = .{
887 .handle = try w.OpenFile(sub_path_w, .{
888888 .dir = self.fd,
889889 .access_mask = w.SYNCHRONIZE |
890890 (if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
891891 (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
892 .share_access = switch (flags.lock) {
893 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
894 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
895 .Exclusive => w.FILE_SHARE_DELETE,
896 },
897 .share_access_nonblocking = flags.lock_nonblocking,
898892 .creation = w.FILE_OPEN,
899893 .io_mode = flags.intended_io_mode,
900894 }),
901895 .capable_io_mode = std.io.default_mode,
902896 .intended_io_mode = flags.intended_io_mode,
903 });
897 };
898 var io: w.IO_STATUS_BLOCK = undefined;
899 const range_off: w.LARGE_INTEGER = 0;
900 const range_len: w.LARGE_INTEGER = 1;
901 const exclusive = switch (flags.lock) {
902 .None => return file,
903 .Shared => false,
904 .Exclusive => true,
905 };
906 try w.LockFile(
907 file.handle,
908 null,
909 null,
910 null,
911 &io,
912 &range_off,
913 &range_len,
914 null,
915 @boolToInt(flags.lock_nonblocking),
916 @boolToInt(exclusive),
917 );
918 return file;
904919 }
905920
906921 /// Creates, opens, or overwrites a file with write access.
......@@ -1019,16 +1034,10 @@ pub const Dir = struct {
10191034 pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
10201035 const w = os.windows;
10211036 const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
1022 return @as(File, .{
1037 const file: File = .{
10231038 .handle = try os.windows.OpenFile(sub_path_w, .{
10241039 .dir = self.fd,
10251040 .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
1026 .share_access = switch (flags.lock) {
1027 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1028 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1029 .Exclusive => w.FILE_SHARE_DELETE,
1030 },
1031 .share_access_nonblocking = flags.lock_nonblocking,
10321041 .creation = if (flags.exclusive)
10331042 @as(u32, w.FILE_CREATE)
10341043 else if (flags.truncate)
......@@ -1039,7 +1048,28 @@ pub const Dir = struct {
10391048 }),
10401049 .capable_io_mode = std.io.default_mode,
10411050 .intended_io_mode = flags.intended_io_mode,
1042 });
1051 };
1052 var io: w.IO_STATUS_BLOCK = undefined;
1053 const range_off: w.LARGE_INTEGER = 0;
1054 const range_len: w.LARGE_INTEGER = 1;
1055 const exclusive = switch (flags.lock) {
1056 .None => return file,
1057 .Shared => false,
1058 .Exclusive => true,
1059 };
1060 try w.LockFile(
1061 file.handle,
1062 null,
1063 null,
1064 null,
1065 &io,
1066 &range_off,
1067 &range_len,
1068 null,
1069 @boolToInt(flags.lock_nonblocking),
1070 @boolToInt(exclusive),
1071 );
1072 return file;
10431073 }
10441074
10451075 pub const openRead = @compileError("deprecated in favor of openFile");
lib/std/fs/file.zig+197-14
......@@ -74,17 +74,28 @@ pub const File = struct {
7474 read: bool = true,
7575 write: bool = false,
7676
77 /// Open the file with a lock to prevent other processes from accessing it at the
78 /// same time. An exclusive lock will prevent other processes from acquiring a lock.
79 /// A shared lock will prevent other processes from acquiring a exclusive lock, but
80 /// doesn't prevent other process from getting their own shared locks.
77 /// Open the file with an advisory lock to coordinate with other processes
78 /// accessing it at the same time. An exclusive lock will prevent other
79 /// processes from acquiring a lock. A shared lock will prevent other
80 /// processes from acquiring a exclusive lock, but does not prevent
81 /// other process from getting their own shared locks.
8182 ///
82 /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
83 /// The lock is advisory, except on Linux in very specific cirsumstances[1].
8384 /// This means that a process that does not respect the locking API can still get access
8485 /// to the file, despite the lock.
8586 ///
86 /// Windows' file locks are mandatory, and any process attempting to access the file will
87 /// receive an error.
87 /// On these operating systems, the lock is acquired atomically with
88 /// opening the file:
89 /// * Darwin
90 /// * DragonFlyBSD
91 /// * FreeBSD
92 /// * Haiku
93 /// * NetBSD
94 /// * OpenBSD
95 /// On these operating systems, the lock is acquired via a separate syscall
96 /// after opening the file:
97 /// * Linux
98 /// * Windows
8899 ///
89100 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
90101 lock: Lock = .None,
......@@ -120,17 +131,28 @@ pub const File = struct {
120131 /// `error.PathAlreadyExists` to be returned.
121132 exclusive: bool = false,
122133
123 /// Open the file with a lock to prevent other processes from accessing it at the
124 /// same time. An exclusive lock will prevent other processes from acquiring a lock.
125 /// A shared lock will prevent other processes from acquiring a exclusive lock, but
126 /// doesn't prevent other process from getting their own shared locks.
134 /// Open the file with an advisory lock to coordinate with other processes
135 /// accessing it at the same time. An exclusive lock will prevent other
136 /// processes from acquiring a lock. A shared lock will prevent other
137 /// processes from acquiring a exclusive lock, but does not prevent
138 /// other process from getting their own shared locks.
127139 ///
128 /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].
140 /// The lock is advisory, except on Linux in very specific cirsumstances[1].
129141 /// This means that a process that does not respect the locking API can still get access
130142 /// to the file, despite the lock.
131143 ///
132 /// Windows's file locks are mandatory, and any process attempting to access the file will
133 /// receive an error.
144 /// On these operating systems, the lock is acquired atomically with
145 /// opening the file:
146 /// * Darwin
147 /// * DragonFlyBSD
148 /// * FreeBSD
149 /// * Haiku
150 /// * NetBSD
151 /// * OpenBSD
152 /// On these operating systems, the lock is acquired via a separate syscall
153 /// after opening the file:
154 /// * Linux
155 /// * Windows
134156 ///
135157 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
136158 lock: Lock = .None,
......@@ -829,4 +851,165 @@ pub const File = struct {
829851 pub fn seekableStream(file: File) SeekableStream {
830852 return .{ .context = file };
831853 }
854
855 const range_off: windows.LARGE_INTEGER = 0;
856 const range_len: windows.LARGE_INTEGER = 1;
857
858 pub const LockError = error{
859 SystemResources,
860 } || os.UnexpectedError;
861
862 /// Blocks when an incompatible lock is held by another process.
863 /// A process may hold only one type of lock (shared or exclusive) on
864 /// a file. When a process terminates in any way, the lock is released.
865 ///
866 /// Assumes the file is unlocked.
867 ///
868 /// TODO: integrate with async I/O
869 pub fn lock(file: File, l: Lock) LockError!void {
870 if (is_windows) {
871 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
872 const exclusive = switch (l) {
873 .None => return,
874 .Shared => false,
875 .Exclusive => true,
876 };
877 return windows.LockFile(
878 file.handle,
879 null,
880 null,
881 null,
882 &io_status_block,
883 &range_off,
884 &range_len,
885 null,
886 windows.FALSE, // non-blocking=false
887 @boolToInt(exclusive),
888 ) catch |err| switch (err) {
889 error.WouldBlock => unreachable, // non-blocking=false
890 else => |e| return e,
891 };
892 } else {
893 return os.flock(file.handle, switch (l) {
894 .None => os.LOCK_UN,
895 .Shared => os.LOCK_SH,
896 .Exclusive => os.LOCK_EX,
897 }) catch |err| switch (err) {
898 error.WouldBlock => unreachable, // non-blocking=false
899 else => |e| return e,
900 };
901 }
902 }
903
904 /// Assumes the file is locked.
905 pub fn unlock(file: File) void {
906 if (is_windows) {
907 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
908 return windows.UnlockFile(
909 file.handle,
910 &io_status_block,
911 &range_off,
912 &range_len,
913 null,
914 ) catch |err| switch (err) {
915 error.RangeNotLocked => unreachable, // Function assumes unlocked.
916 error.Unexpected => unreachable, // Resource deallocation must succeed.
917 };
918 } else {
919 return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
920 error.WouldBlock => unreachable, // unlocking can't block
921 error.SystemResources => unreachable, // We are deallocating resources.
922 error.Unexpected => unreachable, // Resource deallocation must succeed.
923 };
924 }
925 }
926
927 /// Attempts to obtain a lock, returning `true` if the lock is
928 /// obtained, and `false` if there was an existing incompatible lock held.
929 /// A process may hold only one type of lock (shared or exclusive) on
930 /// a file. When a process terminates in any way, the lock is released.
931 ///
932 /// Assumes the file is unlocked.
933 ///
934 /// TODO: integrate with async I/O
935 pub fn tryLock(file: File, l: Lock) LockError!bool {
936 if (is_windows) {
937 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
938 const exclusive = switch (l) {
939 .None => return,
940 .Shared => false,
941 .Exclusive => true,
942 };
943 windows.LockFile(
944 file.handle,
945 null,
946 null,
947 null,
948 &io_status_block,
949 &range_off,
950 &range_len,
951 null,
952 windows.TRUE, // non-blocking=true
953 @boolToInt(exclusive),
954 ) catch |err| switch (err) {
955 error.WouldBlock => return false,
956 else => |e| return e,
957 };
958 } else {
959 os.flock(file.handle, switch (l) {
960 .None => os.LOCK_UN,
961 .Shared => os.LOCK_SH | os.LOCK_NB,
962 .Exclusive => os.LOCK_EX | os.LOCK_NB,
963 }) catch |err| switch (err) {
964 error.WouldBlock => return false,
965 else => |e| return e,
966 };
967 }
968 return true;
969 }
970
971 /// Assumes the file is already locked in exclusive mode.
972 /// Atomically modifies the lock to be in shared mode, without releasing it.
973 ///
974 /// TODO: integrate with async I/O
975 pub fn downgradeLock(file: File) LockError!void {
976 if (is_windows) {
977 // On Windows it works like a semaphore + exclusivity flag. To implement this
978 // function, we first obtain another lock in shared mode. This changes the
979 // exclusivity flag, but increments the semaphore to 2. So we follow up with
980 // an NtUnlockFile which decrements the semaphore but does not modify the
981 // exclusivity flag.
982 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
983 windows.LockFile(
984 file.handle,
985 null,
986 null,
987 null,
988 &io_status_block,
989 &range_off,
990 &range_len,
991 null,
992 windows.TRUE, // non-blocking=true
993 windows.FALSE, // exclusive=false
994 ) catch |err| switch (err) {
995 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
996 else => |e| return e,
997 };
998 return windows.UnlockFile(
999 file.handle,
1000 &io_status_block,
1001 &range_off,
1002 &range_len,
1003 null,
1004 ) catch |err| switch (err) {
1005 error.RangeNotLocked => unreachable, // File was not locked.
1006 error.Unexpected => unreachable, // Resource deallocation must succeed.
1007 };
1008 } else {
1009 return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
1010 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
1011 else => |e| return e,
1012 };
1013 }
1014 }
8321015};
lib/std/os/windows.zig+91-50
......@@ -48,7 +48,6 @@ pub const OpenFileOptions = struct {
4848 dir: ?HANDLE = null,
4949 sa: ?*SECURITY_ATTRIBUTES = null,
5050 share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
51 share_access_nonblocking: bool = false,
5251 creation: ULONG,
5352 io_mode: std.io.ModeOverride,
5453 /// If true, tries to open path as a directory.
......@@ -59,8 +58,6 @@ pub const OpenFileOptions = struct {
5958 follow_symlinks: bool = true,
6059};
6160
62/// TODO when share_access_nonblocking is false, this implementation uses
63/// untinterruptible sleep() to block. This is not the final iteration of the API.
6461pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
6562 if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
6663 return error.IsDir;
......@@ -93,53 +90,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
9390 // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
9491 const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
9592
96 var delay: usize = 1;
97 while (true) {
98 const rc = ntdll.NtCreateFile(
99 &result,
100 options.access_mask,
101 &attr,
102 &io,
103 null,
104 FILE_ATTRIBUTE_NORMAL,
105 options.share_access,
106 options.creation,
107 flags,
108 null,
109 0,
110 );
111 switch (rc) {
112 .SUCCESS => {
113 if (std.io.is_async and options.io_mode == .evented) {
114 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
115 }
116 return result;
117 },
118 .OBJECT_NAME_INVALID => unreachable,
119 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
120 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
121 .NO_MEDIA_IN_DEVICE => return error.NoDevice,
122 .INVALID_PARAMETER => unreachable,
123 .SHARING_VIOLATION => {
124 if (options.share_access_nonblocking) {
125 return error.WouldBlock;
126 }
127 // TODO sleep in a way that is interruptable
128 // TODO integrate with async I/O
129 std.time.sleep(delay);
130 if (delay < 1 * std.time.ns_per_s) {
131 delay *= 2;
132 }
133 continue;
134 },
135 .ACCESS_DENIED => return error.AccessDenied,
136 .PIPE_BUSY => return error.PipeBusy,
137 .OBJECT_PATH_SYNTAX_BAD => unreachable,
138 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
139 .FILE_IS_A_DIRECTORY => return error.IsDir,
140 .NOT_A_DIRECTORY => return error.NotDir,
141 else => return unexpectedStatus(rc),
142 }
93 const rc = ntdll.NtCreateFile(
94 &result,
95 options.access_mask,
96 &attr,
97 &io,
98 null,
99 FILE_ATTRIBUTE_NORMAL,
100 options.share_access,
101 options.creation,
102 flags,
103 null,
104 0,
105 );
106 switch (rc) {
107 .SUCCESS => {
108 if (std.io.is_async and options.io_mode == .evented) {
109 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
110 }
111 return result;
112 },
113 .OBJECT_NAME_INVALID => unreachable,
114 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
115 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
116 .NO_MEDIA_IN_DEVICE => return error.NoDevice,
117 .INVALID_PARAMETER => unreachable,
118 .SHARING_VIOLATION => return error.AccessDenied,
119 .ACCESS_DENIED => return error.AccessDenied,
120 .PIPE_BUSY => return error.PipeBusy,
121 .OBJECT_PATH_SYNTAX_BAD => unreachable,
122 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
123 .FILE_IS_A_DIRECTORY => return error.IsDir,
124 .NOT_A_DIRECTORY => return error.NotDir,
125 else => return unexpectedStatus(rc),
143126 }
144127}
145128
......@@ -1679,6 +1662,64 @@ pub fn SetFileTime(
16791662 }
16801663}
16811664
1665pub const LockFileError = error{
1666 SystemResources,
1667 WouldBlock,
1668} || std.os.UnexpectedError;
1669
1670pub fn LockFile(
1671 FileHandle: HANDLE,
1672 Event: ?HANDLE,
1673 ApcRoutine: ?*IO_APC_ROUTINE,
1674 ApcContext: ?*c_void,
1675 IoStatusBlock: *IO_STATUS_BLOCK,
1676 ByteOffset: *const LARGE_INTEGER,
1677 Length: *const LARGE_INTEGER,
1678 Key: ?*ULONG,
1679 FailImmediately: BOOLEAN,
1680 ExclusiveLock: BOOLEAN,
1681) !void {
1682 const rc = ntdll.NtLockFile(
1683 FileHandle,
1684 Event,
1685 ApcRoutine,
1686 ApcContext,
1687 IoStatusBlock,
1688 ByteOffset,
1689 Length,
1690 Key,
1691 FailImmediately,
1692 ExclusiveLock,
1693 );
1694 switch (rc) {
1695 .SUCCESS => return,
1696 .INSUFFICIENT_RESOURCES => return error.SystemResources,
1697 .LOCK_NOT_GRANTED => return error.WouldBlock,
1698 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
1699 else => return unexpectedStatus(rc),
1700 }
1701}
1702
1703pub const UnlockFileError = error{
1704 RangeNotLocked,
1705} || std.os.UnexpectedError;
1706
1707pub fn UnlockFile(
1708 FileHandle: HANDLE,
1709 IoStatusBlock: *IO_STATUS_BLOCK,
1710 ByteOffset: *const LARGE_INTEGER,
1711 Length: *const LARGE_INTEGER,
1712 Key: ?*ULONG,
1713) !void {
1714 const rc = ntdll.NtUnlockFile(FileHandle, IoStatusBlock, ByteOffset, Length, Key);
1715 switch (rc) {
1716 .SUCCESS => return,
1717 .RANGE_NOT_LOCKED => return error.RangeNotLocked,
1718 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
1719 else => return unexpectedStatus(rc),
1720 }
1721}
1722
16821723pub fn teb() *TEB {
16831724 return switch (builtin.target.cpu.arch) {
16841725 .i386 => asm volatile (
lib/std/os/windows/ntdll.zig+21
......@@ -121,3 +121,24 @@ pub extern "NtDll" fn NtQueryObject(
121121 ObjectInformationLength: ULONG,
122122 ReturnLength: ?*ULONG,
123123) callconv(WINAPI) NTSTATUS;
124
125pub extern "NtDll" fn NtLockFile(
126 FileHandle: HANDLE,
127 Event: ?HANDLE,
128 ApcRoutine: ?*IO_APC_ROUTINE,
129 ApcContext: ?*c_void,
130 IoStatusBlock: *IO_STATUS_BLOCK,
131 ByteOffset: *const LARGE_INTEGER,
132 Length: *const LARGE_INTEGER,
133 Key: ?*ULONG,
134 FailImmediately: BOOLEAN,
135 ExclusiveLock: BOOLEAN,
136) callconv(WINAPI) NTSTATUS;
137
138pub extern "NtDll" fn NtUnlockFile(
139 FileHandle: HANDLE,
140 IoStatusBlock: *IO_STATUS_BLOCK,
141 ByteOffset: *const LARGE_INTEGER,
142 Length: *const LARGE_INTEGER,
143 Key: ?*ULONG,
144) callconv(WINAPI) NTSTATUS;
src/Cache.zig+123-54
......@@ -181,6 +181,12 @@ pub const Manifest = struct {
181181 hash: HashHelper,
182182 manifest_file: ?fs.File,
183183 manifest_dirty: bool,
184 /// Set this flag to true before calling hit() in order to indicate that
185 /// upon a cache hit, the code using the cache will not modify the files
186 /// within the cache directory. This allows multiple processes to utilize
187 /// the same cache directory at the same time.
188 want_shared_lock: bool = true,
189 have_exclusive_lock: bool = false,
184190 files: std.ArrayListUnmanaged(File) = .{},
185191 hex_digest: [hex_digest_len]u8,
186192 /// Populated when hit() returns an error because of one
......@@ -257,7 +263,9 @@ pub const Manifest = struct {
257263 ///
258264 /// This function will also acquire an exclusive lock to the manifest file. This means
259265 /// that a process holding a Manifest will block any other process attempting to
260 /// acquire the lock.
266 /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
267 /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
268 /// file to be locked in exclusive mode.
261269 ///
262270 /// The lock on the manifest file is released when `deinit` is called. As another
263271 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
......@@ -285,31 +293,62 @@ pub const Manifest = struct {
285293 mem.copy(u8, &manifest_file_path, &self.hex_digest);
286294 manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
287295
288 if (self.files.items.len != 0) {
289 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
290 .read = true,
291 .truncate = false,
292 .lock = .Exclusive,
293 });
294 } else {
296 if (self.files.items.len == 0) {
295297 // If there are no file inputs, we check if the manifest file exists instead of
296298 // comparing the hashes on the files used for the cached item
297 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
299 while (true) {
300 if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
301 .read = true,
302 .write = true,
303 .lock = .Exclusive,
304 .lock_nonblocking = self.want_shared_lock,
305 })) |manifest_file| {
306 self.manifest_file = manifest_file;
307 self.have_exclusive_lock = true;
308 break;
309 } else |open_err| switch (open_err) {
310 error.WouldBlock => {
311 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
312 .lock = .Shared,
313 });
314 break;
315 },
316 error.FileNotFound => {
317 if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
318 .read = true,
319 .truncate = false,
320 .lock = .Exclusive,
321 .lock_nonblocking = self.want_shared_lock,
322 })) |manifest_file| {
323 self.manifest_file = manifest_file;
324 self.manifest_dirty = true;
325 self.have_exclusive_lock = true;
326 return false; // cache miss; exclusive lock already held
327 } else |err| switch (err) {
328 error.WouldBlock => continue,
329 else => |e| return e,
330 }
331 },
332 else => |e| return e,
333 }
334 }
335 } else {
336 if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
298337 .read = true,
299 .write = true,
338 .truncate = false,
300339 .lock = .Exclusive,
301 }) catch |err| switch (err) {
302 error.FileNotFound => {
303 self.manifest_dirty = true;
304 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
305 .read = true,
306 .truncate = false,
307 .lock = .Exclusive,
340 .lock_nonblocking = self.want_shared_lock,
341 })) |manifest_file| {
342 self.manifest_file = manifest_file;
343 self.have_exclusive_lock = true;
344 } else |err| switch (err) {
345 error.WouldBlock => {
346 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
347 .lock = .Shared,
308348 });
309 return false;
310349 },
311350 else => |e| return e,
312 };
351 }
313352 }
314353
315354 const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);
......@@ -360,7 +399,10 @@ pub const Manifest = struct {
360399 }
361400
362401 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) {
363 error.FileNotFound => return false,
402 error.FileNotFound => {
403 try self.upgradeToExclusiveLock();
404 return false;
405 },
364406 else => return error.CacheUnavailable,
365407 };
366408 defer this_file.close();
......@@ -405,6 +447,7 @@ pub const Manifest = struct {
405447 // cache miss
406448 // keep the manifest file open
407449 self.unhit(bin_digest, input_file_count);
450 try self.upgradeToExclusiveLock();
408451 return false;
409452 }
410453
......@@ -417,9 +460,11 @@ pub const Manifest = struct {
417460 return err;
418461 };
419462 }
463 try self.upgradeToExclusiveLock();
420464 return false;
421465 }
422466
467 try self.downgradeToSharedLock();
423468 return true;
424469 }
425470
......@@ -585,34 +630,58 @@ pub const Manifest = struct {
585630 return out_digest;
586631 }
587632
633 /// If `want_shared_lock` is true, this function automatically downgrades the
634 /// lock from exclusive to shared.
588635 pub fn writeManifest(self: *Manifest) !void {
589636 const manifest_file = self.manifest_file.?;
590 if (!self.manifest_dirty) return;
591
592 var contents = std.ArrayList(u8).init(self.cache.gpa);
593 defer contents.deinit();
637 if (self.manifest_dirty) {
638 self.manifest_dirty = false;
639
640 var contents = std.ArrayList(u8).init(self.cache.gpa);
641 defer contents.deinit();
642
643 const writer = contents.writer();
644 var encoded_digest: [hex_digest_len]u8 = undefined;
645
646 for (self.files.items) |file| {
647 _ = std.fmt.bufPrint(
648 &encoded_digest,
649 "{s}",
650 .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
651 ) catch unreachable;
652 try writer.print("{d} {d} {d} {s} {s}\n", .{
653 file.stat.size,
654 file.stat.inode,
655 file.stat.mtime,
656 &encoded_digest,
657 file.path,
658 });
659 }
594660
595 const writer = contents.writer();
596 var encoded_digest: [hex_digest_len]u8 = undefined;
661 try manifest_file.setEndPos(contents.items.len);
662 try manifest_file.pwriteAll(contents.items, 0);
663 }
597664
598 for (self.files.items) |file| {
599 _ = std.fmt.bufPrint(
600 &encoded_digest,
601 "{s}",
602 .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
603 ) catch unreachable;
604 try writer.print("{d} {d} {d} {s} {s}\n", .{
605 file.stat.size,
606 file.stat.inode,
607 file.stat.mtime,
608 &encoded_digest,
609 file.path,
610 });
665 if (self.want_shared_lock) {
666 try self.downgradeToSharedLock();
611667 }
668 }
669
670 fn downgradeToSharedLock(self: *Manifest) !void {
671 if (!self.have_exclusive_lock) return;
672 const manifest_file = self.manifest_file.?;
673 try manifest_file.downgradeLock();
674 self.have_exclusive_lock = false;
675 }
612676
613 try manifest_file.setEndPos(contents.items.len);
614 try manifest_file.pwriteAll(contents.items, 0);
615 self.manifest_dirty = false;
677 fn upgradeToExclusiveLock(self: *Manifest) !void {
678 if (self.have_exclusive_lock) return;
679 const manifest_file = self.manifest_file.?;
680 // Here we intentionally have a period where the lock is released, in case there are
681 // other processes holding a shared lock.
682 manifest_file.unlock();
683 try manifest_file.lock(.Exclusive);
684 self.have_exclusive_lock = true;
616685 }
617686
618687 /// Obtain only the data needed to maintain a lock on the manifest file.
......@@ -881,27 +950,27 @@ test "no file inputs" {
881950 defer cache.manifest_dir.close();
882951
883952 {
884 var ch = cache.obtain();
885 defer ch.deinit();
953 var man = cache.obtain();
954 defer man.deinit();
886955
887 ch.hash.addBytes("1234");
956 man.hash.addBytes("1234");
888957
889958 // There should be nothing in the cache
890 try testing.expectEqual(false, try ch.hit());
959 try testing.expectEqual(false, try man.hit());
891960
892 digest1 = ch.final();
961 digest1 = man.final();
893962
894 try ch.writeManifest();
963 try man.writeManifest();
895964 }
896965 {
897 var ch = cache.obtain();
898 defer ch.deinit();
966 var man = cache.obtain();
967 defer man.deinit();
899968
900 ch.hash.addBytes("1234");
969 man.hash.addBytes("1234");
901970
902 try testing.expect(try ch.hit());
903 digest2 = ch.final();
904 try ch.writeManifest();
971 try testing.expect(try man.hit());
972 digest2 = man.final();
973 try man.writeManifest();
905974 }
906975
907976 try testing.expectEqual(digest1, digest2);
src/Compilation.zig-22
......@@ -39,7 +39,6 @@ gpa: *Allocator,
3939arena_state: std.heap.ArenaAllocator.State,
4040bin_file: *link.File,
4141c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
42c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{},
4342stage1_lock: ?Cache.Lock = null,
4443stage1_cache_manifest: *Cache.Manifest = undefined,
4544
......@@ -1570,7 +1569,6 @@ pub fn destroy(self: *Compilation) void {
15701569 key.destroy(gpa);
15711570 }
15721571 self.c_object_table.deinit(gpa);
1573 self.c_object_cache_digest_set.deinit(gpa);
15741572
15751573 for (self.failed_c_objects.values()) |value| {
15761574 value.destroy(gpa);
......@@ -1607,7 +1605,6 @@ pub fn update(self: *Compilation) !void {
16071605 defer tracy.end();
16081606
16091607 self.clearMiscFailures();
1610 self.c_object_cache_digest_set.clearRetainingCapacity();
16111608
16121609 // For compiling C objects, we rely on the cache hash system to avoid duplicating work.
16131610 // Add a Job for each C object.
......@@ -2566,25 +2563,6 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
25662563
25672564 try man.hashCSource(c_object.src);
25682565
2569 {
2570 const is_collision = blk: {
2571 const bin_digest = man.hash.peekBin();
2572
2573 const lock = comp.mutex.acquire();
2574 defer lock.release();
2575
2576 const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
2577 break :blk gop.found_existing;
2578 };
2579 if (is_collision) {
2580 return comp.failCObj(
2581 c_object,
2582 "the same source file was already added to the same compilation with the same flags",
2583 .{},
2584 );
2585 }
2586 }
2587
25882566 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
25892567 defer arena_allocator.deinit();
25902568 const arena = &arena_allocator.allocator;