authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-29 17:29:59-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-07-03 15:28:29-07:00
log390a4ded986237381fffbe46719445b6cd9733f3
tree4d92daac7b55c38d9887828a9c9264ccede3f98a
parentb0be0c77d8fb11e1abed578f37a859a2084a934e

Merge pull request #9258 from ziglang/shared-cache-locking

Shared Cache Locking

7 files changed, 487 insertions(+), 157 deletions(-)

doc/langref.html.in+8
...@@ -10516,6 +10516,14 @@ fn readU32Be() u32 {}...@@ -10516,6 +10516,14 @@ fn readU32Be() u32 {}
10516 See the Zig Standard Library for more examples.10516 See the Zig Standard Library for more examples.
10517 </p>10517 </p>
10518 {#header_close#}10518 {#header_close#}
10519 {#header_open|Doc Comment Guidance#}
10520 <ul>
10521 <li>Omit any information that is redundant based on the name of the thing being documented.</li>
10522 <li>Duplicating information onto multiple similar functions is encouraged because it helps IDEs and other tools provide better help text.</li>
10523 <li>Use the word <strong>assume</strong> to indicate invariants that cause {#link|Undefined Behavior#} when violated.</li>
10524 <li>Use the word <strong>assert</strong> to indicate invariants that cause <em>safety-checked</em> {#link|Undefined Behavior#} when violated.</li>
10525 </ul>
10526 {#header_close#}
10519 {#header_close#}10527 {#header_close#}
10520 {#header_open|Source Encoding#}10528 {#header_open|Source Encoding#}
10521 <p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>10529 <p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>
lib/std/fs.zig+47-17
...@@ -883,24 +883,39 @@ pub const Dir = struct {...@@ -883,24 +883,39 @@ pub const Dir = struct {
883 /// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.883 /// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
884 pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {884 pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
885 const w = os.windows;885 const w = os.windows;
886 return @as(File, .{886 const file: File = .{
887 .handle = try os.windows.OpenFile(sub_path_w, .{887 .handle = try w.OpenFile(sub_path_w, .{
888 .dir = self.fd,888 .dir = self.fd,
889 .access_mask = w.SYNCHRONIZE |889 .access_mask = w.SYNCHRONIZE |
890 (if (flags.read) @as(u32, w.GENERIC_READ) else 0) |890 (if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
891 (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),891 (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
892 .share_access = switch (flags.lock) {
893 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
894 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
895 .Exclusive => w.FILE_SHARE_DELETE,
896 },
897 .share_access_nonblocking = flags.lock_nonblocking,
898 .creation = w.FILE_OPEN,892 .creation = w.FILE_OPEN,
899 .io_mode = flags.intended_io_mode,893 .io_mode = flags.intended_io_mode,
900 }),894 }),
901 .capable_io_mode = std.io.default_mode,895 .capable_io_mode = std.io.default_mode,
902 .intended_io_mode = flags.intended_io_mode,896 .intended_io_mode = flags.intended_io_mode,
903 });897 };
898 var io: w.IO_STATUS_BLOCK = undefined;
899 const range_off: w.LARGE_INTEGER = 0;
900 const range_len: w.LARGE_INTEGER = 1;
901 const exclusive = switch (flags.lock) {
902 .None => return file,
903 .Shared => false,
904 .Exclusive => true,
905 };
906 try w.LockFile(
907 file.handle,
908 null,
909 null,
910 null,
911 &io,
912 &range_off,
913 &range_len,
914 null,
915 @boolToInt(flags.lock_nonblocking),
916 @boolToInt(exclusive),
917 );
918 return file;
904 }919 }
905920
906 /// Creates, opens, or overwrites a file with write access.921 /// Creates, opens, or overwrites a file with write access.
...@@ -1019,16 +1034,10 @@ pub const Dir = struct {...@@ -1019,16 +1034,10 @@ pub const Dir = struct {
1019 pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {1034 pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
1020 const w = os.windows;1035 const w = os.windows;
1021 const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;1036 const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
1022 return @as(File, .{1037 const file: File = .{
1023 .handle = try os.windows.OpenFile(sub_path_w, .{1038 .handle = try os.windows.OpenFile(sub_path_w, .{
1024 .dir = self.fd,1039 .dir = self.fd,
1025 .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,1040 .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
1026 .share_access = switch (flags.lock) {
1027 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1028 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1029 .Exclusive => w.FILE_SHARE_DELETE,
1030 },
1031 .share_access_nonblocking = flags.lock_nonblocking,
1032 .creation = if (flags.exclusive)1041 .creation = if (flags.exclusive)
1033 @as(u32, w.FILE_CREATE)1042 @as(u32, w.FILE_CREATE)
1034 else if (flags.truncate)1043 else if (flags.truncate)
...@@ -1039,7 +1048,28 @@ pub const Dir = struct {...@@ -1039,7 +1048,28 @@ pub const Dir = struct {
1039 }),1048 }),
1040 .capable_io_mode = std.io.default_mode,1049 .capable_io_mode = std.io.default_mode,
1041 .intended_io_mode = flags.intended_io_mode,1050 .intended_io_mode = flags.intended_io_mode,
1042 });1051 };
1052 var io: w.IO_STATUS_BLOCK = undefined;
1053 const range_off: w.LARGE_INTEGER = 0;
1054 const range_len: w.LARGE_INTEGER = 1;
1055 const exclusive = switch (flags.lock) {
1056 .None => return file,
1057 .Shared => false,
1058 .Exclusive => true,
1059 };
1060 try w.LockFile(
1061 file.handle,
1062 null,
1063 null,
1064 null,
1065 &io,
1066 &range_off,
1067 &range_len,
1068 null,
1069 @boolToInt(flags.lock_nonblocking),
1070 @boolToInt(exclusive),
1071 );
1072 return file;
1043 }1073 }
10441074
1045 pub const openRead = @compileError("deprecated in favor of openFile");1075 pub const openRead = @compileError("deprecated in favor of openFile");
lib/std/fs/file.zig+197-14
...@@ -74,17 +74,28 @@ pub const File = struct {...@@ -74,17 +74,28 @@ pub const File = struct {
74 read: bool = true,74 read: bool = true,
75 write: bool = false,75 write: bool = false,
7676
77 /// Open the file with a lock to prevent other processes from accessing it at the77 /// Open the file with an advisory lock to coordinate with other processes
78 /// same time. An exclusive lock will prevent other processes from acquiring a lock.78 /// accessing it at the same time. An exclusive lock will prevent other
79 /// A shared lock will prevent other processes from acquiring a exclusive lock, but79 /// processes from acquiring a lock. A shared lock will prevent other
80 /// doesn't prevent other process from getting their own shared locks.80 /// processes from acquiring a exclusive lock, but does not prevent
81 /// other process from getting their own shared locks.
81 ///82 ///
82 /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].83 /// The lock is advisory, except on Linux in very specific cirsumstances[1].
83 /// This means that a process that does not respect the locking API can still get access84 /// This means that a process that does not respect the locking API can still get access
84 /// to the file, despite the lock.85 /// to the file, despite the lock.
85 ///86 ///
86 /// Windows' file locks are mandatory, and any process attempting to access the file will87 /// On these operating systems, the lock is acquired atomically with
87 /// receive an error.88 /// opening the file:
89 /// * Darwin
90 /// * DragonFlyBSD
91 /// * FreeBSD
92 /// * Haiku
93 /// * NetBSD
94 /// * OpenBSD
95 /// On these operating systems, the lock is acquired via a separate syscall
96 /// after opening the file:
97 /// * Linux
98 /// * Windows
88 ///99 ///
89 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt100 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
90 lock: Lock = .None,101 lock: Lock = .None,
...@@ -120,17 +131,28 @@ pub const File = struct {...@@ -120,17 +131,28 @@ pub const File = struct {
120 /// `error.PathAlreadyExists` to be returned.131 /// `error.PathAlreadyExists` to be returned.
121 exclusive: bool = false,132 exclusive: bool = false,
122133
123 /// Open the file with a lock to prevent other processes from accessing it at the134 /// Open the file with an advisory lock to coordinate with other processes
124 /// same time. An exclusive lock will prevent other processes from acquiring a lock.135 /// accessing it at the same time. An exclusive lock will prevent other
125 /// A shared lock will prevent other processes from acquiring a exclusive lock, but136 /// processes from acquiring a lock. A shared lock will prevent other
126 /// doesn't prevent other process from getting their own shared locks.137 /// processes from acquiring a exclusive lock, but does not prevent
138 /// other process from getting their own shared locks.
127 ///139 ///
128 /// Note that the lock is only advisory on Linux, except in very specific cirsumstances[1].140 /// The lock is advisory, except on Linux in very specific cirsumstances[1].
129 /// This means that a process that does not respect the locking API can still get access141 /// This means that a process that does not respect the locking API can still get access
130 /// to the file, despite the lock.142 /// to the file, despite the lock.
131 ///143 ///
132 /// Windows's file locks are mandatory, and any process attempting to access the file will144 /// On these operating systems, the lock is acquired atomically with
133 /// receive an error.145 /// opening the file:
146 /// * Darwin
147 /// * DragonFlyBSD
148 /// * FreeBSD
149 /// * Haiku
150 /// * NetBSD
151 /// * OpenBSD
152 /// On these operating systems, the lock is acquired via a separate syscall
153 /// after opening the file:
154 /// * Linux
155 /// * Windows
134 ///156 ///
135 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt157 /// [1]: https://www.kernel.org/doc/Documentation/filesystems/mandatory-locking.txt
136 lock: Lock = .None,158 lock: Lock = .None,
...@@ -829,4 +851,165 @@ pub const File = struct {...@@ -829,4 +851,165 @@ pub const File = struct {
829 pub fn seekableStream(file: File) SeekableStream {851 pub fn seekableStream(file: File) SeekableStream {
830 return .{ .context = file };852 return .{ .context = file };
831 }853 }
854
855 const range_off: windows.LARGE_INTEGER = 0;
856 const range_len: windows.LARGE_INTEGER = 1;
857
858 pub const LockError = error{
859 SystemResources,
860 } || os.UnexpectedError;
861
862 /// Blocks when an incompatible lock is held by another process.
863 /// A process may hold only one type of lock (shared or exclusive) on
864 /// a file. When a process terminates in any way, the lock is released.
865 ///
866 /// Assumes the file is unlocked.
867 ///
868 /// TODO: integrate with async I/O
869 pub fn lock(file: File, l: Lock) LockError!void {
870 if (is_windows) {
871 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
872 const exclusive = switch (l) {
873 .None => return,
874 .Shared => false,
875 .Exclusive => true,
876 };
877 return windows.LockFile(
878 file.handle,
879 null,
880 null,
881 null,
882 &io_status_block,
883 &range_off,
884 &range_len,
885 null,
886 windows.FALSE, // non-blocking=false
887 @boolToInt(exclusive),
888 ) catch |err| switch (err) {
889 error.WouldBlock => unreachable, // non-blocking=false
890 else => |e| return e,
891 };
892 } else {
893 return os.flock(file.handle, switch (l) {
894 .None => os.LOCK_UN,
895 .Shared => os.LOCK_SH,
896 .Exclusive => os.LOCK_EX,
897 }) catch |err| switch (err) {
898 error.WouldBlock => unreachable, // non-blocking=false
899 else => |e| return e,
900 };
901 }
902 }
903
904 /// Assumes the file is locked.
905 pub fn unlock(file: File) void {
906 if (is_windows) {
907 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
908 return windows.UnlockFile(
909 file.handle,
910 &io_status_block,
911 &range_off,
912 &range_len,
913 null,
914 ) catch |err| switch (err) {
915 error.RangeNotLocked => unreachable, // Function assumes unlocked.
916 error.Unexpected => unreachable, // Resource deallocation must succeed.
917 };
918 } else {
919 return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
920 error.WouldBlock => unreachable, // unlocking can't block
921 error.SystemResources => unreachable, // We are deallocating resources.
922 error.Unexpected => unreachable, // Resource deallocation must succeed.
923 };
924 }
925 }
926
927 /// Attempts to obtain a lock, returning `true` if the lock is
928 /// obtained, and `false` if there was an existing incompatible lock held.
929 /// A process may hold only one type of lock (shared or exclusive) on
930 /// a file. When a process terminates in any way, the lock is released.
931 ///
932 /// Assumes the file is unlocked.
933 ///
934 /// TODO: integrate with async I/O
935 pub fn tryLock(file: File, l: Lock) LockError!bool {
936 if (is_windows) {
937 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
938 const exclusive = switch (l) {
939 .None => return,
940 .Shared => false,
941 .Exclusive => true,
942 };
943 windows.LockFile(
944 file.handle,
945 null,
946 null,
947 null,
948 &io_status_block,
949 &range_off,
950 &range_len,
951 null,
952 windows.TRUE, // non-blocking=true
953 @boolToInt(exclusive),
954 ) catch |err| switch (err) {
955 error.WouldBlock => return false,
956 else => |e| return e,
957 };
958 } else {
959 os.flock(file.handle, switch (l) {
960 .None => os.LOCK_UN,
961 .Shared => os.LOCK_SH | os.LOCK_NB,
962 .Exclusive => os.LOCK_EX | os.LOCK_NB,
963 }) catch |err| switch (err) {
964 error.WouldBlock => return false,
965 else => |e| return e,
966 };
967 }
968 return true;
969 }
970
971 /// Assumes the file is already locked in exclusive mode.
972 /// Atomically modifies the lock to be in shared mode, without releasing it.
973 ///
974 /// TODO: integrate with async I/O
975 pub fn downgradeLock(file: File) LockError!void {
976 if (is_windows) {
977 // On Windows it works like a semaphore + exclusivity flag. To implement this
978 // function, we first obtain another lock in shared mode. This changes the
979 // exclusivity flag, but increments the semaphore to 2. So we follow up with
980 // an NtUnlockFile which decrements the semaphore but does not modify the
981 // exclusivity flag.
982 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
983 windows.LockFile(
984 file.handle,
985 null,
986 null,
987 null,
988 &io_status_block,
989 &range_off,
990 &range_len,
991 null,
992 windows.TRUE, // non-blocking=true
993 windows.FALSE, // exclusive=false
994 ) catch |err| switch (err) {
995 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
996 else => |e| return e,
997 };
998 return windows.UnlockFile(
999 file.handle,
1000 &io_status_block,
1001 &range_off,
1002 &range_len,
1003 null,
1004 ) catch |err| switch (err) {
1005 error.RangeNotLocked => unreachable, // File was not locked.
1006 error.Unexpected => unreachable, // Resource deallocation must succeed.
1007 };
1008 } else {
1009 return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
1010 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
1011 else => |e| return e,
1012 };
1013 }
1014 }
832};1015};
lib/std/os/windows.zig+91-50
...@@ -48,7 +48,6 @@ pub const OpenFileOptions = struct {...@@ -48,7 +48,6 @@ pub const OpenFileOptions = struct {
48 dir: ?HANDLE = null,48 dir: ?HANDLE = null,
49 sa: ?*SECURITY_ATTRIBUTES = null,49 sa: ?*SECURITY_ATTRIBUTES = null,
50 share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,50 share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
51 share_access_nonblocking: bool = false,
52 creation: ULONG,51 creation: ULONG,
53 io_mode: std.io.ModeOverride,52 io_mode: std.io.ModeOverride,
54 /// If true, tries to open path as a directory.53 /// If true, tries to open path as a directory.
...@@ -59,8 +58,6 @@ pub const OpenFileOptions = struct {...@@ -59,8 +58,6 @@ pub const OpenFileOptions = struct {
59 follow_symlinks: bool = true,58 follow_symlinks: bool = true,
60};59};
6160
62/// TODO when share_access_nonblocking is false, this implementation uses
63/// untinterruptible sleep() to block. This is not the final iteration of the API.
64pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {61pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
65 if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {62 if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
66 return error.IsDir;63 return error.IsDir;
...@@ -93,53 +90,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN...@@ -93,53 +90,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
93 // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.90 // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
94 const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;91 const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
9592
96 var delay: usize = 1;93 const rc = ntdll.NtCreateFile(
97 while (true) {94 &result,
98 const rc = ntdll.NtCreateFile(95 options.access_mask,
99 &result,96 &attr,
100 options.access_mask,97 &io,
101 &attr,98 null,
102 &io,99 FILE_ATTRIBUTE_NORMAL,
103 null,100 options.share_access,
104 FILE_ATTRIBUTE_NORMAL,101 options.creation,
105 options.share_access,102 flags,
106 options.creation,103 null,
107 flags,104 0,
108 null,105 );
109 0,106 switch (rc) {
110 );107 .SUCCESS => {
111 switch (rc) {108 if (std.io.is_async and options.io_mode == .evented) {
112 .SUCCESS => {109 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
113 if (std.io.is_async and options.io_mode == .evented) {110 }
114 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;111 return result;
115 }112 },
116 return result;113 .OBJECT_NAME_INVALID => unreachable,
117 },114 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
118 .OBJECT_NAME_INVALID => unreachable,115 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
119 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,116 .NO_MEDIA_IN_DEVICE => return error.NoDevice,
120 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,117 .INVALID_PARAMETER => unreachable,
121 .NO_MEDIA_IN_DEVICE => return error.NoDevice,118 .SHARING_VIOLATION => return error.AccessDenied,
122 .INVALID_PARAMETER => unreachable,119 .ACCESS_DENIED => return error.AccessDenied,
123 .SHARING_VIOLATION => {120 .PIPE_BUSY => return error.PipeBusy,
124 if (options.share_access_nonblocking) {121 .OBJECT_PATH_SYNTAX_BAD => unreachable,
125 return error.WouldBlock;122 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
126 }123 .FILE_IS_A_DIRECTORY => return error.IsDir,
127 // TODO sleep in a way that is interruptable124 .NOT_A_DIRECTORY => return error.NotDir,
128 // TODO integrate with async I/O125 else => return unexpectedStatus(rc),
129 std.time.sleep(delay);
130 if (delay < 1 * std.time.ns_per_s) {
131 delay *= 2;
132 }
133 continue;
134 },
135 .ACCESS_DENIED => return error.AccessDenied,
136 .PIPE_BUSY => return error.PipeBusy,
137 .OBJECT_PATH_SYNTAX_BAD => unreachable,
138 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
139 .FILE_IS_A_DIRECTORY => return error.IsDir,
140 .NOT_A_DIRECTORY => return error.NotDir,
141 else => return unexpectedStatus(rc),
142 }
143 }126 }
144}127}
145128
...@@ -1679,6 +1662,64 @@ pub fn SetFileTime(...@@ -1679,6 +1662,64 @@ pub fn SetFileTime(
1679 }1662 }
1680}1663}
16811664
1665pub const LockFileError = error{
1666 SystemResources,
1667 WouldBlock,
1668} || std.os.UnexpectedError;
1669
1670pub fn LockFile(
1671 FileHandle: HANDLE,
1672 Event: ?HANDLE,
1673 ApcRoutine: ?*IO_APC_ROUTINE,
1674 ApcContext: ?*c_void,
1675 IoStatusBlock: *IO_STATUS_BLOCK,
1676 ByteOffset: *const LARGE_INTEGER,
1677 Length: *const LARGE_INTEGER,
1678 Key: ?*ULONG,
1679 FailImmediately: BOOLEAN,
1680 ExclusiveLock: BOOLEAN,
1681) !void {
1682 const rc = ntdll.NtLockFile(
1683 FileHandle,
1684 Event,
1685 ApcRoutine,
1686 ApcContext,
1687 IoStatusBlock,
1688 ByteOffset,
1689 Length,
1690 Key,
1691 FailImmediately,
1692 ExclusiveLock,
1693 );
1694 switch (rc) {
1695 .SUCCESS => return,
1696 .INSUFFICIENT_RESOURCES => return error.SystemResources,
1697 .LOCK_NOT_GRANTED => return error.WouldBlock,
1698 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
1699 else => return unexpectedStatus(rc),
1700 }
1701}
1702
1703pub const UnlockFileError = error{
1704 RangeNotLocked,
1705} || std.os.UnexpectedError;
1706
1707pub fn UnlockFile(
1708 FileHandle: HANDLE,
1709 IoStatusBlock: *IO_STATUS_BLOCK,
1710 ByteOffset: *const LARGE_INTEGER,
1711 Length: *const LARGE_INTEGER,
1712 Key: ?*ULONG,
1713) !void {
1714 const rc = ntdll.NtUnlockFile(FileHandle, IoStatusBlock, ByteOffset, Length, Key);
1715 switch (rc) {
1716 .SUCCESS => return,
1717 .RANGE_NOT_LOCKED => return error.RangeNotLocked,
1718 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
1719 else => return unexpectedStatus(rc),
1720 }
1721}
1722
1682pub fn teb() *TEB {1723pub fn teb() *TEB {
1683 return switch (builtin.target.cpu.arch) {1724 return switch (builtin.target.cpu.arch) {
1684 .i386 => asm volatile (1725 .i386 => asm volatile (
lib/std/os/windows/ntdll.zig+21
...@@ -121,3 +121,24 @@ pub extern "NtDll" fn NtQueryObject(...@@ -121,3 +121,24 @@ pub extern "NtDll" fn NtQueryObject(
121 ObjectInformationLength: ULONG,121 ObjectInformationLength: ULONG,
122 ReturnLength: ?*ULONG,122 ReturnLength: ?*ULONG,
123) callconv(WINAPI) NTSTATUS;123) callconv(WINAPI) NTSTATUS;
124
125pub extern "NtDll" fn NtLockFile(
126 FileHandle: HANDLE,
127 Event: ?HANDLE,
128 ApcRoutine: ?*IO_APC_ROUTINE,
129 ApcContext: ?*c_void,
130 IoStatusBlock: *IO_STATUS_BLOCK,
131 ByteOffset: *const LARGE_INTEGER,
132 Length: *const LARGE_INTEGER,
133 Key: ?*ULONG,
134 FailImmediately: BOOLEAN,
135 ExclusiveLock: BOOLEAN,
136) callconv(WINAPI) NTSTATUS;
137
138pub extern "NtDll" fn NtUnlockFile(
139 FileHandle: HANDLE,
140 IoStatusBlock: *IO_STATUS_BLOCK,
141 ByteOffset: *const LARGE_INTEGER,
142 Length: *const LARGE_INTEGER,
143 Key: ?*ULONG,
144) callconv(WINAPI) NTSTATUS;
src/Cache.zig+123-54
...@@ -181,6 +181,12 @@ pub const Manifest = struct {...@@ -181,6 +181,12 @@ pub const Manifest = struct {
181 hash: HashHelper,181 hash: HashHelper,
182 manifest_file: ?fs.File,182 manifest_file: ?fs.File,
183 manifest_dirty: bool,183 manifest_dirty: bool,
184 /// Set this flag to true before calling hit() in order to indicate that
185 /// upon a cache hit, the code using the cache will not modify the files
186 /// within the cache directory. This allows multiple processes to utilize
187 /// the same cache directory at the same time.
188 want_shared_lock: bool = true,
189 have_exclusive_lock: bool = false,
184 files: std.ArrayListUnmanaged(File) = .{},190 files: std.ArrayListUnmanaged(File) = .{},
185 hex_digest: [hex_digest_len]u8,191 hex_digest: [hex_digest_len]u8,
186 /// Populated when hit() returns an error because of one192 /// Populated when hit() returns an error because of one
...@@ -257,7 +263,9 @@ pub const Manifest = struct {...@@ -257,7 +263,9 @@ pub const Manifest = struct {
257 ///263 ///
258 /// This function will also acquire an exclusive lock to the manifest file. This means264 /// This function will also acquire an exclusive lock to the manifest file. This means
259 /// that a process holding a Manifest will block any other process attempting to265 /// that a process holding a Manifest will block any other process attempting to
260 /// acquire the lock.266 /// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
267 /// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
268 /// file to be locked in exclusive mode.
261 ///269 ///
262 /// The lock on the manifest file is released when `deinit` is called. As another270 /// The lock on the manifest file is released when `deinit` is called. As another
263 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent271 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
...@@ -285,31 +293,62 @@ pub const Manifest = struct {...@@ -285,31 +293,62 @@ pub const Manifest = struct {
285 mem.copy(u8, &manifest_file_path, &self.hex_digest);293 mem.copy(u8, &manifest_file_path, &self.hex_digest);
286 manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;294 manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
287295
288 if (self.files.items.len != 0) {296 if (self.files.items.len == 0) {
289 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{
290 .read = true,
291 .truncate = false,
292 .lock = .Exclusive,
293 });
294 } else {
295 // If there are no file inputs, we check if the manifest file exists instead of297 // If there are no file inputs, we check if the manifest file exists instead of
296 // comparing the hashes on the files used for the cached item298 // comparing the hashes on the files used for the cached item
297 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{299 while (true) {
300 if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
301 .read = true,
302 .write = true,
303 .lock = .Exclusive,
304 .lock_nonblocking = self.want_shared_lock,
305 })) |manifest_file| {
306 self.manifest_file = manifest_file;
307 self.have_exclusive_lock = true;
308 break;
309 } else |open_err| switch (open_err) {
310 error.WouldBlock => {
311 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
312 .lock = .Shared,
313 });
314 break;
315 },
316 error.FileNotFound => {
317 if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
318 .read = true,
319 .truncate = false,
320 .lock = .Exclusive,
321 .lock_nonblocking = self.want_shared_lock,
322 })) |manifest_file| {
323 self.manifest_file = manifest_file;
324 self.manifest_dirty = true;
325 self.have_exclusive_lock = true;
326 return false; // cache miss; exclusive lock already held
327 } else |err| switch (err) {
328 error.WouldBlock => continue,
329 else => |e| return e,
330 }
331 },
332 else => |e| return e,
333 }
334 }
335 } else {
336 if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
298 .read = true,337 .read = true,
299 .write = true,338 .truncate = false,
300 .lock = .Exclusive,339 .lock = .Exclusive,
301 }) catch |err| switch (err) {340 .lock_nonblocking = self.want_shared_lock,
302 error.FileNotFound => {341 })) |manifest_file| {
303 self.manifest_dirty = true;342 self.manifest_file = manifest_file;
304 self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{343 self.have_exclusive_lock = true;
305 .read = true,344 } else |err| switch (err) {
306 .truncate = false,345 error.WouldBlock => {
307 .lock = .Exclusive,346 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
347 .lock = .Shared,
308 });348 });
309 return false;
310 },349 },
311 else => |e| return e,350 else => |e| return e,
312 };351 }
313 }352 }
314353
315 const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);354 const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);
...@@ -360,7 +399,10 @@ pub const Manifest = struct {...@@ -360,7 +399,10 @@ pub const Manifest = struct {
360 }399 }
361400
362 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) {401 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch |err| switch (err) {
363 error.FileNotFound => return false,402 error.FileNotFound => {
403 try self.upgradeToExclusiveLock();
404 return false;
405 },
364 else => return error.CacheUnavailable,406 else => return error.CacheUnavailable,
365 };407 };
366 defer this_file.close();408 defer this_file.close();
...@@ -405,6 +447,7 @@ pub const Manifest = struct {...@@ -405,6 +447,7 @@ pub const Manifest = struct {
405 // cache miss447 // cache miss
406 // keep the manifest file open448 // keep the manifest file open
407 self.unhit(bin_digest, input_file_count);449 self.unhit(bin_digest, input_file_count);
450 try self.upgradeToExclusiveLock();
408 return false;451 return false;
409 }452 }
410453
...@@ -417,9 +460,11 @@ pub const Manifest = struct {...@@ -417,9 +460,11 @@ pub const Manifest = struct {
417 return err;460 return err;
418 };461 };
419 }462 }
463 try self.upgradeToExclusiveLock();
420 return false;464 return false;
421 }465 }
422466
467 try self.downgradeToSharedLock();
423 return true;468 return true;
424 }469 }
425470
...@@ -585,34 +630,58 @@ pub const Manifest = struct {...@@ -585,34 +630,58 @@ pub const Manifest = struct {
585 return out_digest;630 return out_digest;
586 }631 }
587632
633 /// If `want_shared_lock` is true, this function automatically downgrades the
634 /// lock from exclusive to shared.
588 pub fn writeManifest(self: *Manifest) !void {635 pub fn writeManifest(self: *Manifest) !void {
589 const manifest_file = self.manifest_file.?;636 const manifest_file = self.manifest_file.?;
590 if (!self.manifest_dirty) return;637 if (self.manifest_dirty) {
591638 self.manifest_dirty = false;
592 var contents = std.ArrayList(u8).init(self.cache.gpa);639
593 defer contents.deinit();640 var contents = std.ArrayList(u8).init(self.cache.gpa);
641 defer contents.deinit();
642
643 const writer = contents.writer();
644 var encoded_digest: [hex_digest_len]u8 = undefined;
645
646 for (self.files.items) |file| {
647 _ = std.fmt.bufPrint(
648 &encoded_digest,
649 "{s}",
650 .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
651 ) catch unreachable;
652 try writer.print("{d} {d} {d} {s} {s}\n", .{
653 file.stat.size,
654 file.stat.inode,
655 file.stat.mtime,
656 &encoded_digest,
657 file.path,
658 });
659 }
594660
595 const writer = contents.writer();661 try manifest_file.setEndPos(contents.items.len);
596 var encoded_digest: [hex_digest_len]u8 = undefined;662 try manifest_file.pwriteAll(contents.items, 0);
663 }
597664
598 for (self.files.items) |file| {665 if (self.want_shared_lock) {
599 _ = std.fmt.bufPrint(666 try self.downgradeToSharedLock();
600 &encoded_digest,
601 "{s}",
602 .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
603 ) catch unreachable;
604 try writer.print("{d} {d} {d} {s} {s}\n", .{
605 file.stat.size,
606 file.stat.inode,
607 file.stat.mtime,
608 &encoded_digest,
609 file.path,
610 });
611 }667 }
668 }
669
670 fn downgradeToSharedLock(self: *Manifest) !void {
671 if (!self.have_exclusive_lock) return;
672 const manifest_file = self.manifest_file.?;
673 try manifest_file.downgradeLock();
674 self.have_exclusive_lock = false;
675 }
612676
613 try manifest_file.setEndPos(contents.items.len);677 fn upgradeToExclusiveLock(self: *Manifest) !void {
614 try manifest_file.pwriteAll(contents.items, 0);678 if (self.have_exclusive_lock) return;
615 self.manifest_dirty = false;679 const manifest_file = self.manifest_file.?;
680 // Here we intentionally have a period where the lock is released, in case there are
681 // other processes holding a shared lock.
682 manifest_file.unlock();
683 try manifest_file.lock(.Exclusive);
684 self.have_exclusive_lock = true;
616 }685 }
617686
618 /// Obtain only the data needed to maintain a lock on the manifest file.687 /// Obtain only the data needed to maintain a lock on the manifest file.
...@@ -881,27 +950,27 @@ test "no file inputs" {...@@ -881,27 +950,27 @@ test "no file inputs" {
881 defer cache.manifest_dir.close();950 defer cache.manifest_dir.close();
882951
883 {952 {
884 var ch = cache.obtain();953 var man = cache.obtain();
885 defer ch.deinit();954 defer man.deinit();
886955
887 ch.hash.addBytes("1234");956 man.hash.addBytes("1234");
888957
889 // There should be nothing in the cache958 // There should be nothing in the cache
890 try testing.expectEqual(false, try ch.hit());959 try testing.expectEqual(false, try man.hit());
891960
892 digest1 = ch.final();961 digest1 = man.final();
893962
894 try ch.writeManifest();963 try man.writeManifest();
895 }964 }
896 {965 {
897 var ch = cache.obtain();966 var man = cache.obtain();
898 defer ch.deinit();967 defer man.deinit();
899968
900 ch.hash.addBytes("1234");969 man.hash.addBytes("1234");
901970
902 try testing.expect(try ch.hit());971 try testing.expect(try man.hit());
903 digest2 = ch.final();972 digest2 = man.final();
904 try ch.writeManifest();973 try man.writeManifest();
905 }974 }
906975
907 try testing.expectEqual(digest1, digest2);976 try testing.expectEqual(digest1, digest2);
src/Compilation.zig-22
...@@ -39,7 +39,6 @@ gpa: *Allocator,...@@ -39,7 +39,6 @@ gpa: *Allocator,
39arena_state: std.heap.ArenaAllocator.State,39arena_state: std.heap.ArenaAllocator.State,
40bin_file: *link.File,40bin_file: *link.File,
41c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},41c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
42c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{},
43stage1_lock: ?Cache.Lock = null,42stage1_lock: ?Cache.Lock = null,
44stage1_cache_manifest: *Cache.Manifest = undefined,43stage1_cache_manifest: *Cache.Manifest = undefined,
4544
...@@ -1570,7 +1569,6 @@ pub fn destroy(self: *Compilation) void {...@@ -1570,7 +1569,6 @@ pub fn destroy(self: *Compilation) void {
1570 key.destroy(gpa);1569 key.destroy(gpa);
1571 }1570 }
1572 self.c_object_table.deinit(gpa);1571 self.c_object_table.deinit(gpa);
1573 self.c_object_cache_digest_set.deinit(gpa);
15741572
1575 for (self.failed_c_objects.values()) |value| {1573 for (self.failed_c_objects.values()) |value| {
1576 value.destroy(gpa);1574 value.destroy(gpa);
...@@ -1607,7 +1605,6 @@ pub fn update(self: *Compilation) !void {...@@ -1607,7 +1605,6 @@ pub fn update(self: *Compilation) !void {
1607 defer tracy.end();1605 defer tracy.end();
16081606
1609 self.clearMiscFailures();1607 self.clearMiscFailures();
1610 self.c_object_cache_digest_set.clearRetainingCapacity();
16111608
1612 // For compiling C objects, we rely on the cache hash system to avoid duplicating work.1609 // For compiling C objects, we rely on the cache hash system to avoid duplicating work.
1613 // Add a Job for each C object.1610 // Add a Job for each C object.
...@@ -2566,25 +2563,6 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P...@@ -2566,25 +2563,6 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
25662563
2567 try man.hashCSource(c_object.src);2564 try man.hashCSource(c_object.src);
25682565
2569 {
2570 const is_collision = blk: {
2571 const bin_digest = man.hash.peekBin();
2572
2573 const lock = comp.mutex.acquire();
2574 defer lock.release();
2575
2576 const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, bin_digest);
2577 break :blk gop.found_existing;
2578 };
2579 if (is_collision) {
2580 return comp.failCObj(
2581 c_object,
2582 "the same source file was already added to the same compilation with the same flags",
2583 .{},
2584 );
2585 }
2586 }
2587
2588 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);2566 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
2589 defer arena_allocator.deinit();2567 defer arena_allocator.deinit();
2590 const arena = &arena_allocator.allocator;2568 const arena = &arena_allocator.allocator;