authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-28 16:46:37-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-06-29 14:25:04-07:00
log06129d7e3d5a8d9449edc98510a6d4f7a171b27f
tree9aa7bddedea56cb33ace3e7fed78d7e92b06472a
parent488f68069ba0a81f46a5afe721b908cfd3c61f76

std: implement a cross platform file locking abstraction

This modifies the lock semantics from using AccessMode to using NtLockFile/NtUnlockFile. This is a breaking change.

6 files changed, 239 insertions(+), 100 deletions(-)

doc/langref.html.in+8
......@@ -10713,6 +10713,14 @@ fn readU32Be() u32 {}
1071310713 See the Zig Standard Library for more examples.
1071410714 </p>
1071510715 {#header_close#}
10716 {#header_open|Doc Comment Guidance#}
10717 <ul>
10718 <li>Omit any information that is redundant based on the name of the thing being documented.</li>
10719 <li>Duplicating information onto multiple similar functions is encouraged because it helps IDEs and other tools provide better help text.</li>
10720 <li>Use the word <strong>assume</strong> to indicate invariants that cause {#link|Undefined Behavior#} when violated.</li>
10721 <li>Use the word <strong>assert</strong> to indicate invariants that cause <em>safety-checked</em> {#link|Undefined Behavior#} when violated.</li>
10722 </ul>
10723 {#header_close#}
1071610724 {#header_close#}
1071710725 {#header_open|Source Encoding#}
1071810726 <p>Zig source code is encoded in UTF-8. An invalid UTF-8 byte sequence results in a compile error.</p>
lib/std/fs.zig+47-17
......@@ -883,24 +883,39 @@ pub const Dir = struct {
883883 /// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded.
884884 pub fn openFileW(self: Dir, sub_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File {
885885 const w = os.windows;
886 return @as(File, .{
887 .handle = try os.windows.OpenFile(sub_path_w, .{
886 const file: File = .{
887 .handle = try w.OpenFile(sub_path_w, .{
888888 .dir = self.fd,
889889 .access_mask = w.SYNCHRONIZE |
890890 (if (flags.read) @as(u32, w.GENERIC_READ) else 0) |
891891 (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0),
892 .share_access = switch (flags.lock) {
893 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
894 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
895 .Exclusive => w.FILE_SHARE_DELETE,
896 },
897 .share_access_nonblocking = flags.lock_nonblocking,
898892 .creation = w.FILE_OPEN,
899893 .io_mode = flags.intended_io_mode,
900894 }),
901895 .capable_io_mode = std.io.default_mode,
902896 .intended_io_mode = flags.intended_io_mode,
903 });
897 };
898 var io: w.IO_STATUS_BLOCK = undefined;
899 const range_off: w.LARGE_INTEGER = 0;
900 const range_len: w.LARGE_INTEGER = 1;
901 const exclusive = switch (flags.lock) {
902 .None => return file,
903 .Shared => false,
904 .Exclusive => true,
905 };
906 try w.LockFile(
907 file.handle,
908 null,
909 null,
910 null,
911 &io,
912 &range_off,
913 &range_len,
914 null,
915 @boolToInt(flags.lock_nonblocking),
916 @boolToInt(exclusive),
917 );
918 return file;
904919 }
905920
906921 /// Creates, opens, or overwrites a file with write access.
......@@ -1019,16 +1034,10 @@ pub const Dir = struct {
10191034 pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) File.OpenError!File {
10201035 const w = os.windows;
10211036 const read_flag = if (flags.read) @as(u32, w.GENERIC_READ) else 0;
1022 return @as(File, .{
1037 const file: File = .{
10231038 .handle = try os.windows.OpenFile(sub_path_w, .{
10241039 .dir = self.fd,
10251040 .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag,
1026 .share_access = switch (flags.lock) {
1027 .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1028 .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE,
1029 .Exclusive => w.FILE_SHARE_DELETE,
1030 },
1031 .share_access_nonblocking = flags.lock_nonblocking,
10321041 .creation = if (flags.exclusive)
10331042 @as(u32, w.FILE_CREATE)
10341043 else if (flags.truncate)
......@@ -1039,7 +1048,28 @@ pub const Dir = struct {
10391048 }),
10401049 .capable_io_mode = std.io.default_mode,
10411050 .intended_io_mode = flags.intended_io_mode,
1042 });
1051 };
1052 var io: w.IO_STATUS_BLOCK = undefined;
1053 const range_off: w.LARGE_INTEGER = 0;
1054 const range_len: w.LARGE_INTEGER = 1;
1055 const exclusive = switch (flags.lock) {
1056 .None => return file,
1057 .Shared => false,
1058 .Exclusive => true,
1059 };
1060 try w.LockFile(
1061 file.handle,
1062 null,
1063 null,
1064 null,
1065 &io,
1066 &range_off,
1067 &range_len,
1068 null,
1069 @boolToInt(flags.lock_nonblocking),
1070 @boolToInt(exclusive),
1071 );
1072 return file;
10431073 }
10441074
10451075 pub const openRead = @compileError("deprecated in favor of openFile");
lib/std/fs/file.zig+140-26
......@@ -830,29 +830,25 @@ pub const File = struct {
830830 return .{ .context = file };
831831 }
832832
833 pub const SetLockError = os.FlockError;
833 const range_off: windows.LARGE_INTEGER = 0;
834 const range_len: windows.LARGE_INTEGER = 1;
835
836 pub const LockError = error{
837 SystemResources,
838 } || os.UnexpectedError;
834839
835840 /// Blocks when an incompatible lock is held by another process.
836 /// `non_blocking` may be used to make a non-blocking request,
837 /// causing this function to possibly return `error.WouldBlock`.
838841 /// A process may hold only one type of lock (shared or exclusive) on
839842 /// a file. When a process terminates in any way, the lock is released.
843 ///
844 /// Assumes the file is unlocked.
845 ///
840846 /// TODO: integrate with async I/O
841 pub fn setLock(file: File, lock: Lock, non_blocking: bool) SetLockError!void {
847 pub fn lock(file: File, l: Lock) LockError!void {
842848 if (is_windows) {
843 const range_off: windows.LARGE_INTEGER = 0;
844 const range_len: windows.LARGE_INTEGER = 1;
845 const exclusive = switch (lock) {
846 .None => return windows.UnlockFile(
847 file.handle,
848 null,
849 &range_off,
850 &range_len,
851 null,
852 ) catch |err| switch (err) {
853 error.RangeNotLocked => return,
854 else => |e| return e,
855 },
849 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
850 const exclusive = switch (l) {
851 .None => return,
856852 .Shared => false,
857853 .Exclusive => true,
858854 };
......@@ -861,19 +857,137 @@ pub const File = struct {
861857 null,
862858 null,
863859 null,
860 &io_status_block,
861 &range_off,
862 &range_len,
864863 null,
864 windows.FALSE, // non-blocking=false
865 @boolToInt(exclusive),
866 ) catch |err| switch (err) {
867 error.WouldBlock => unreachable, // non-blocking=false
868 else => |e| return e,
869 };
870 } else {
871 return os.flock(file.handle, switch (l) {
872 .None => os.LOCK_UN,
873 .Shared => os.LOCK_SH,
874 .Exclusive => os.LOCK_EX,
875 }) catch |err| switch (err) {
876 error.WouldBlock => unreachable, // non-blocking=false
877 else => |e| return e,
878 };
879 }
880 }
881
882 /// Assumes the file is locked.
883 pub fn unlock(file: File) void {
884 if (is_windows) {
885 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
886 return windows.UnlockFile(
887 file.handle,
888 &io_status_block,
865889 &range_off,
866890 &range_len,
867891 null,
868 @boolToInt(non_blocking),
892 ) catch |err| switch (err) {
893 error.RangeNotLocked => unreachable, // Function assumes unlocked.
894 error.Unexpected => unreachable, // Resource deallocation must succeed.
895 };
896 } else {
897 return os.flock(file.handle, os.LOCK_UN) catch |err| switch (err) {
898 error.WouldBlock => unreachable, // unlocking can't block
899 error.SystemResources => unreachable, // We are deallocating resources.
900 error.Unexpected => unreachable, // Resource deallocation must succeed.
901 };
902 }
903 }
904
905 /// Attempts to obtain a lock, returning `true` if the lock is
906 /// obtained, and `false` if there was an existing incompatible lock held.
907 /// A process may hold only one type of lock (shared or exclusive) on
908 /// a file. When a process terminates in any way, the lock is released.
909 ///
910 /// Assumes the file is unlocked.
911 ///
912 /// TODO: integrate with async I/O
913 pub fn tryLock(file: File, l: Lock) LockError!bool {
914 if (is_windows) {
915 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
916 const exclusive = switch (l) {
917 .None => return,
918 .Shared => false,
919 .Exclusive => true,
920 };
921 windows.LockFile(
922 file.handle,
923 null,
924 null,
925 null,
926 &io_status_block,
927 &range_off,
928 &range_len,
929 null,
930 windows.TRUE, // non-blocking=true
869931 @boolToInt(exclusive),
870 );
871 }
872 const non_blocking_flag = if (non_blocking) os.LOCK_NB else @as(i32, 0);
873 return os.flock(file.handle, switch (lock) {
874 .None => os.LOCK_UN,
875 .Shared => os.LOCK_SH | non_blocking_flag,
876 .Exclusive => os.LOCK_EX | non_blocking_flag,
877 });
932 ) catch |err| switch (err) {
933 error.WouldBlock => return false,
934 else => |e| return e,
935 };
936 } else {
937 os.flock(file.handle, switch (l) {
938 .None => os.LOCK_UN,
939 .Shared => os.LOCK_SH | os.LOCK_NB,
940 .Exclusive => os.LOCK_EX | os.LOCK_NB,
941 }) catch |err| switch (err) {
942 error.WouldBlock => return false,
943 else => |e| return e,
944 };
945 }
946 return true;
947 }
948
949 /// Assumes the file is already locked in exclusive mode.
950 /// Atomically modifies the lock to be in shared mode, without releasing it.
951 ///
952 /// TODO: integrate with async I/O
953 pub fn downgradeLock(file: File) LockError!void {
954 if (is_windows) {
955 // On Windows it works like a semaphore + exclusivity flag. To implement this
956 // function, we first obtain another lock in shared mode. This changes the
957 // exclusivity flag, but increments the semaphore to 2. So we follow up with
958 // an NtUnlockFile which decrements the semaphore but does not modify the
959 // exclusivity flag.
960 var io_status_block: windows.IO_STATUS_BLOCK = undefined;
961 windows.LockFile(
962 file.handle,
963 null,
964 null,
965 null,
966 &io_status_block,
967 &range_off,
968 &range_len,
969 null,
970 windows.TRUE, // non-blocking=true
971 windows.FALSE, // exclusive=false
972 ) catch |err| switch (err) {
973 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
974 else => |e| return e,
975 };
976 return windows.UnlockFile(
977 file.handle,
978 &io_status_block,
979 &range_off,
980 &range_len,
981 null,
982 ) catch |err| switch (err) {
983 error.RangeNotLocked => unreachable, // File was not locked.
984 error.Unexpected => unreachable, // Resource deallocation must succeed.
985 };
986 } else {
987 return os.flock(file.handle, os.LOCK_SH | os.LOCK_NB) catch |err| switch (err) {
988 error.WouldBlock => unreachable, // File was not locked in exclusive mode.
989 else => |e| return e,
990 };
991 }
878992 }
879993};
lib/std/os/windows.zig+37-52
......@@ -49,7 +49,6 @@ pub const OpenFileOptions = struct {
4949 dir: ?HANDLE = null,
5050 sa: ?*SECURITY_ATTRIBUTES = null,
5151 share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE,
52 share_access_nonblocking: bool = false,
5352 creation: ULONG,
5453 io_mode: std.io.ModeOverride,
5554 /// If true, tries to open path as a directory.
......@@ -60,8 +59,6 @@ pub const OpenFileOptions = struct {
6059 follow_symlinks: bool = true,
6160};
6261
63/// TODO when share_access_nonblocking is false, this implementation uses
64/// untinterruptible sleep() to block. This is not the final iteration of the API.
6562pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE {
6663 if (mem.eql(u16, sub_path_w, &[_]u16{'.'}) and !options.open_dir) {
6764 return error.IsDir;
......@@ -94,53 +91,39 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN
9491 // If we're not following symlinks, we need to ensure we don't pass in any synchronization flags such as FILE_SYNCHRONOUS_IO_NONALERT.
9592 const flags: ULONG = if (options.follow_symlinks) file_or_dir_flag | blocking_flag else file_or_dir_flag | FILE_OPEN_REPARSE_POINT;
9693
97 var delay: usize = 1;
98 while (true) {
99 const rc = ntdll.NtCreateFile(
100 &result,
101 options.access_mask,
102 &attr,
103 &io,
104 null,
105 FILE_ATTRIBUTE_NORMAL,
106 options.share_access,
107 options.creation,
108 flags,
109 null,
110 0,
111 );
112 switch (rc) {
113 .SUCCESS => {
114 if (std.io.is_async and options.io_mode == .evented) {
115 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
116 }
117 return result;
118 },
119 .OBJECT_NAME_INVALID => unreachable,
120 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
121 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
122 .NO_MEDIA_IN_DEVICE => return error.NoDevice,
123 .INVALID_PARAMETER => unreachable,
124 .SHARING_VIOLATION => {
125 if (options.share_access_nonblocking) {
126 return error.WouldBlock;
127 }
128 // TODO sleep in a way that is interruptable
129 // TODO integrate with async I/O
130 std.time.sleep(delay);
131 if (delay < 1 * std.time.ns_per_s) {
132 delay *= 2;
133 }
134 continue;
135 },
136 .ACCESS_DENIED => return error.AccessDenied,
137 .PIPE_BUSY => return error.PipeBusy,
138 .OBJECT_PATH_SYNTAX_BAD => unreachable,
139 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
140 .FILE_IS_A_DIRECTORY => return error.IsDir,
141 .NOT_A_DIRECTORY => return error.NotDir,
142 else => return unexpectedStatus(rc),
143 }
94 const rc = ntdll.NtCreateFile(
95 &result,
96 options.access_mask,
97 &attr,
98 &io,
99 null,
100 FILE_ATTRIBUTE_NORMAL,
101 options.share_access,
102 options.creation,
103 flags,
104 null,
105 0,
106 );
107 switch (rc) {
108 .SUCCESS => {
109 if (std.io.is_async and options.io_mode == .evented) {
110 _ = CreateIoCompletionPort(result, std.event.Loop.instance.?.os_data.io_port, undefined, undefined) catch undefined;
111 }
112 return result;
113 },
114 .OBJECT_NAME_INVALID => unreachable,
115 .OBJECT_NAME_NOT_FOUND => return error.FileNotFound,
116 .OBJECT_PATH_NOT_FOUND => return error.FileNotFound,
117 .NO_MEDIA_IN_DEVICE => return error.NoDevice,
118 .INVALID_PARAMETER => unreachable,
119 .SHARING_VIOLATION => return error.AccessDenied,
120 .ACCESS_DENIED => return error.AccessDenied,
121 .PIPE_BUSY => return error.PipeBusy,
122 .OBJECT_PATH_SYNTAX_BAD => unreachable,
123 .OBJECT_NAME_COLLISION => return error.PathAlreadyExists,
124 .FILE_IS_A_DIRECTORY => return error.IsDir,
125 .NOT_A_DIRECTORY => return error.NotDir,
126 else => return unexpectedStatus(rc),
144127 }
145128}
146129
......@@ -1689,7 +1672,7 @@ pub fn LockFile(
16891672 Event: ?HANDLE,
16901673 ApcRoutine: ?*IO_APC_ROUTINE,
16911674 ApcContext: ?*c_void,
1692 IoStatusBlock: ?*IO_STATUS_BLOCK,
1675 IoStatusBlock: *IO_STATUS_BLOCK,
16931676 ByteOffset: *const LARGE_INTEGER,
16941677 Length: *const LARGE_INTEGER,
16951678 Key: ?*ULONG,
......@@ -1712,6 +1695,7 @@ pub fn LockFile(
17121695 .SUCCESS => return,
17131696 .INSUFFICIENT_RESOURCES => return error.SystemResources,
17141697 .LOCK_NOT_GRANTED => return error.WouldBlock,
1698 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
17151699 else => return unexpectedStatus(rc),
17161700 }
17171701}
......@@ -1722,7 +1706,7 @@ pub const UnlockFileError = error{
17221706
17231707pub fn UnlockFile(
17241708 FileHandle: HANDLE,
1725 IoStatusBlock: ?*IO_STATUS_BLOCK,
1709 IoStatusBlock: *IO_STATUS_BLOCK,
17261710 ByteOffset: *const LARGE_INTEGER,
17271711 Length: *const LARGE_INTEGER,
17281712 Key: ?*ULONG,
......@@ -1731,6 +1715,7 @@ pub fn UnlockFile(
17311715 switch (rc) {
17321716 .SUCCESS => return,
17331717 .RANGE_NOT_LOCKED => return error.RangeNotLocked,
1718 .ACCESS_VIOLATION => unreachable, // bad io_status_block pointer
17341719 else => return unexpectedStatus(rc),
17351720 }
17361721}
lib/std/os/windows/ntdll.zig+2-2
......@@ -145,7 +145,7 @@ pub extern "NtDll" fn NtLockFile(
145145 Event: ?HANDLE,
146146 ApcRoutine: ?*IO_APC_ROUTINE,
147147 ApcContext: ?*c_void,
148 IoStatusBlock: ?*IO_STATUS_BLOCK,
148 IoStatusBlock: *IO_STATUS_BLOCK,
149149 ByteOffset: *const LARGE_INTEGER,
150150 Length: *const LARGE_INTEGER,
151151 Key: ?*ULONG,
......@@ -155,7 +155,7 @@ pub extern "NtDll" fn NtLockFile(
155155
156156pub extern "NtDll" fn NtUnlockFile(
157157 FileHandle: HANDLE,
158 IoStatusBlock: ?*IO_STATUS_BLOCK,
158 IoStatusBlock: *IO_STATUS_BLOCK,
159159 ByteOffset: *const LARGE_INTEGER,
160160 Length: *const LARGE_INTEGER,
161161 Key: ?*ULONG,
src/Cache.zig+5-3
......@@ -670,15 +670,17 @@ pub const Manifest = struct {
670670 fn downgradeToSharedLock(self: *Manifest) !void {
671671 if (!self.have_exclusive_lock) return;
672672 const manifest_file = self.manifest_file.?;
673 try manifest_file.setLock(.Shared, false);
673 try manifest_file.downgradeLock();
674674 self.have_exclusive_lock = false;
675675 }
676676
677677 fn upgradeToExclusiveLock(self: *Manifest) !void {
678678 if (self.have_exclusive_lock) return;
679679 const manifest_file = self.manifest_file.?;
680 try manifest_file.setLock(.None, false);
681 try manifest_file.setLock(.Exclusive, false);
680 // Here we intentionally have a period where the lock is released, in case there are
681 // other processes holding a shared lock.
682 manifest_file.unlock();
683 try manifest_file.lock(.Exclusive);
682684 self.have_exclusive_lock = true;
683685 }
684686