| ... | ... | @@ -4,19 +4,27 @@ const root = @import("root"); |
| 4 | 4 | const assert = std.debug.assert; |
| 5 | 5 | const testing = std.testing; |
| 6 | 6 | const mem = std.mem; |
| 7 | | const AtomicRmwOp = builtin.AtomicRmwOp; |
| 8 | | const AtomicOrder = builtin.AtomicOrder; |
| 9 | 7 | const os = std.os; |
| 10 | 8 | const windows = os.windows; |
| 11 | 9 | const maxInt = std.math.maxInt; |
| 12 | 10 | const Thread = std.Thread; |
| 13 | 11 | |
| 12 | const is_windows = std.Target.current.os.tag == .windows; |
| 13 | |
| 14 | 14 | pub const Loop = struct { |
| 15 | 15 | next_tick_queue: std.atomic.Queue(anyframe), |
| 16 | 16 | os_data: OsData, |
| 17 | 17 | final_resume_node: ResumeNode, |
| 18 | 18 | pending_event_count: usize, |
| 19 | 19 | extra_threads: []*Thread, |
| 20 | /// TODO change this to a pool of configurable number of threads |
| 21 | /// and rename it to be not file-system-specific. it will become |
| 22 | /// a thread pool for turning non-CPU-bound blocking things into |
| 23 | /// async things. A fallback for any missing OS-specific API. |
| 24 | fs_thread: *Thread, |
| 25 | fs_queue: std.atomic.Queue(Request), |
| 26 | fs_end_request: Request.Node, |
| 27 | fs_thread_wakeup: std.ResetEvent, |
| 20 | 28 | |
| 21 | 29 | /// For resources that have the same lifetime as the `Loop`. |
| 22 | 30 | /// This is only used by `Loop` for the thread pool and associated resources. |
| ... | ... | @@ -143,7 +151,12 @@ pub const Loop = struct { |
| 143 | 151 | .handle = undefined, |
| 144 | 152 | .overlapped = ResumeNode.overlapped_init, |
| 145 | 153 | }, |
| 154 | .fs_end_request = .{ .data = .{ .msg = .end, .finish = .NoAction } }, |
| 155 | .fs_queue = std.atomic.Queue(Request).init(), |
| 156 | .fs_thread = undefined, |
| 157 | .fs_thread_wakeup = std.ResetEvent.init(), |
| 146 | 158 | }; |
| 159 | errdefer self.fs_thread_wakeup.deinit(); |
| 147 | 160 | errdefer self.arena.deinit(); |
| 148 | 161 | |
| 149 | 162 | // We need at least one of these in case the fs thread wants to use onNextTick |
| ... | ... | @@ -158,10 +171,19 @@ pub const Loop = struct { |
| 158 | 171 | |
| 159 | 172 | try self.initOsData(extra_thread_count); |
| 160 | 173 | errdefer self.deinitOsData(); |
| 174 | |
| 175 | if (!builtin.single_threaded) { |
| 176 | self.fs_thread = try Thread.spawn(self, posixFsRun); |
| 177 | } |
| 178 | errdefer if (!builtin.single_threaded) { |
| 179 | self.posixFsRequest(&self.fs_end_request); |
| 180 | self.fs_thread.wait(); |
| 181 | }; |
| 161 | 182 | } |
| 162 | 183 | |
| 163 | 184 | pub fn deinit(self: *Loop) void { |
| 164 | 185 | self.deinitOsData(); |
| 186 | self.fs_thread_wakeup.deinit(); |
| 165 | 187 | self.arena.deinit(); |
| 166 | 188 | self.* = undefined; |
| 167 | 189 | } |
| ... | ... | @@ -173,21 +195,10 @@ pub const Loop = struct { |
| 173 | 195 | const wakeup_bytes = [_]u8{0x1} ** 8; |
| 174 | 196 | |
| 175 | 197 | fn initOsData(self: *Loop, extra_thread_count: usize) InitOsDataError!void { |
| 176 | | switch (builtin.os.tag) { |
| 198 | noasync switch (builtin.os.tag) { |
| 177 | 199 | .linux => { |
| 178 | | self.os_data.fs_queue = std.atomic.Queue(Request).init(); |
| 179 | | self.os_data.fs_queue_item = 0; |
| 180 | | // we need another thread for the file system because Linux does not have an async |
| 181 | | // file system I/O API. |
| 182 | | self.os_data.fs_end_request = Request.Node{ |
| 183 | | .data = Request{ |
| 184 | | .msg = .end, |
| 185 | | .finish = .NoAction, |
| 186 | | }, |
| 187 | | }; |
| 188 | | |
| 189 | 200 | errdefer { |
| 190 | | while (self.available_eventfd_resume_nodes.pop()) |node| noasync os.close(node.data.eventfd); |
| 201 | while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); |
| 191 | 202 | } |
| 192 | 203 | for (self.eventfd_resume_nodes) |*eventfd_node| { |
| 193 | 204 | eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{ |
| ... | ... | @@ -206,10 +217,10 @@ pub const Loop = struct { |
| 206 | 217 | } |
| 207 | 218 | |
| 208 | 219 | self.os_data.epollfd = try os.epoll_create1(os.EPOLL_CLOEXEC); |
| 209 | | errdefer noasync os.close(self.os_data.epollfd); |
| 220 | errdefer os.close(self.os_data.epollfd); |
| 210 | 221 | |
| 211 | 222 | self.os_data.final_eventfd = try os.eventfd(0, os.EFD_CLOEXEC | os.EFD_NONBLOCK); |
| 212 | | errdefer noasync os.close(self.os_data.final_eventfd); |
| 223 | errdefer os.close(self.os_data.final_eventfd); |
| 213 | 224 | |
| 214 | 225 | self.os_data.final_eventfd_event = os.epoll_event{ |
| 215 | 226 | .events = os.EPOLLIN, |
| ... | ... | @@ -222,12 +233,6 @@ pub const Loop = struct { |
| 222 | 233 | &self.os_data.final_eventfd_event, |
| 223 | 234 | ); |
| 224 | 235 | |
| 225 | | self.os_data.fs_thread = try Thread.spawn(self, posixFsRun); |
| 226 | | errdefer { |
| 227 | | self.posixFsRequest(&self.os_data.fs_end_request); |
| 228 | | self.os_data.fs_thread.wait(); |
| 229 | | } |
| 230 | | |
| 231 | 236 | if (builtin.single_threaded) { |
| 232 | 237 | assert(extra_thread_count == 0); |
| 233 | 238 | return; |
| ... | ... | @@ -236,7 +241,7 @@ pub const Loop = struct { |
| 236 | 241 | var extra_thread_index: usize = 0; |
| 237 | 242 | errdefer { |
| 238 | 243 | // writing 8 bytes to an eventfd cannot fail |
| 239 | | const amt = noasync os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; |
| 244 | const amt = os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; |
| 240 | 245 | assert(amt == wakeup_bytes.len); |
| 241 | 246 | while (extra_thread_index != 0) { |
| 242 | 247 | extra_thread_index -= 1; |
| ... | ... | @@ -249,22 +254,7 @@ pub const Loop = struct { |
| 249 | 254 | }, |
| 250 | 255 | .macosx, .freebsd, .netbsd, .dragonfly => { |
| 251 | 256 | self.os_data.kqfd = try os.kqueue(); |
| 252 | | errdefer noasync os.close(self.os_data.kqfd); |
| 253 | | |
| 254 | | self.os_data.fs_kqfd = try os.kqueue(); |
| 255 | | errdefer noasync os.close(self.os_data.fs_kqfd); |
| 256 | | |
| 257 | | self.os_data.fs_queue = std.atomic.Queue(Request).init(); |
| 258 | | // we need another thread for the file system because Darwin does not have an async |
| 259 | | // file system I/O API. |
| 260 | | self.os_data.fs_end_request = Request.Node{ |
| 261 | | .prev = undefined, |
| 262 | | .next = undefined, |
| 263 | | .data = Request{ |
| 264 | | .msg = .end, |
| 265 | | .finish = .NoAction, |
| 266 | | }, |
| 267 | | }; |
| 257 | errdefer os.close(self.os_data.kqfd); |
| 268 | 258 | |
| 269 | 259 | const empty_kevs = &[0]os.Kevent{}; |
| 270 | 260 | |
| ... | ... | @@ -310,30 +300,6 @@ pub const Loop = struct { |
| 310 | 300 | self.os_data.final_kevent.flags = os.EV_ENABLE; |
| 311 | 301 | self.os_data.final_kevent.fflags = os.NOTE_TRIGGER; |
| 312 | 302 | |
| 313 | | self.os_data.fs_kevent_wake = os.Kevent{ |
| 314 | | .ident = 0, |
| 315 | | .filter = os.EVFILT_USER, |
| 316 | | .flags = os.EV_ADD | os.EV_ENABLE, |
| 317 | | .fflags = os.NOTE_TRIGGER, |
| 318 | | .data = 0, |
| 319 | | .udata = undefined, |
| 320 | | }; |
| 321 | | |
| 322 | | self.os_data.fs_kevent_wait = os.Kevent{ |
| 323 | | .ident = 0, |
| 324 | | .filter = os.EVFILT_USER, |
| 325 | | .flags = os.EV_ADD | os.EV_CLEAR, |
| 326 | | .fflags = 0, |
| 327 | | .data = 0, |
| 328 | | .udata = undefined, |
| 329 | | }; |
| 330 | | |
| 331 | | self.os_data.fs_thread = try Thread.spawn(self, posixFsRun); |
| 332 | | errdefer { |
| 333 | | self.posixFsRequest(&self.os_data.fs_end_request); |
| 334 | | self.os_data.fs_thread.wait(); |
| 335 | | } |
| 336 | | |
| 337 | 303 | if (builtin.single_threaded) { |
| 338 | 304 | assert(extra_thread_count == 0); |
| 339 | 305 | return; |
| ... | ... | @@ -401,25 +367,24 @@ pub const Loop = struct { |
| 401 | 367 | } |
| 402 | 368 | }, |
| 403 | 369 | else => {}, |
| 404 | | } |
| 370 | }; |
| 405 | 371 | } |
| 406 | 372 | |
| 407 | 373 | fn deinitOsData(self: *Loop) void { |
| 408 | | switch (builtin.os.tag) { |
| 374 | noasync switch (builtin.os.tag) { |
| 409 | 375 | .linux => { |
| 410 | | noasync os.close(self.os_data.final_eventfd); |
| 411 | | while (self.available_eventfd_resume_nodes.pop()) |node| noasync os.close(node.data.eventfd); |
| 412 | | noasync os.close(self.os_data.epollfd); |
| 376 | os.close(self.os_data.final_eventfd); |
| 377 | while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); |
| 378 | os.close(self.os_data.epollfd); |
| 413 | 379 | }, |
| 414 | 380 | .macosx, .freebsd, .netbsd, .dragonfly => { |
| 415 | | noasync os.close(self.os_data.kqfd); |
| 416 | | noasync os.close(self.os_data.fs_kqfd); |
| 381 | os.close(self.os_data.kqfd); |
| 417 | 382 | }, |
| 418 | 383 | .windows => { |
| 419 | 384 | windows.CloseHandle(self.os_data.io_port); |
| 420 | 385 | }, |
| 421 | 386 | else => {}, |
| 422 | | } |
| 387 | }; |
| 423 | 388 | } |
| 424 | 389 | |
| 425 | 390 | /// resume_node must live longer than the anyframe that it holds a reference to. |
| ... | ... | @@ -635,7 +600,7 @@ pub const Loop = struct { |
| 635 | 600 | .freebsd, |
| 636 | 601 | .netbsd, |
| 637 | 602 | .dragonfly, |
| 638 | | => self.os_data.fs_thread.wait(), |
| 603 | => self.fs_thread.wait(), |
| 639 | 604 | else => {}, |
| 640 | 605 | } |
| 641 | 606 | |
| ... | ... | @@ -672,23 +637,25 @@ pub const Loop = struct { |
| 672 | 637 | |
| 673 | 638 | /// call finishOneEvent when done |
| 674 | 639 | pub fn beginOneEvent(self: *Loop) void { |
| 675 | | _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); |
| 640 | _ = @atomicRmw(usize, &self.pending_event_count, .Add, 1, .SeqCst); |
| 676 | 641 | } |
| 677 | 642 | |
| 678 | 643 | pub fn finishOneEvent(self: *Loop) void { |
| 679 | | const prev = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst); |
| 680 | | if (prev == 1) { |
| 644 | noasync { |
| 645 | const prev = @atomicRmw(usize, &self.pending_event_count, .Sub, 1, .SeqCst); |
| 646 | if (prev != 1) return; |
| 647 | |
| 681 | 648 | // cause all the threads to stop |
| 649 | self.posixFsRequest(&self.fs_end_request); |
| 650 | |
| 682 | 651 | switch (builtin.os.tag) { |
| 683 | 652 | .linux => { |
| 684 | | self.posixFsRequest(&self.os_data.fs_end_request); |
| 685 | 653 | // writing 8 bytes to an eventfd cannot fail |
| 686 | | const amt = noasync os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; |
| 654 | const amt = os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; |
| 687 | 655 | assert(amt == wakeup_bytes.len); |
| 688 | 656 | return; |
| 689 | 657 | }, |
| 690 | 658 | .macosx, .freebsd, .netbsd, .dragonfly => { |
| 691 | | self.posixFsRequest(&self.os_data.fs_end_request); |
| 692 | 659 | const final_kevent = @as(*const [1]os.Kevent, &self.os_data.final_kevent); |
| 693 | 660 | const empty_kevs = &[0]os.Kevent{}; |
| 694 | 661 | // cannot fail because we already added it and this just enables it |
| ... | ... | @@ -1041,73 +1008,55 @@ pub const Loop = struct { |
| 1041 | 1008 | |
| 1042 | 1009 | fn posixFsRequest(self: *Loop, request_node: *Request.Node) void { |
| 1043 | 1010 | self.beginOneEvent(); // finished in posixFsRun after processing the msg |
| 1044 | | self.os_data.fs_queue.put(request_node); |
| 1045 | | switch (builtin.os.tag) { |
| 1046 | | .macosx, .freebsd, .netbsd, .dragonfly => { |
| 1047 | | const fs_kevs = @as(*const [1]os.Kevent, &self.os_data.fs_kevent_wake); |
| 1048 | | const empty_kevs = &[0]os.Kevent{}; |
| 1049 | | _ = os.kevent(self.os_data.fs_kqfd, fs_kevs, empty_kevs, null) catch unreachable; |
| 1050 | | }, |
| 1051 | | .linux => { |
| 1052 | | @atomicStore(i32, &self.os_data.fs_queue_item, 1, AtomicOrder.SeqCst); |
| 1053 | | const rc = os.linux.futex_wake(&self.os_data.fs_queue_item, os.linux.FUTEX_WAKE, 1); |
| 1054 | | switch (os.linux.getErrno(rc)) { |
| 1055 | | 0 => {}, |
| 1056 | | os.EINVAL => unreachable, |
| 1057 | | else => unreachable, |
| 1058 | | } |
| 1059 | | }, |
| 1060 | | else => @compileError("Unsupported OS"), |
| 1061 | | } |
| 1011 | self.fs_queue.put(request_node); |
| 1012 | self.fs_thread_wakeup.set(); |
| 1062 | 1013 | } |
| 1063 | 1014 | |
| 1064 | 1015 | fn posixFsCancel(self: *Loop, request_node: *Request.Node) void { |
| 1065 | | if (self.os_data.fs_queue.remove(request_node)) { |
| 1016 | if (self.fs_queue.remove(request_node)) { |
| 1066 | 1017 | self.finishOneEvent(); |
| 1067 | 1018 | } |
| 1068 | 1019 | } |
| 1069 | 1020 | |
| 1070 | | // TODO make this whole function noasync |
| 1071 | | // https://github.com/ziglang/zig/issues/3157 |
| 1072 | 1021 | fn posixFsRun(self: *Loop) void { |
| 1073 | | while (true) { |
| 1074 | | if (builtin.os.tag == .linux) { |
| 1075 | | @atomicStore(i32, &self.os_data.fs_queue_item, 0, .SeqCst); |
| 1076 | | } |
| 1077 | | while (self.os_data.fs_queue.get()) |node| { |
| 1022 | noasync while (true) { |
| 1023 | self.fs_thread_wakeup.reset(); |
| 1024 | while (self.fs_queue.get()) |node| { |
| 1078 | 1025 | switch (node.data.msg) { |
| 1079 | 1026 | .end => return, |
| 1080 | 1027 | .read => |*msg| { |
| 1081 | | msg.result = noasync os.read(msg.fd, msg.buf); |
| 1028 | msg.result = os.read(msg.fd, msg.buf); |
| 1082 | 1029 | }, |
| 1083 | 1030 | .readv => |*msg| { |
| 1084 | | msg.result = noasync os.readv(msg.fd, msg.iov); |
| 1031 | msg.result = os.readv(msg.fd, msg.iov); |
| 1085 | 1032 | }, |
| 1086 | 1033 | .write => |*msg| { |
| 1087 | | msg.result = noasync os.write(msg.fd, msg.bytes); |
| 1034 | msg.result = os.write(msg.fd, msg.bytes); |
| 1088 | 1035 | }, |
| 1089 | 1036 | .writev => |*msg| { |
| 1090 | | msg.result = noasync os.writev(msg.fd, msg.iov); |
| 1037 | msg.result = os.writev(msg.fd, msg.iov); |
| 1091 | 1038 | }, |
| 1092 | 1039 | .pwritev => |*msg| { |
| 1093 | | msg.result = noasync os.pwritev(msg.fd, msg.iov, msg.offset); |
| 1040 | msg.result = os.pwritev(msg.fd, msg.iov, msg.offset); |
| 1094 | 1041 | }, |
| 1095 | 1042 | .pread => |*msg| { |
| 1096 | | msg.result = noasync os.pread(msg.fd, msg.buf, msg.offset); |
| 1043 | msg.result = os.pread(msg.fd, msg.buf, msg.offset); |
| 1097 | 1044 | }, |
| 1098 | 1045 | .preadv => |*msg| { |
| 1099 | | msg.result = noasync os.preadv(msg.fd, msg.iov, msg.offset); |
| 1046 | msg.result = os.preadv(msg.fd, msg.iov, msg.offset); |
| 1100 | 1047 | }, |
| 1101 | 1048 | .open => |*msg| { |
| 1102 | | msg.result = noasync os.openZ(msg.path, msg.flags, msg.mode); |
| 1049 | if (is_windows) unreachable; // TODO |
| 1050 | msg.result = os.openZ(msg.path, msg.flags, msg.mode); |
| 1103 | 1051 | }, |
| 1104 | 1052 | .openat => |*msg| { |
| 1105 | | msg.result = noasync os.openatZ(msg.fd, msg.path, msg.flags, msg.mode); |
| 1053 | if (is_windows) unreachable; // TODO |
| 1054 | msg.result = os.openatZ(msg.fd, msg.path, msg.flags, msg.mode); |
| 1106 | 1055 | }, |
| 1107 | 1056 | .faccessat => |*msg| { |
| 1108 | | msg.result = noasync os.faccessatZ(msg.dirfd, msg.path, msg.mode, msg.flags); |
| 1057 | msg.result = os.faccessatZ(msg.dirfd, msg.path, msg.mode, msg.flags); |
| 1109 | 1058 | }, |
| 1110 | | .close => |*msg| noasync os.close(msg.fd), |
| 1059 | .close => |*msg| os.close(msg.fd), |
| 1111 | 1060 | } |
| 1112 | 1061 | switch (node.data.finish) { |
| 1113 | 1062 | .TickNode => |*tick_node| self.onNextTick(tick_node), |
| ... | ... | @@ -1115,22 +1064,8 @@ pub const Loop = struct { |
| 1115 | 1064 | } |
| 1116 | 1065 | self.finishOneEvent(); |
| 1117 | 1066 | } |
| 1118 | | switch (builtin.os.tag) { |
| 1119 | | .linux => { |
| 1120 | | const rc = os.linux.futex_wait(&self.os_data.fs_queue_item, os.linux.FUTEX_WAIT, 0, null); |
| 1121 | | switch (os.linux.getErrno(rc)) { |
| 1122 | | 0, os.EINTR, os.EAGAIN => continue, |
| 1123 | | else => unreachable, |
| 1124 | | } |
| 1125 | | }, |
| 1126 | | .macosx, .freebsd, .netbsd, .dragonfly => { |
| 1127 | | const fs_kevs = @as(*const [1]os.Kevent, &self.os_data.fs_kevent_wait); |
| 1128 | | var out_kevs: [1]os.Kevent = undefined; |
| 1129 | | _ = os.kevent(self.os_data.fs_kqfd, fs_kevs, out_kevs[0..], null) catch unreachable; |
| 1130 | | }, |
| 1131 | | else => @compileError("Unsupported OS"), |
| 1132 | | } |
| 1133 | | } |
| 1067 | self.fs_thread_wakeup.wait(); |
| 1068 | }; |
| 1134 | 1069 | } |
| 1135 | 1070 | |
| 1136 | 1071 | const OsData = switch (builtin.os.tag) { |
| ... | ... | @@ -1146,22 +1081,12 @@ pub const Loop = struct { |
| 1146 | 1081 | const KEventData = struct { |
| 1147 | 1082 | kqfd: i32, |
| 1148 | 1083 | final_kevent: os.Kevent, |
| 1149 | | fs_kevent_wake: os.Kevent, |
| 1150 | | fs_kevent_wait: os.Kevent, |
| 1151 | | fs_thread: *Thread, |
| 1152 | | fs_kqfd: i32, |
| 1153 | | fs_queue: std.atomic.Queue(Request), |
| 1154 | | fs_end_request: Request.Node, |
| 1155 | 1084 | }; |
| 1156 | 1085 | |
| 1157 | 1086 | const LinuxOsData = struct { |
| 1158 | 1087 | epollfd: i32, |
| 1159 | 1088 | final_eventfd: i32, |
| 1160 | 1089 | final_eventfd_event: os.linux.epoll_event, |
| 1161 | | fs_thread: *Thread, |
| 1162 | | fs_queue_item: i32, |
| 1163 | | fs_queue: std.atomic.Queue(Request), |
| 1164 | | fs_end_request: Request.Node, |
| 1165 | 1090 | }; |
| 1166 | 1091 | |
| 1167 | 1092 | pub const Request = struct { |
| ... | ... | @@ -1302,11 +1227,11 @@ test "std.event.Loop - basic" { |
| 1302 | 1227 | loop.run(); |
| 1303 | 1228 | } |
| 1304 | 1229 | |
| 1305 | | async fn testEventLoop() i32 { |
| 1230 | fn testEventLoop() i32 { |
| 1306 | 1231 | return 1234; |
| 1307 | 1232 | } |
| 1308 | 1233 | |
| 1309 | | async fn testEventLoop2(h: anyframe->i32, did_it: *bool) void { |
| 1234 | fn testEventLoop2(h: anyframe->i32, did_it: *bool) void { |
| 1310 | 1235 | const value = await h; |
| 1311 | 1236 | testing.expect(value == 1234); |
| 1312 | 1237 | did_it.* = true; |