authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-08 02:09:51-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-11-08 02:09:51-05:00
log6d28b28ccc689e6bf8849b1d39e969e8da760999
tree0226ba98b0c3a1cd5c5a441819902a1ca8df16e9
parent9b0536e6f43ba916b2b488377f1e87d0ecf4ccf9
parent2723ffc2b2d837aec8185f86814a6e6aba45c34d
signaturelock-open Commit is signed but in an unrecognized format.

Merge branch 'kprotty-adaptive_lock'

28 files changed, 521 insertions(+), 213 deletions(-)

lib/std/c.zig+23-1
......@@ -8,9 +8,16 @@ pub usingnamespace switch (builtin.os) {
88 .linux => @import("c/linux.zig"),
99 .windows => @import("c/windows.zig"),
1010 .macosx, .ios, .tvos, .watchos => @import("c/darwin.zig"),
11 .freebsd => @import("c/freebsd.zig"),
11 .freebsd, .kfreebsd => @import("c/freebsd.zig"),
1212 .netbsd => @import("c/netbsd.zig"),
1313 .dragonfly => @import("c/dragonfly.zig"),
14 .openbsd => @import("c/openbsd.zig"),
15 .haiku => @import("c/haiku.zig"),
16 .hermit => @import("c/hermit.zig"),
17 .solaris => @import("c/solaris.zig"),
18 .fuchsia => @import("c/fuchsia.zig"),
19 .minix => @import("c/minix.zig"),
20 .emscripten => @import("c/emscripten.zig"),
1421 else => struct {},
1522};
1623
......@@ -203,3 +210,18 @@ pub extern "c" fn dn_expand(
203210 exp_dn: [*]u8,
204211 length: c_int,
205212) c_int;
213
214pub extern "c" fn sched_yield() c_int;
215
216pub const PTHREAD_MUTEX_INITIALIZER = pthread_mutex_t{};
217pub extern "c" fn pthread_mutex_lock(mutex: *pthread_mutex_t) c_int;
218pub extern "c" fn pthread_mutex_unlock(mutex: *pthread_mutex_t) c_int;
219pub extern "c" fn pthread_mutex_destroy(mutex: *pthread_mutex_t) c_int;
220
221pub const PTHREAD_COND_INITIALIZER = pthread_cond_t{};
222pub extern "c" fn pthread_cond_wait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t) c_int;
223pub extern "c" fn pthread_cond_signal(cond: *pthread_cond_t) c_int;
224pub extern "c" fn pthread_cond_destroy(cond: *pthread_cond_t) c_int;
225
226pub const pthread_t = *@OpaqueType();
227pub const FILE = @OpaqueType();
lib/std/c/darwin.zig+16
......@@ -112,3 +112,19 @@ pub const EAI_PROTOCOL = 13;
112112/// argument buffer overflow
113113pub const EAI_OVERFLOW = 14;
114114pub const EAI_MAX = 15;
115
116pub const pthread_mutex_t = extern struct {
117 __sig: c_long = 0x32AAABA7,
118 __opaque: [__PTHREAD_MUTEX_SIZE__]u8 = [_]u8{0} ** __PTHREAD_MUTEX_SIZE__,
119};
120pub const pthread_cond_t = extern struct {
121 __sig: c_long = 0x3CB0B1BB,
122 __opaque: [__PTHREAD_COND_SIZE__]u8 = [_]u8{0} ** __PTHREAD_COND_SIZE__,
123};
124const __PTHREAD_MUTEX_SIZE__ = if (@sizeOf(usize) == 8) 56 else 40;
125const __PTHREAD_COND_SIZE__ = if (@sizeOf(usize) == 8) 40 else 24;
126
127pub const pthread_attr_t = extern struct {
128 __sig: c_long,
129 __opaque: [56]u8,
130};
lib/std/c/dragonfly.zig+12-1
......@@ -1,6 +1,5 @@
11const std = @import("../std.zig");
22usingnamespace std.c;
3
43extern "c" threadlocal var errno: c_int;
54pub fn _errno() *c_int {
65 return &errno;
......@@ -12,3 +11,15 @@ pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize
1211
1312pub const dl_iterate_phdr_callback = extern fn (info: *dl_phdr_info, size: usize, data: ?*c_void) c_int;
1413pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int;
14
15pub const pthread_mutex_t = extern struct {
16 inner: ?*c_void = null,
17};
18pub const pthread_cond_t = extern struct {
19 inner: ?*c_void = null,
20};
21
22pub const pthread_attr_t = extern struct { // copied from freebsd
23 __size: [56]u8,
24 __align: c_long,
25};
lib/std/c/emscripten.zig created+8
......@@ -0,0 +1,8 @@
1pub const pthread_mutex_t = extern struct {
2 size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(4) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T,
3};
4pub const pthread_cond_t = extern struct {
5 size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T,
6};
7const __SIZEOF_PTHREAD_COND_T = 48;
8const __SIZEOF_PTHREAD_MUTEX_T = 28;
lib/std/c/freebsd.zig+12
......@@ -10,3 +10,15 @@ pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize
1010
1111pub const dl_iterate_phdr_callback = extern fn (info: *dl_phdr_info, size: usize, data: ?*c_void) c_int;
1212pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int;
13
14pub const pthread_mutex_t = extern struct {
15 inner: ?*c_void = null,
16};
17pub const pthread_cond_t = extern struct {
18 inner: ?*c_void = null,
19};
20
21pub const pthread_attr_t = extern struct {
22 __size: [56]u8,
23 __align: c_long,
24};
lib/std/c/fuchsia.zig created+8
......@@ -0,0 +1,8 @@
1pub const pthread_mutex_t = extern struct {
2 size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T,
3};
4pub const pthread_cond_t = extern struct {
5 size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T,
6};
7const __SIZEOF_PTHREAD_COND_T = 48;
8const __SIZEOF_PTHREAD_MUTEX_T = 40;
lib/std/c/haiku.zig created+14
......@@ -0,0 +1,14 @@
1pub const pthread_mutex_t = extern struct {
2 flags: u32 = 0,
3 lock: i32 = 0,
4 unused: i32 = -42,
5 owner: i32 = -1,
6 owner_count: i32 = 0,
7};
8pub const pthread_cond_t = extern struct {
9 flags: u32 = 0,
10 unused: i32 = -42,
11 mutex: ?*c_void = null,
12 waiter_count: i32 = 0,
13 lock: i32 = 0,
14};
lib/std/c/hermit.zig created+6
......@@ -0,0 +1,6 @@
1pub const pthread_mutex_t = extern struct {
2 inner: usize = ~usize(0),
3};
4pub const pthread_cond_t = extern struct {
5 inner: usize = ~usize(0),
6};
lib/std/c/linux.zig+23
......@@ -75,3 +75,26 @@ pub const dl_iterate_phdr_callback = extern fn (info: *dl_phdr_info, size: usize
7575pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int;
7676
7777pub extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int;
78
79pub const pthread_attr_t = extern struct {
80 __size: [56]u8,
81 __align: c_long,
82};
83
84pub const pthread_mutex_t = extern struct {
85 size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T,
86};
87pub const pthread_cond_t = extern struct {
88 size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T,
89};
90const __SIZEOF_PTHREAD_COND_T = 48;
91const __SIZEOF_PTHREAD_MUTEX_T = if (builtin.os == .fuchsia) 40 else switch (builtin.abi) {
92 .musl, .musleabi, .musleabihf => if (@sizeOf(usize) == 8) 40 else 24,
93 .gnu, .gnuabin32, .gnuabi64, .gnueabi, .gnueabihf, .gnux32 => switch (builtin.arch) {
94 .aarch64 => 48,
95 .x86_64 => if (builtin.abi == .gnux32) 40 else 32,
96 .mips64, .powerpc64, .powerpc64le, .sparcv9 => 40,
97 else => if (@sizeOf(usize) == 8) 40 else 24,
98 },
99 else => unreachable,
100};
lib/std/c/minix.zig created+18
......@@ -0,0 +1,18 @@
1const builtin = @import("builtin");
2pub const pthread_mutex_t = extern struct {
3 size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T,
4};
5pub const pthread_cond_t = extern struct {
6 size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T,
7};
8const __SIZEOF_PTHREAD_COND_T = 48;
9const __SIZEOF_PTHREAD_MUTEX_T = switch (builtin.abi) {
10 .musl, .musleabi, .musleabihf => if (@sizeOf(usize) == 8) 40 else 24,
11 .gnu, .gnuabin32, .gnuabi64, .gnueabi, .gnueabihf, .gnux32 => switch (builtin.arch) {
12 .aarch64 => 48,
13 .x86_64 => if (builtin.abi == .gnux32) 40 else 32,
14 .mips64, .powerpc64, .powerpc64le, .sparcv9 => 40,
15 else => if (@sizeOf(usize) == 8) 40 else 24,
16 },
17 else => unreachable,
18};
lib/std/c/netbsd.zig+29
......@@ -6,3 +6,32 @@ pub const _errno = __errno;
66
77pub extern "c" fn getdents(fd: c_int, buf_ptr: [*]u8, nbytes: usize) usize;
88pub extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int;
9
10pub const pthread_mutex_t = extern struct {
11 ptm_magic: c_uint = 0x33330003,
12 ptm_errorcheck: padded_spin_t = 0,
13 ptm_unused: padded_spin_t = 0,
14 ptm_owner: usize = 0,
15 ptm_waiters: ?*u8 = null,
16 ptm_recursed: c_uint = 0,
17 ptm_spare2: ?*c_void = null,
18};
19pub const pthread_cond_t = extern struct {
20 ptc_magic: c_uint = 0x55550005,
21 ptc_lock: pthread_spin_t = 0,
22 ptc_waiters_first: ?*u8 = null,
23 ptc_waiters_last: ?*u8 = null,
24 ptc_mutex: ?*pthread_mutex_t = null,
25 ptc_private: ?*c_void = null,
26};
27const pthread_spin_t = if (builtin.arch == .arm or .arch == .powerpc) c_int else u8;
28const padded_spin_t = switch (builtin.arch) {
29 .sparc, .sparcel, .sparcv9, .i386, .x86_64, .le64 => u32,
30 else => spin_t,
31};
32
33pub const pthread_attr_t = extern struct {
34 pta_magic: u32,
35 pta_flags: c_int,
36 pta_private: *c_void,
37};
lib/std/c/openbsd.zig created+6
......@@ -0,0 +1,6 @@
1pub const pthread_mutex_t = extern struct {
2 inner: ?*c_void = null,
3};
4pub const pthread_cond_t = extern struct {
5 inner: ?*c_void = null,
6};
lib/std/c/solaris.zig created+15
......@@ -0,0 +1,15 @@
1pub const pthread_mutex_t = extern struct {
2 __pthread_mutex_flag1: u16 = 0,
3 __pthread_mutex_flag2: u8 = 0,
4 __pthread_mutex_ceiling: u8 = 0,
5 __pthread_mutex_type: u16 = 0,
6 __pthread_mutex_magic: u16 = 0x4d58,
7 __pthread_mutex_lock: u64 = 0,
8 __pthread_mutex_data: u64 = 0,
9};
10pub const pthread_cond_t = extern struct {
11 __pthread_cond_flag: u32 = 0,
12 __pthread_cond_type: u16 = 0,
13 __pthread_cond_magic: u16 = 0x4356,
14 __pthread_cond_data: u64 = 0,
15};
lib/std/mutex.zig+62-69
......@@ -1,19 +1,13 @@
11const std = @import("std.zig");
22const builtin = @import("builtin");
3const AtomicOrder = builtin.AtomicOrder;
4const AtomicRmwOp = builtin.AtomicRmwOp;
53const testing = std.testing;
64const SpinLock = std.SpinLock;
7const linux = std.os.linux;
8const windows = std.os.windows;
5const ThreadParker = std.ThreadParker;
96
107/// Lock may be held only once. If the same thread
118/// tries to acquire the same mutex twice, it deadlocks.
12/// This type must be initialized at runtime, and then deinitialized when no
13/// longer needed, to free resources.
14/// If you need static initialization, use std.StaticallyInitializedMutex.
15/// The Linux implementation is based on mutex3 from
16/// https://www.akkadia.org/drepper/futex.pdf
9/// This type supports static initialization and is based off of Golang 1.13 runtime.lock_futex:
10/// https://github.com/golang/go/blob/master/src/runtime/lock_futex.go
1711/// When an application is built in single threaded release mode, all the functions are
1812/// no-ops. In single threaded debug mode, there is deadlock detection.
1913pub const Mutex = if (builtin.single_threaded)
......@@ -43,84 +37,83 @@ pub const Mutex = if (builtin.single_threaded)
4337 return Held{ .mutex = self };
4438 }
4539 }
46else switch (builtin.os) {
47 builtin.Os.linux => struct {
48 /// 0: unlocked
49 /// 1: locked, no waiters
50 /// 2: locked, one or more waiters
51 lock: i32,
40else
41 struct {
42 state: u32, // TODO: make this an enum
43 parker: ThreadParker,
5244
53 pub const Held = struct {
54 mutex: *Mutex,
45 const Unlocked = 0;
46 const Sleeping = 1;
47 const Locked = 2;
5548
56 pub fn release(self: Held) void {
57 const c = @atomicRmw(i32, &self.mutex.lock, AtomicRmwOp.Sub, 1, AtomicOrder.Release);
58 if (c != 1) {
59 _ = @atomicRmw(i32, &self.mutex.lock, AtomicRmwOp.Xchg, 0, AtomicOrder.Release);
60 const rc = linux.futex_wake(&self.mutex.lock, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, 1);
61 switch (linux.getErrno(rc)) {
62 0 => {},
63 linux.EINVAL => unreachable,
64 else => unreachable,
65 }
66 }
67 }
68 };
49 /// number of iterations to spin yielding the cpu
50 const SPIN_CPU = 4;
51
52 /// number of iterations to perform in the cpu yield loop
53 const SPIN_CPU_COUNT = 30;
54
55 /// number of iterations to spin yielding the thread
56 const SPIN_THREAD = 1;
6957
7058 pub fn init() Mutex {
71 return Mutex{ .lock = 0 };
59 return Mutex{
60 .state = Unlocked,
61 .parker = ThreadParker.init(),
62 };
7263 }
7364
74 pub fn deinit(self: *Mutex) void {}
75
76 pub fn acquire(self: *Mutex) Held {
77 var c = @cmpxchgWeak(i32, &self.lock, 0, 1, AtomicOrder.Acquire, AtomicOrder.Monotonic) orelse
78 return Held{ .mutex = self };
79 if (c != 2)
80 c = @atomicRmw(i32, &self.lock, AtomicRmwOp.Xchg, 2, AtomicOrder.Acquire);
81 while (c != 0) {
82 const rc = linux.futex_wait(&self.lock, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, 2, null);
83 switch (linux.getErrno(rc)) {
84 0, linux.EINTR, linux.EAGAIN => {},
85 linux.EINVAL => unreachable,
86 else => unreachable,
87 }
88 c = @atomicRmw(i32, &self.lock, AtomicRmwOp.Xchg, 2, AtomicOrder.Acquire);
89 }
90 return Held{ .mutex = self };
65 pub fn deinit(self: *Mutex) void {
66 self.parker.deinit();
9167 }
92 },
93 // TODO once https://github.com/ziglang/zig/issues/287 (copy elision) is solved, we can make a
94 // better implementation of this. The problem is we need the init() function to have access to
95 // the address of the CRITICAL_SECTION, and then have it not move.
96 builtin.Os.windows => std.StaticallyInitializedMutex,
97 else => struct {
98 /// TODO better implementation than spin lock.
99 /// When changing this, one must also change the corresponding
100 /// std.StaticallyInitializedMutex code, since it aliases this type,
101 /// under the assumption that it works both statically and at runtime.
102 lock: SpinLock,
10368
10469 pub const Held = struct {
10570 mutex: *Mutex,
10671
10772 pub fn release(self: Held) void {
108 SpinLock.Held.release(SpinLock.Held{ .spinlock = &self.mutex.lock });
73 switch (@atomicRmw(u32, &self.mutex.state, .Xchg, Unlocked, .Release)) {
74 Locked => {},
75 Sleeping => self.mutex.parker.unpark(&self.mutex.state),
76 Unlocked => unreachable, // unlocking an unlocked mutex
77 else => unreachable, // should never be anything else
78 }
10979 }
11080 };
11181
112 pub fn init() Mutex {
113 return Mutex{ .lock = SpinLock.init() };
114 }
82 pub fn acquire(self: *Mutex) Held {
83 // Try and speculatively grab the lock.
84 // If it fails, the state is either Locked or Sleeping
85 // depending on if theres a thread stuck sleeping below.
86 var state = @atomicRmw(u32, &self.state, .Xchg, Locked, .Acquire);
87 if (state == Unlocked)
88 return Held{ .mutex = self };
11589
116 pub fn deinit(self: *Mutex) void {}
90 while (true) {
91 // try and acquire the lock using cpu spinning on failure
92 var spin: usize = 0;
93 while (spin < SPIN_CPU) : (spin += 1) {
94 var value = @atomicLoad(u32, &self.state, .Monotonic);
95 while (value == Unlocked)
96 value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self };
97 SpinLock.yield(SPIN_CPU_COUNT);
98 }
11799
118 pub fn acquire(self: *Mutex) Held {
119 _ = self.lock.acquire();
120 return Held{ .mutex = self };
100 // try and acquire the lock using thread rescheduling on failure
101 spin = 0;
102 while (spin < SPIN_THREAD) : (spin += 1) {
103 var value = @atomicLoad(u32, &self.state, .Monotonic);
104 while (value == Unlocked)
105 value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self };
106 std.os.sched_yield() catch std.time.sleep(1);
107 }
108
109 // failed to acquire the lock, go to sleep until woken up by `Held.release()`
110 if (@atomicRmw(u32, &self.state, .Xchg, Sleeping, .Acquire) == Unlocked)
111 return Held{ .mutex = self };
112 state = Sleeping;
113 self.parker.park(&self.state, Sleeping);
114 }
121115 }
122 },
123};
116 };
124117
125118const TestContext = struct {
126119 mutex: *Mutex,
lib/std/os.zig+19
......@@ -3171,3 +3171,22 @@ pub fn dn_expand(
31713171 }
31723172 return error.InvalidDnsPacket;
31733173}
3174
3175pub const SchedYieldError = error{
3176 /// The system is not configured to allow yielding
3177 SystemCannotYield,
3178};
3179
3180pub fn sched_yield() SchedYieldError!void {
3181 if (builtin.os == .windows) {
3182 // The return value has to do with how many other threads there are; it is not
3183 // an error condition on Windows.
3184 _ = windows.kernel32.SwitchToThread();
3185 return;
3186 }
3187 switch (errno(system.sched_yield())) {
3188 0 => return,
3189 ENOSYS => return error.SystemCannotYield,
3190 else => return error.SystemCannotYield,
3191 }
3192}
lib/std/os/bits.zig-3
......@@ -14,9 +14,6 @@ pub usingnamespace switch (builtin.os) {
1414 else => struct {},
1515};
1616
17pub const pthread_t = *@OpaqueType();
18pub const FILE = @OpaqueType();
19
2017pub const iovec = extern struct {
2118 iov_base: [*]u8,
2219 iov_len: usize,
lib/std/os/bits/darwin.zig-5
......@@ -133,11 +133,6 @@ pub const dirent = extern struct {
133133 }
134134};
135135
136pub const pthread_attr_t = extern struct {
137 __sig: c_long,
138 __opaque: [56]u8,
139};
140
141136/// Renamed from `kevent` to `Kevent` to avoid conflict with function name.
142137pub const Kevent = extern struct {
143138 ident: usize,
lib/std/os/bits/dragonfly.zig+3-8
......@@ -360,11 +360,6 @@ pub const Kevent = extern struct {
360360 udata: usize,
361361};
362362
363pub const pthread_attr_t = extern struct { // copied from freebsd
364 __size: [56]u8,
365 __align: c_long,
366};
367
368363pub const EVFILT_FS = -10;
369364pub const EVFILT_USER = -9;
370365pub const EVFILT_EXCEPT = -8;
......@@ -515,13 +510,13 @@ pub const sigset_t = extern struct {
515510pub const sig_atomic_t = c_int;
516511pub const Sigaction = extern struct {
517512 __sigaction_u: extern union {
518 __sa_handler: ?extern fn(c_int) void,
519 __sa_sigaction: ?extern fn(c_int, [*c]siginfo_t, ?*c_void) void,
513 __sa_handler: ?extern fn (c_int) void,
514 __sa_sigaction: ?extern fn (c_int, [*c]siginfo_t, ?*c_void) void,
520515 },
521516 sa_flags: c_int,
522517 sa_mask: sigset_t,
523518};
524pub const sig_t = [*c]extern fn(c_int) void;
519pub const sig_t = [*c]extern fn (c_int) void;
525520
526521pub const sigvec = extern struct {
527522 sv_handler: [*c]__sighandler_t,
lib/std/os/bits/freebsd.zig-5
......@@ -15,11 +15,6 @@ pub const Kevent = extern struct {
1515 // TODO ext
1616};
1717
18pub const pthread_attr_t = extern struct {
19 __size: [56]u8,
20 __align: c_long,
21};
22
2318pub const dl_phdr_info = extern struct {
2419 dlpi_addr: usize,
2520 dlpi_name: ?[*]const u8,
lib/std/os/bits/linux.zig-5
......@@ -1000,11 +1000,6 @@ pub const dl_phdr_info = extern struct {
10001000 dlpi_phnum: u16,
10011001};
10021002
1003pub const pthread_attr_t = extern struct {
1004 __size: [56]u8,
1005 __align: c_long,
1006};
1007
10081003pub const CPU_SETSIZE = 128;
10091004pub const cpu_set_t = [CPU_SETSIZE / @sizeOf(usize)]usize;
10101005pub const cpu_count_t = @IntType(false, std.math.log2(CPU_SETSIZE * 8));
lib/std/os/bits/netbsd.zig-6
......@@ -14,12 +14,6 @@ pub const Kevent = extern struct {
1414 udata: usize,
1515};
1616
17pub const pthread_attr_t = extern struct {
18 pta_magic: u32,
19 pta_flags: c_int,
20 pta_private: *c_void,
21};
22
2317pub const dl_phdr_info = extern struct {
2418 dlpi_addr: usize,
2519 dlpi_name: ?[*]const u8,
lib/std/os/linux.zig+4
......@@ -954,6 +954,10 @@ pub fn fremovexattr(fd: usize, name: [*]const u8) usize {
954954 return syscall2(SYS_fremovexattr, fd, @ptrToInt(name));
955955}
956956
957pub fn sched_yield() usize {
958 return syscall0(SYS_sched_yield);
959}
960
957961pub fn sched_getaffinity(pid: i32, size: usize, set: *cpu_set_t) usize {
958962 const rc = syscall3(SYS_sched_getaffinity, @bitCast(usize, isize(pid)), size, @ptrToInt(set));
959963 if (@bitCast(isize, rc) < 0) return rc;
lib/std/os/windows/kernel32.zig+2
......@@ -184,6 +184,8 @@ pub extern "kernel32" stdcallcc fn SetHandleInformation(hObject: HANDLE, dwMask:
184184
185185pub extern "kernel32" stdcallcc fn Sleep(dwMilliseconds: DWORD) void;
186186
187pub extern "kernel32" stdcallcc fn SwitchToThread() BOOL;
188
187189pub extern "kernel32" stdcallcc fn TerminateProcess(hProcess: HANDLE, uExitCode: UINT) BOOL;
188190
189191pub extern "kernel32" stdcallcc fn TlsAlloc() DWORD;
lib/std/os/windows/ntdll.zig+18
......@@ -43,3 +43,21 @@ pub extern "NtDll" stdcallcc fn NtQueryDirectoryFile(
4343 FileName: ?*UNICODE_STRING,
4444 RestartScan: BOOLEAN,
4545) NTSTATUS;
46pub extern "NtDll" stdcallcc fn NtCreateKeyedEvent(
47 KeyedEventHandle: *HANDLE,
48 DesiredAccess: ACCESS_MASK,
49 ObjectAttributes: ?PVOID,
50 Flags: ULONG,
51) NTSTATUS;
52pub extern "NtDll" stdcallcc fn NtReleaseKeyedEvent(
53 EventHandle: HANDLE,
54 Key: *const c_void,
55 Alertable: BOOLEAN,
56 Timeout: ?*LARGE_INTEGER,
57) NTSTATUS;
58pub extern "NtDll" stdcallcc fn NtWaitForKeyedEvent(
59 EventHandle: HANDLE,
60 Key: *const c_void,
61 Alertable: BOOLEAN,
62 Timeout: ?*LARGE_INTEGER,
63) NTSTATUS;
lib/std/parker.zig created+180
......@@ -0,0 +1,180 @@
1const std = @import("std.zig");
2const builtin = @import("builtin");
3const time = std.time;
4const testing = std.testing;
5const assert = std.debug.assert;
6const SpinLock = std.SpinLock;
7const linux = std.os.linux;
8const windows = std.os.windows;
9
10pub const ThreadParker = switch (builtin.os) {
11 .linux => if (builtin.link_libc) PosixParker else LinuxParker,
12 .windows => WindowsParker,
13 else => if (builtin.link_libc) PosixParker else SpinParker,
14};
15
16const SpinParker = struct {
17 pub fn init() SpinParker {
18 return SpinParker{};
19 }
20 pub fn deinit(self: *SpinParker) void {}
21
22 pub fn unpark(self: *SpinParker, ptr: *const u32) void {}
23
24 pub fn park(self: *SpinParker, ptr: *const u32, expected: u32) void {
25 var backoff = SpinLock.Backoff.init();
26 while (@atomicLoad(u32, ptr, .Acquire) == expected)
27 backoff.yield();
28 }
29};
30
31const LinuxParker = struct {
32 pub fn init() LinuxParker {
33 return LinuxParker{};
34 }
35 pub fn deinit(self: *LinuxParker) void {}
36
37 pub fn unpark(self: *LinuxParker, ptr: *const u32) void {
38 const rc = linux.futex_wake(@ptrCast(*const i32, ptr), linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, 1);
39 assert(linux.getErrno(rc) == 0);
40 }
41
42 pub fn park(self: *LinuxParker, ptr: *const u32, expected: u32) void {
43 const value = @intCast(i32, expected);
44 while (@atomicLoad(u32, ptr, .Acquire) == expected) {
45 const rc = linux.futex_wait(@ptrCast(*const i32, ptr), linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, value, null);
46 switch (linux.getErrno(rc)) {
47 0, linux.EAGAIN => return,
48 linux.EINTR => continue,
49 linux.EINVAL => unreachable,
50 else => continue,
51 }
52 }
53 }
54};
55
56const WindowsParker = struct {
57 waiters: u32,
58
59 pub fn init() WindowsParker {
60 return WindowsParker{ .waiters = 0 };
61 }
62 pub fn deinit(self: *WindowsParker) void {}
63
64 pub fn unpark(self: *WindowsParker, ptr: *const u32) void {
65 const key = @ptrCast(*const c_void, ptr);
66 const handle = getEventHandle() orelse return;
67
68 var waiting = @atomicLoad(u32, &self.waiters, .Monotonic);
69 while (waiting != 0) {
70 waiting = @cmpxchgWeak(u32, &self.waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse {
71 const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null);
72 assert(rc == 0);
73 return;
74 };
75 }
76 }
77
78 pub fn park(self: *WindowsParker, ptr: *const u32, expected: u32) void {
79 var spin = SpinLock.Backoff.init();
80 const ev_handle = getEventHandle();
81 const key = @ptrCast(*const c_void, ptr);
82
83 while (@atomicLoad(u32, ptr, .Monotonic) == expected) {
84 if (ev_handle) |handle| {
85 _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release);
86 const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null);
87 assert(rc == 0);
88 } else {
89 spin.yield();
90 }
91 }
92 }
93
94 var event_handle = std.lazyInit(windows.HANDLE);
95
96 fn getEventHandle() ?windows.HANDLE {
97 if (event_handle.get()) |handle_ptr|
98 return handle_ptr.*;
99 defer event_handle.resolve();
100
101 const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE;
102 if (windows.ntdll.NtCreateKeyedEvent(&event_handle.data, access_mask, null, 0) != 0)
103 return null;
104 return event_handle.data;
105 }
106};
107
108const PosixParker = struct {
109 cond: c.pthread_cond_t,
110 mutex: c.pthread_mutex_t,
111
112 const c = std.c;
113
114 pub fn init() PosixParker {
115 return PosixParker{
116 .cond = c.PTHREAD_COND_INITIALIZER,
117 .mutex = c.PTHREAD_MUTEX_INITIALIZER,
118 };
119 }
120
121 pub fn deinit(self: *PosixParker) void {
122 // On dragonfly, the destroy functions return EINVAL if they were initialized statically.
123 const retm = c.pthread_mutex_destroy(&self.mutex);
124 assert(retm == 0 or retm == (if (builtin.os == .dragonfly) os.EINVAL else 0));
125 const retc = c.pthread_cond_destroy(&self.cond);
126 assert(retc == 0 or retc == (if (builtin.os == .dragonfly) os.EINVAL else 0));
127 }
128
129 pub fn unpark(self: *PosixParker, ptr: *const u32) void {
130 assert(c.pthread_mutex_lock(&self.mutex) == 0);
131 defer assert(c.pthread_mutex_unlock(&self.mutex) == 0);
132 assert(c.pthread_cond_signal(&self.cond) == 0);
133 }
134
135 pub fn park(self: *PosixParker, ptr: *const u32, expected: u32) void {
136 assert(c.pthread_mutex_lock(&self.mutex) == 0);
137 defer assert(c.pthread_mutex_unlock(&self.mutex) == 0);
138 while (@atomicLoad(u32, ptr, .Acquire) == expected)
139 assert(c.pthread_cond_wait(&self.cond, &self.mutex) == 0);
140 }
141};
142
143test "std.ThreadParker" {
144 if (builtin.single_threaded)
145 return error.SkipZigTest;
146
147 const Context = struct {
148 parker: ThreadParker,
149 data: u32,
150
151 fn receiver(self: *@This()) void {
152 self.parker.park(&self.data, 0); // receives 1
153 assert(@atomicRmw(u32, &self.data, .Xchg, 2, .SeqCst) == 1); // sends 2
154 self.parker.unpark(&self.data); // wakes up waiters on 2
155 self.parker.park(&self.data, 2); // receives 3
156 assert(@atomicRmw(u32, &self.data, .Xchg, 4, .SeqCst) == 3); // sends 4
157 self.parker.unpark(&self.data); // wakes up waiters on 4
158 }
159
160 fn sender(self: *@This()) void {
161 assert(@atomicRmw(u32, &self.data, .Xchg, 1, .SeqCst) == 0); // sends 1
162 self.parker.unpark(&self.data); // wakes up waiters on 1
163 self.parker.park(&self.data, 1); // receives 2
164 assert(@atomicRmw(u32, &self.data, .Xchg, 3, .SeqCst) == 2); // sends 3
165 self.parker.unpark(&self.data); // wakes up waiters on 3
166 self.parker.park(&self.data, 3); // receives 4
167 }
168 };
169
170 var context = Context{
171 .parker = ThreadParker.init(),
172 .data = 0,
173 };
174 defer context.parker.deinit();
175
176 var receiver = try std.Thread.spawn(&context, Context.receiver);
177 defer receiver.wait();
178
179 context.sender();
180}
lib/std/spinlock.zig+42-4
......@@ -1,8 +1,8 @@
11const std = @import("std.zig");
22const builtin = @import("builtin");
3const AtomicOrder = builtin.AtomicOrder;
4const AtomicRmwOp = builtin.AtomicRmwOp;
53const assert = std.debug.assert;
4const time = std.time;
5const os = std.os;
66
77pub const SpinLock = struct {
88 lock: u8, // TODO use a bool or enum
......@@ -11,7 +11,8 @@ pub const SpinLock = struct {
1111 spinlock: *SpinLock,
1212
1313 pub fn release(self: Held) void {
14 assert(@atomicRmw(u8, &self.spinlock.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1);
14 // TODO: @atomicStore() https://github.com/ziglang/zig/issues/2995
15 assert(@atomicRmw(u8, &self.spinlock.lock, .Xchg, 0, .Release) == 1);
1516 }
1617 };
1718
......@@ -20,9 +21,46 @@ pub const SpinLock = struct {
2021 }
2122
2223 pub fn acquire(self: *SpinLock) Held {
23 while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {}
24 var backoff = Backoff.init();
25 while (@atomicRmw(u8, &self.lock, .Xchg, 1, .Acquire) != 0)
26 backoff.yield();
2427 return Held{ .spinlock = self };
2528 }
29
30 pub fn yield(iterations: usize) void {
31 var i = iterations;
32 while (i != 0) : (i -= 1) {
33 switch (builtin.arch) {
34 .i386, .x86_64 => asm volatile ("pause"),
35 .arm, .aarch64 => asm volatile ("yield"),
36 else => time.sleep(0),
37 }
38 }
39 }
40
41 /// Provides a method to incrementally yield longer each time its called.
42 pub const Backoff = struct {
43 iteration: usize,
44
45 pub fn init() @This() {
46 return @This(){ .iteration = 0 };
47 }
48
49 /// Modified hybrid yielding from
50 /// http://www.1024cores.net/home/lock-free-algorithms/tricks/spinning
51 pub fn yield(self: *@This()) void {
52 defer self.iteration +%= 1;
53 if (self.iteration < 20) {
54 SpinLock.yield(self.iteration);
55 } else if (self.iteration < 24) {
56 os.sched_yield() catch time.sleep(1);
57 } else if (self.iteration < 26) {
58 time.sleep(1 * time.millisecond);
59 } else {
60 time.sleep(10 * time.millisecond);
61 }
62 }
63 };
2664};
2765
2866test "spinlock" {
lib/std/statically_initialized_mutex.zig deleted-105
......@@ -1,105 +0,0 @@
1const std = @import("std.zig");
2const builtin = @import("builtin");
3const AtomicOrder = builtin.AtomicOrder;
4const AtomicRmwOp = builtin.AtomicRmwOp;
5const assert = std.debug.assert;
6const expect = std.testing.expect;
7const windows = std.os.windows;
8
9/// Lock may be held only once. If the same thread
10/// tries to acquire the same mutex twice, it deadlocks.
11/// This type is intended to be initialized statically. If you don't
12/// require static initialization, use std.Mutex.
13/// On Windows, this mutex allocates resources when it is
14/// first used, and the resources cannot be freed.
15/// On Linux, this is an alias of std.Mutex.
16pub const StaticallyInitializedMutex = switch (builtin.os) {
17 builtin.Os.linux => std.Mutex,
18 builtin.Os.windows => struct {
19 lock: windows.CRITICAL_SECTION,
20 init_once: windows.RTL_RUN_ONCE,
21
22 pub const Held = struct {
23 mutex: *StaticallyInitializedMutex,
24
25 pub fn release(self: Held) void {
26 windows.kernel32.LeaveCriticalSection(&self.mutex.lock);
27 }
28 };
29
30 pub fn init() StaticallyInitializedMutex {
31 return StaticallyInitializedMutex{
32 .lock = undefined,
33 .init_once = windows.INIT_ONCE_STATIC_INIT,
34 };
35 }
36
37 extern fn initCriticalSection(
38 InitOnce: *windows.RTL_RUN_ONCE,
39 Parameter: ?*c_void,
40 Context: ?*c_void,
41 ) windows.BOOL {
42 const lock = @ptrCast(*windows.CRITICAL_SECTION, @alignCast(@alignOf(windows.CRITICAL_SECTION), Parameter));
43 windows.kernel32.InitializeCriticalSection(lock);
44 return windows.TRUE;
45 }
46
47 /// TODO: once https://github.com/ziglang/zig/issues/287 is solved and std.Mutex has a better
48 /// implementation of a runtime initialized mutex, remove this function.
49 pub fn deinit(self: *StaticallyInitializedMutex) void {
50 windows.InitOnceExecuteOnce(&self.init_once, initCriticalSection, &self.lock, null);
51 windows.kernel32.DeleteCriticalSection(&self.lock);
52 }
53
54 pub fn acquire(self: *StaticallyInitializedMutex) Held {
55 windows.InitOnceExecuteOnce(&self.init_once, initCriticalSection, &self.lock, null);
56 windows.kernel32.EnterCriticalSection(&self.lock);
57 return Held{ .mutex = self };
58 }
59 },
60 else => std.Mutex,
61};
62
63test "std.StaticallyInitializedMutex" {
64 const TestContext = struct {
65 data: i128,
66
67 const TestContext = @This();
68 const incr_count = 10000;
69
70 var mutex = StaticallyInitializedMutex.init();
71
72 fn worker(ctx: *TestContext) void {
73 var i: usize = 0;
74 while (i != TestContext.incr_count) : (i += 1) {
75 const held = mutex.acquire();
76 defer held.release();
77
78 ctx.data += 1;
79 }
80 }
81 };
82
83 var plenty_of_memory = try std.heap.direct_allocator.alloc(u8, 300 * 1024);
84 defer std.heap.direct_allocator.free(plenty_of_memory);
85
86 var fixed_buffer_allocator = std.heap.ThreadSafeFixedBufferAllocator.init(plenty_of_memory);
87 var a = &fixed_buffer_allocator.allocator;
88
89 var context = TestContext{ .data = 0 };
90
91 if (builtin.single_threaded) {
92 TestContext.worker(&context);
93 expect(context.data == TestContext.incr_count);
94 } else {
95 const thread_count = 10;
96 var threads: [thread_count]*std.Thread = undefined;
97 for (threads) |*t| {
98 t.* = try std.Thread.spawn(&context, TestContext.worker);
99 }
100 for (threads) |t|
101 t.wait();
102
103 expect(context.data == thread_count * TestContext.incr_count);
104 }
105}
lib/std/std.zig+1-1
......@@ -19,11 +19,11 @@ pub const Progress = @import("progress.zig").Progress;
1919pub const SegmentedList = @import("segmented_list.zig").SegmentedList;
2020pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList;
2121pub const SpinLock = @import("spinlock.zig").SpinLock;
22pub const StaticallyInitializedMutex = @import("statically_initialized_mutex.zig").StaticallyInitializedMutex;
2322pub const StringHashMap = @import("hash_map.zig").StringHashMap;
2423pub const TailQueue = @import("linked_list.zig").TailQueue;
2524pub const Target = @import("target.zig").Target;
2625pub const Thread = @import("thread.zig").Thread;
26pub const ThreadParker = @import("parker.zig").ThreadParker;
2727
2828pub const atomic = @import("atomic.zig");
2929pub const base64 = @import("base64.zig");