1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice unmodified, this list of conditions, and the following
12 * disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 *
28 */
29
30#ifndef _SYS_UMTXVAR_H_
31#define _SYS_UMTXVAR_H_
32
33#ifdef _KERNEL
34
35#include <sys/_timespec.h>
36
37/*
38 * The umtx_key structure is used by both the Linux futex code and the
39 * umtx implementation to map userland addresses to unique keys.
40 */
41enum {
42 TYPE_SIMPLE_WAIT,
43 TYPE_CV,
44 TYPE_SEM,
45 TYPE_SIMPLE_LOCK,
46 TYPE_NORMAL_UMUTEX,
47 TYPE_PI_UMUTEX,
48 TYPE_PP_UMUTEX,
49 TYPE_RWLOCK,
50 TYPE_FUTEX,
51 TYPE_SHM,
52 TYPE_PI_ROBUST_UMUTEX,
53 TYPE_PP_ROBUST_UMUTEX,
54 TYPE_PI_FUTEX,
55};
56
57/* Key to represent a unique userland synchronous object */
58struct umtx_key {
59 int hash;
60 int type;
61 int shared;
62 union {
63 struct {
64 struct vm_object *object;
65 uintptr_t offset;
66 } shared;
67 struct {
68 struct vmspace *vs;
69 uintptr_t addr;
70 } private;
71 struct {
72 void *a;
73 uintptr_t b;
74 } both;
75 } info;
76};
77
78#define THREAD_SHARE 0
79#define PROCESS_SHARE 1
80#define AUTO_SHARE 2
81
82struct umtx_abs_timeout {
83 int clockid;
84 bool is_abs_real; /* TIMER_ABSTIME && CLOCK_REALTIME* */
85 struct timespec cur;
86 struct timespec end;
87};
88
89struct thread;
90
91/* Priority inheritance mutex info. */
92struct umtx_pi {
93 /* Owner thread */
94 struct thread *pi_owner;
95
96 /* Reference count */
97 int pi_refcount;
98
99 /* List entry to link umtx holding by thread */
100 TAILQ_ENTRY(umtx_pi) pi_link;
101
102 /* List entry in hash */
103 TAILQ_ENTRY(umtx_pi) pi_hashlink;
104
105 /* List for waiters */
106 TAILQ_HEAD(,umtx_q) pi_blocked;
107
108 /* Identify a userland lock object */
109 struct umtx_key pi_key;
110};
111
112/* A userland synchronous object user. */
113struct umtx_q {
114 /* Linked list for the hash. */
115 TAILQ_ENTRY(umtx_q) uq_link;
116
117 /* Umtx key. */
118 struct umtx_key uq_key;
119
120 /* Umtx flags. */
121 int uq_flags;
122#define UQF_UMTXQ 0x0001
123
124 /* Futex bitset mask */
125 u_int uq_bitset;
126
127 /* The thread waits on. */
128 struct thread *uq_thread;
129
130 /*
131 * Blocked on PI mutex. read can use chain lock
132 * or umtx_lock, write must have both chain lock and
133 * umtx_lock being hold.
134 */
135 struct umtx_pi *uq_pi_blocked;
136
137 /* On blocked list */
138 TAILQ_ENTRY(umtx_q) uq_lockq;
139
140 /* Thread contending with us */
141 TAILQ_HEAD(,umtx_pi) uq_pi_contested;
142
143 /* Inherited priority from PP mutex */
144 u_char uq_inherited_pri;
145
146 /* Spare queue ready to be reused */
147 struct umtxq_queue *uq_spare_queue;
148
149 /* The queue we on */
150 struct umtxq_queue *uq_cur_queue;
151};
152
153TAILQ_HEAD(umtxq_head, umtx_q);
154
155/* Per-key wait-queue */
156struct umtxq_queue {
157 struct umtxq_head head;
158 struct umtx_key key;
159 LIST_ENTRY(umtxq_queue) link;
160 int length;
161};
162
163LIST_HEAD(umtxq_list, umtxq_queue);
164
165/* Userland lock object's wait-queue chain */
166struct umtxq_chain {
167 /* Lock for this chain. */
168 struct mtx uc_lock;
169
170 /* List of sleep queues. */
171 struct umtxq_list uc_queue[2];
172#define UMTX_SHARED_QUEUE 0
173#define UMTX_EXCLUSIVE_QUEUE 1
174
175 LIST_HEAD(, umtxq_queue) uc_spare_queue;
176
177 /* Busy flag */
178 char uc_busy;
179
180 /* Chain lock waiters */
181 int uc_waiters;
182
183 /* All PI in the list */
184 TAILQ_HEAD(,umtx_pi) uc_pi_list;
185
186#ifdef UMTX_PROFILING
187 u_int length;
188 u_int max_length;
189#endif
190};
191
192static inline int
193umtx_key_match(const struct umtx_key *k1, const struct umtx_key *k2)
194{
195
196 return (k1->type == k2->type &&
197 k1->info.both.a == k2->info.both.a &&
198 k1->info.both.b == k2->info.both.b);
199}
200
201void umtx_abs_timeout_init(struct umtx_abs_timeout *, int, int,
202 const struct timespec *);
203int umtx_copyin_timeout(const void *, struct timespec *);
204void umtx_exec(struct proc *p);
205int umtx_key_get(const void *, int, int, struct umtx_key *);
206void umtx_key_release(struct umtx_key *);
207struct umtx_q *umtxq_alloc(void);
208void umtxq_busy(struct umtx_key *);
209void umtxq_busy_unlocked(struct umtx_key *);
210int umtxq_count(struct umtx_key *);
211void umtxq_free(struct umtx_q *);
212struct umtxq_chain *umtxq_getchain(struct umtx_key *);
213void umtxq_insert_queue(struct umtx_q *, int);
214void umtxq_remove_queue(struct umtx_q *, int);
215int umtxq_requeue(struct umtx_key *, int, struct umtx_key *, int);
216int umtxq_signal_mask(struct umtx_key *, int, u_int);
217int umtxq_sleep(struct umtx_q *, const char *,
218 struct umtx_abs_timeout *);
219int umtxq_sleep_pi(struct umtx_q *, struct umtx_pi *, uint32_t,
220 const char *, struct umtx_abs_timeout *, bool);
221void umtxq_unbusy(struct umtx_key *);
222void umtxq_unbusy_unlocked(struct umtx_key *);
223int kern_umtx_wake(struct thread *, void *, int, int);
224void umtx_pi_adjust(struct thread *, u_char);
225struct umtx_pi *umtx_pi_alloc(int);
226int umtx_pi_claim(struct umtx_pi *, struct thread *);
227int umtx_pi_drop(struct thread *, struct umtx_key *, bool, int *);
228void umtx_pi_free(struct umtx_pi *);
229void umtx_pi_insert(struct umtx_pi *);
230struct umtx_pi *umtx_pi_lookup(struct umtx_key *);
231void umtx_pi_ref(struct umtx_pi *);
232void umtx_pi_unref(struct umtx_pi *);
233void umtx_thread_init(struct thread *);
234void umtx_thread_fini(struct thread *);
235void umtx_thread_alloc(struct thread *);
236void umtx_thread_exit(struct thread *);
237
238#define umtxq_insert(uq) umtxq_insert_queue((uq), UMTX_SHARED_QUEUE)
239#define umtxq_remove(uq) umtxq_remove_queue((uq), UMTX_SHARED_QUEUE)
240
241/*
242 * Lock a chain.
243 *
244 * The code is a macro so that file/line information is taken from the caller.
245 */
246#define umtxq_lock(key) do { \
247 struct umtx_key *_key = (key); \
248 struct umtxq_chain *_uc; \
249 \
250 _uc = umtxq_getchain(_key); \
251 mtx_lock(&_uc->uc_lock); \
252} while (0)
253
254/*
255 * Unlock a chain.
256 */
257static inline void
258umtxq_unlock(struct umtx_key *key)
259{
260 struct umtxq_chain *uc;
261
262 uc = umtxq_getchain(key);
263 mtx_unlock(&uc->uc_lock);
264}
265
266#endif /* _KERNEL */
267#endif /* !_SYS_UMTXVAR_H_ */