1/* $OpenBSD: ifq.h,v 1.44 2025/03/04 01:13:37 dlg Exp $ */
2
3/*
4 * Copyright (c) 2015 David Gwynne <dlg@openbsd.org>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#ifndef _NET_IFQ_H_
20#define _NET_IFQ_H_
21
22struct ifnet;
23struct kstat;
24
25struct ifq_ops;
26
27struct ifqueue {
28 struct ifnet *ifq_if;
29 struct taskq *ifq_softnet;
30 union {
31 void *_ifq_softc;
32 /*
33 * a rings sndq is found by looking up an array of pointers.
34 * by default we only have one sndq and the default drivers
35 * dont use ifq_softc, so we can borrow it for the map until
36 * we need to allocate a proper map.
37 */
38 struct ifqueue *_ifq_ifqs[1];
39 } _ifq_ptr;
40#define ifq_softc _ifq_ptr._ifq_softc
41#define ifq_ifqs _ifq_ptr._ifq_ifqs
42
43 /* mbuf handling */
44 struct mutex ifq_mtx;
45 const struct ifq_ops *ifq_ops;
46 void *ifq_q;
47 struct mbuf_list ifq_free;
48 unsigned int ifq_len;
49 unsigned int ifq_oactive;
50
51 /* statistics */
52 uint64_t ifq_packets;
53 uint64_t ifq_bytes;
54 uint64_t ifq_qdrops;
55 uint64_t ifq_errors;
56 uint64_t ifq_mcasts;
57 uint32_t ifq_oactives;
58
59 struct kstat *ifq_kstat;
60
61 /* work serialisation */
62 struct mutex ifq_task_mtx;
63 struct task_list ifq_task_list;
64 void *ifq_serializer;
65 struct task ifq_bundle;
66
67 /* work to be serialised */
68 struct task ifq_start;
69 struct task ifq_restart;
70
71 /* properties */
72 unsigned int ifq_maxlen;
73 unsigned int ifq_idx;
74};
75
76struct ifiqueue {
77 struct ifnet *ifiq_if;
78 caddr_t *ifiq_bpfp;
79 struct taskq *ifiq_softnet;
80 union {
81 void *_ifiq_softc;
82 struct ifiqueue *_ifiq_ifiqs[1];
83 } _ifiq_ptr;
84#define ifiq_softc _ifiq_ptr._ifiq_softc
85#define ifiq_ifiqs _ifiq_ptr._ifiq_ifiqs
86
87 struct mutex ifiq_mtx;
88 struct mbuf_list ifiq_ml;
89 struct task ifiq_task;
90 unsigned int ifiq_pressure;
91
92 /* counters */
93 uint64_t ifiq_packets;
94 uint64_t ifiq_bytes;
95 uint64_t ifiq_fdrops;
96 uint64_t ifiq_qdrops;
97 uint64_t ifiq_errors;
98 uint64_t ifiq_mcasts;
99 uint64_t ifiq_noproto;
100
101 /* number of times a list of packets were put on ifiq_ml */
102 uint64_t ifiq_enqueues;
103 /* number of times a list of packets were pulled off ifiq_ml */
104 uint64_t ifiq_dequeues;
105
106 struct kstat *ifiq_kstat;
107
108 /* properties */
109 unsigned int ifiq_idx;
110};
111
112#ifdef _KERNEL
113
114#define IFQ_MAXLEN 256
115
116/*
117 *
118 * Interface Send Queues
119 *
120 * struct ifqueue sits between the network stack and a drivers
121 * transmission of packets. The high level view is that when the stack
122 * has finished generating a packet it hands it to a driver for
123 * transmission. It does this by queueing the packet on an ifqueue and
124 * notifying the driver to start transmission of the queued packets.
125 *
126 * A network device may have multiple contexts for the transmission
127 * of packets, ie, independent transmit rings. Such a network device,
128 * represented by a struct ifnet, would then have multiple ifqueue
129 * structures, each of which maps to an independent transmit ring.
130 *
131 * struct ifqueue also provides the point where conditioning of
132 * traffic (ie, priq and hfsc) is implemented, and provides some
133 * infrastructure to assist in the implementation of network drivers.
134 *
135 * = ifq API
136 *
137 * The ifq API provides functions for three distinct consumers:
138 *
139 * 1. The network stack
140 * 2. Traffic QoS/conditioning implementations
141 * 3. Network drivers
142 *
143 * == Network Stack API
144 *
145 * The network stack is responsible for initialising and destroying
146 * the ifqueue structures, changing the traffic conditioner on an
147 * interface, enqueuing packets for transmission, and notifying
148 * the driver to start transmission of a particular ifqueue.
149 *
150 * === ifq_init()
151 *
152 * During if_attach(), the network stack calls ifq_init to initialise
153 * the ifqueue structure. By default it configures the priq traffic
154 * conditioner.
155 *
156 * === ifq_destroy()
157 *
158 * The network stack calls ifq_destroy() during if_detach to tear down
159 * the ifqueue structure. It frees the traffic conditioner state, and
160 * frees any mbufs that were left queued.
161 *
162 * === ifq_attach()
163 *
164 * ifq_attach() is used to replace the current traffic conditioner on
165 * the ifqueue. All the pending mbufs are removed from the previous
166 * conditioner and requeued on the new.
167 *
168 * === ifq_idx()
169 *
170 * ifq_idx() selects a specific ifqueue from the current ifnet
171 * structure for use in the transmission of the mbuf.
172 *
173 * === ifq_enqueue()
174 *
175 * ifq_enqueue() attempts to fit an mbuf onto the ifqueue. The
176 * current traffic conditioner may drop a packet to make space on the
177 * queue.
178 *
179 * === ifq_start()
180 *
181 * Once a packet has been successfully queued with ifq_enqueue(),
182 * the network card is notified with a call to ifq_start().
183 * Calls to ifq_start() run in the ifqueue serialisation context,
184 * guaranteeing that only one instance of ifp->if_qstart() will be
185 * running on behalf of a specific ifqueue in the system at any point
186 * in time.
187 *
188 * == Traffic conditioners API
189 *
190 * The majority of interaction between struct ifqueue and a traffic
191 * conditioner occurs via the callbacks a traffic conditioner provides
192 * in an instance of struct ifq_ops.
193 *
194 * XXX document ifqop_*
195 *
196 * The ifqueue API implements the locking on behalf of the conditioning
197 * implementations so conditioners only have to reject or keep mbufs.
198 * If something needs to inspect a conditioners internals, the queue lock
199 * needs to be taken to allow for a consistent or safe view. The queue
200 * lock may be taken and released with ifq_q_enter() and ifq_q_leave().
201 *
202 * === ifq_q_enter()
203 *
204 * Code wishing to access a conditioners internals may take the queue
205 * lock with ifq_q_enter(). The caller must pass a reference to the
206 * conditioners ifq_ops structure so the infrastructure can ensure the
207 * caller is able to understand the internals. ifq_q_enter() returns
208 * a pointer to the conditioners internal structures, or NULL if the
209 * ifq_ops did not match the current conditioner.
210 *
211 * === ifq_q_leave()
212 *
213 * The queue lock acquired with ifq_q_enter() is released with
214 * ifq_q_leave().
215 *
216 * === ifq_mfreem() and ifq_mfreeml()
217 *
218 * A goal of the API is to avoid freeing an mbuf while mutexes are
219 * held. Because the ifq API manages the lock on behalf of the backend
220 * ifqops, the backend should not directly free mbufs. If a conditioner
221 * backend needs to drop a packet during the handling of ifqop_deq_begin,
222 * it may free it by calling ifq_mfreem(). This accounts for the drop,
223 * and schedules the free of the mbuf outside the hold of ifq_mtx.
224 * ifq_mfreeml() takes an mbuf list as an argument instead.
225 *
226 *
227 * == Network Driver API
228 *
229 * The API used by network drivers is mostly documented in the
230 * ifq_dequeue(9) manpage except for ifq_serialize().
231 *
232 * === ifq_serialize()
233 *
234 * A driver may run arbitrary work in the ifqueue serialiser context
235 * via ifq_serialize(). The work to be done is represented by a task
236 * that has been prepared with task_set.
237 *
238 * The work will be run in series with any other work dispatched by
239 * ifq_start(), ifq_restart(), or other ifq_serialize() calls.
240 *
241 * Because the work may be run on another CPU, the lifetime of the
242 * task and the work it represents can extend beyond the end of the
243 * call to ifq_serialize() that dispatched it.
244 *
245 *
246 * = ifqueue work serialisation
247 *
248 * ifqueues provide a mechanism to dispatch work to be run in a single
249 * context. Work in this mechanism is represented by task structures.
250 *
251 * The tasks are run in a context similar to a taskq serviced by a
252 * single kernel thread, except the work is run immediately by the
253 * first CPU that dispatches work. If a second CPU attempts to dispatch
254 * additional tasks while the first is still running, it will be queued
255 * to be run by the first CPU. The second CPU will return immediately.
256 *
257 * = MP Safe Network Drivers
258 *
259 * An MP safe network driver is one in which its start routine can be
260 * called by the network stack without holding the big kernel lock.
261 *
262 * == Attach
263 *
264 * A driver advertises its ability to run its start routine without
265 * the kernel lock by setting the IFXF_MPSAFE flag in ifp->if_xflags
266 * before calling if_attach(). Advertising an MPSAFE start routine
267 * also implies that the driver understands that a network card can
268 * have multiple rings or transmit queues, and therefore provides
269 * if_qstart function (which takes an ifqueue pointer) instead of an
270 * if_start function (which takes an ifnet pointer).
271 *
272 * If the hardware supports multiple transmit rings, it advertises
273 * support for multiple rings to the network stack with if_attach_queues()
274 * after the call to if_attach(). if_attach_queues allocates a struct
275 * ifqueue for each hardware ring, which can then be initialised by
276 * the driver with data for each ring.
277 *
278 * void drv_start(struct ifqueue *);
279 *
280 * void
281 * drv_attach()
282 * {
283 * ...
284 * ifp->if_xflags = IFXF_MPSAFE;
285 * ifp->if_qstart = drv_start;
286 * if_attach(ifp);
287 *
288 * if_attach_queues(ifp, DRV_NUM_TX_RINGS);
289 * for (i = 0; i < DRV_NUM_TX_RINGS; i++) {
290 * struct ifqueue *ifq = ifp->if_ifqs[i];
291 * struct drv_tx_ring *ring = &sc->sc_tx_rings[i];
292 *
293 * ifq->ifq_softc = ring;
294 * ring->ifq = ifq;
295 * }
296 * }
297 *
298 * The network stack will then call ifp->if_qstart via ifq_start()
299 * to guarantee there is only one instance of that function running
300 * for each ifq in the system, and to serialise it with other work
301 * the driver may provide.
302 *
303 * == Initialise
304 *
305 * When the stack requests an interface be brought up (ie, drv_ioctl()
306 * is called to handle SIOCSIFFLAGS with IFF_UP set in ifp->if_flags)
307 * drivers should set IFF_RUNNING in ifp->if_flags, and then call
308 * ifq_clr_oactive() against each ifq.
309 *
310 * == if_start
311 *
312 * ifq_start() checks that IFF_RUNNING is set in ifp->if_flags, that
313 * ifq_is_oactive() does not return true, and that there are pending
314 * packets to transmit via a call to ifq_len(). Therefore, drivers are
315 * no longer responsible for doing this themselves.
316 *
317 * If a driver should not transmit packets while its link is down, use
318 * ifq_purge() to flush pending packets from the transmit queue.
319 *
320 * Drivers for hardware should use the following pattern to transmit
321 * packets:
322 *
323 * void
324 * drv_start(struct ifqueue *ifq)
325 * {
326 * struct drv_tx_ring *ring = ifq->ifq_softc;
327 * struct ifnet *ifp = ifq->ifq_if;
328 * struct drv_softc *sc = ifp->if_softc;
329 * struct mbuf *m;
330 * int kick = 0;
331 *
332 * if (NO_LINK) {
333 * ifq_purge(ifq);
334 * return;
335 * }
336 *
337 * for (;;) {
338 * if (NO_SPACE(ring)) {
339 * ifq_set_oactive(ifq);
340 * break;
341 * }
342 *
343 * m = ifq_dequeue(ifq);
344 * if (m == NULL)
345 * break;
346 *
347 * if (drv_encap(sc, ring, m) != 0) { // map and fill ring
348 * m_freem(m);
349 * continue;
350 * }
351 *
352 * bpf_mtap();
353 * }
354 *
355 * drv_kick(ring); // notify hw of new descriptors on the ring
356 * }
357 *
358 * == Transmission completion
359 *
360 * The following pattern should be used for transmit queue interrupt
361 * processing:
362 *
363 * void
364 * drv_txeof(struct drv_tx_ring *ring)
365 * {
366 * struct ifqueue *ifq = ring->ifq;
367 *
368 * while (COMPLETED_PKTS(ring)) {
369 * // unmap packets, m_freem() the mbufs.
370 * }
371 *
372 * if (ifq_is_oactive(ifq))
373 * ifq_restart(ifq);
374 * }
375 *
376 * == Stop
377 *
378 * Bringing an interface down (ie, IFF_UP was cleared in ifp->if_flags)
379 * should clear IFF_RUNNING in ifp->if_flags, and guarantee the start
380 * routine is not running before freeing any resources it uses:
381 *
382 * void
383 * drv_down(struct drv_softc *sc)
384 * {
385 * struct ifnet *ifp = &sc->sc_if;
386 * struct ifqueue *ifq;
387 * int i;
388 *
389 * CLR(ifp->if_flags, IFF_RUNNING);
390 * DISABLE_INTERRUPTS();
391 *
392 * for (i = 0; i < sc->sc_num_queues; i++) {
393 * ifq = ifp->if_ifqs[i];
394 * ifq_barrier(ifq);
395 * }
396 *
397 * intr_barrier(sc->sc_ih);
398 *
399 * FREE_RESOURCES();
400 *
401 * for (i = 0; i < sc->sc_num_queues; i++) {
402 * ifq = ifp->if_ifqs[i];
403 * ifq_clr_oactive(ifq);
404 * }
405 * }
406 *
407 */
408
409struct ifq_ops {
410 unsigned int (*ifqop_idx)(unsigned int,
411 const struct mbuf *);
412 struct mbuf *(*ifqop_enq)(struct ifqueue *, struct mbuf *);
413 struct mbuf *(*ifqop_deq_begin)(struct ifqueue *, void **);
414 void (*ifqop_deq_commit)(struct ifqueue *,
415 struct mbuf *, void *);
416 void (*ifqop_purge)(struct ifqueue *,
417 struct mbuf_list *);
418 void *(*ifqop_alloc)(unsigned int, void *);
419 void (*ifqop_free)(unsigned int, void *);
420};
421
422extern const struct ifq_ops * const ifq_priq_ops;
423
424/*
425 * Interface send queues.
426 */
427
428void ifq_init(struct ifqueue *, struct ifnet *, unsigned int);
429void ifq_attach(struct ifqueue *, const struct ifq_ops *, void *);
430void ifq_destroy(struct ifqueue *);
431void ifq_add_data(struct ifqueue *, struct if_data *);
432int ifq_enqueue(struct ifqueue *, struct mbuf *);
433void ifq_start(struct ifqueue *);
434struct mbuf *ifq_deq_begin(struct ifqueue *);
435void ifq_deq_commit(struct ifqueue *, struct mbuf *);
436void ifq_deq_rollback(struct ifqueue *, struct mbuf *);
437struct mbuf *ifq_dequeue(struct ifqueue *);
438int ifq_hdatalen(struct ifqueue *);
439void ifq_init_maxlen(struct ifqueue *, unsigned int);
440void ifq_mfreem(struct ifqueue *, struct mbuf *);
441void ifq_mfreeml(struct ifqueue *, struct mbuf_list *);
442unsigned int ifq_purge(struct ifqueue *);
443void *ifq_q_enter(struct ifqueue *, const struct ifq_ops *);
444void ifq_q_leave(struct ifqueue *, void *);
445void ifq_serialize(struct ifqueue *, struct task *);
446void ifq_barrier(struct ifqueue *);
447void ifq_set_oactive(struct ifqueue *);
448void ifq_deq_set_oactive(struct ifqueue *);
449
450int ifq_deq_sleep(struct ifqueue *, struct mbuf **, int, int,
451 const char *, volatile unsigned int *,
452 volatile unsigned int *);
453
454#define ifq_len(_ifq) READ_ONCE((_ifq)->ifq_len)
455#define ifq_empty(_ifq) (ifq_len(_ifq) == 0)
456
457static inline int
458ifq_is_priq(struct ifqueue *ifq)
459{
460 return (ifq->ifq_ops == ifq_priq_ops);
461}
462
463static inline void
464ifq_clr_oactive(struct ifqueue *ifq)
465{
466 ifq->ifq_oactive = 0;
467}
468
469static inline unsigned int
470ifq_is_oactive(struct ifqueue *ifq)
471{
472 return (ifq->ifq_oactive);
473}
474
475static inline void
476ifq_restart(struct ifqueue *ifq)
477{
478 ifq_serialize(ifq, &ifq->ifq_restart);
479}
480
481static inline unsigned int
482ifq_idx(struct ifqueue *ifq, unsigned int nifqs, const struct mbuf *m)
483{
484 return ((*ifq->ifq_ops->ifqop_idx)(nifqs, m));
485}
486
487/* ifiq */
488
489void ifiq_init(struct ifiqueue *, struct ifnet *, unsigned int);
490void ifiq_destroy(struct ifiqueue *);
491int ifiq_input(struct ifiqueue *, struct mbuf_list *);
492int ifiq_enqueue_qlim(struct ifiqueue *, struct mbuf *,
493 unsigned int);
494void ifiq_add_data(struct ifiqueue *, struct if_data *);
495
496#define ifiq_len(_ifiq) READ_ONCE(ml_len(&(_ifiq)->ifiq_ml))
497#define ifiq_empty(_ifiq) (ifiq_len(_ifiq) == 0)
498
499static inline int
500ifiq_enqueue(struct ifiqueue *ifiq, struct mbuf *m)
501{
502 return ifiq_enqueue_qlim(ifiq, m, 0);
503}
504
505#endif /* _KERNEL */
506
507#endif /* _NET_IFQ_H_ */