pktqueue.c revision 1.13 1 1.13 skrll /* $NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $ */
2 1.1 rmind
3 1.1 rmind /*-
4 1.1 rmind * Copyright (c) 2014 The NetBSD Foundation, Inc.
5 1.1 rmind * All rights reserved.
6 1.1 rmind *
7 1.1 rmind * This code is derived from software contributed to The NetBSD Foundation
8 1.1 rmind * by Mindaugas Rasiukevicius.
9 1.1 rmind *
10 1.1 rmind * Redistribution and use in source and binary forms, with or without
11 1.1 rmind * modification, are permitted provided that the following conditions
12 1.1 rmind * are met:
13 1.1 rmind * 1. Redistributions of source code must retain the above copyright
14 1.1 rmind * notice, this list of conditions and the following disclaimer.
15 1.1 rmind * 2. Redistributions in binary form must reproduce the above copyright
16 1.1 rmind * notice, this list of conditions and the following disclaimer in the
17 1.1 rmind * documentation and/or other materials provided with the distribution.
18 1.1 rmind *
19 1.1 rmind * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 1.1 rmind * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 1.1 rmind * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 1.1 rmind * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 1.1 rmind * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 1.1 rmind * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 1.1 rmind * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 1.1 rmind * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 1.1 rmind * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 1.1 rmind * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 1.1 rmind * POSSIBILITY OF SUCH DAMAGE.
30 1.1 rmind */
31 1.1 rmind
32 1.4 rmind /*
33 1.4 rmind * The packet queue (pktqueue) interface is a lockless IP input queue
34 1.4 rmind * which also abstracts and handles network ISR scheduling. It provides
35 1.4 rmind * a mechanism to enable receiver-side packet steering (RPS).
36 1.4 rmind */
37 1.4 rmind
38 1.1 rmind #include <sys/cdefs.h>
39 1.13 skrll __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $");
40 1.1 rmind
41 1.1 rmind #include <sys/param.h>
42 1.1 rmind #include <sys/types.h>
43 1.1 rmind
44 1.1 rmind #include <sys/atomic.h>
45 1.1 rmind #include <sys/cpu.h>
46 1.1 rmind #include <sys/pcq.h>
47 1.1 rmind #include <sys/intr.h>
48 1.1 rmind #include <sys/mbuf.h>
49 1.1 rmind #include <sys/proc.h>
50 1.1 rmind #include <sys/percpu.h>
51 1.11 thorpej #include <sys/xcall.h>
52 1.1 rmind
53 1.1 rmind #include <net/pktqueue.h>
54 1.1 rmind
55 1.1 rmind struct pktqueue {
56 1.1 rmind /*
57 1.1 rmind * The lock used for a barrier mechanism. The barrier counter,
58 1.1 rmind * as well as the drop counter, are managed atomically though.
59 1.1 rmind * Ensure this group is in a separate cache line.
60 1.1 rmind */
61 1.13 skrll union {
62 1.13 skrll struct {
63 1.13 skrll kmutex_t pq_lock;
64 1.13 skrll volatile u_int pq_barrier;
65 1.13 skrll };
66 1.13 skrll uint8_t _pad[COHERENCY_UNIT];
67 1.13 skrll };
68 1.1 rmind
69 1.1 rmind /* The size of the queue, counters and the interrupt handler. */
70 1.1 rmind u_int pq_maxlen;
71 1.1 rmind percpu_t * pq_counters;
72 1.1 rmind void * pq_sih;
73 1.1 rmind
74 1.1 rmind /* Finally, per-CPU queues. */
75 1.12 riastrad struct percpu * pq_pcq; /* struct pcq * */
76 1.1 rmind };
77 1.1 rmind
78 1.1 rmind /* The counters of the packet queue. */
79 1.1 rmind #define PQCNT_ENQUEUE 0
80 1.1 rmind #define PQCNT_DEQUEUE 1
81 1.1 rmind #define PQCNT_DROP 2
82 1.1 rmind #define PQCNT_NCOUNTERS 3
83 1.1 rmind
84 1.1 rmind typedef struct {
85 1.1 rmind uint64_t count[PQCNT_NCOUNTERS];
86 1.1 rmind } pktq_counters_t;
87 1.1 rmind
88 1.1 rmind /* Special marker value used by pktq_barrier() mechanism. */
89 1.1 rmind #define PKTQ_MARKER ((void *)(~0ULL))
90 1.1 rmind
91 1.12 riastrad static void
92 1.12 riastrad pktq_init_cpu(void *vqp, void *vpq, struct cpu_info *ci)
93 1.12 riastrad {
94 1.12 riastrad struct pcq **qp = vqp;
95 1.12 riastrad struct pktqueue *pq = vpq;
96 1.12 riastrad
97 1.12 riastrad *qp = pcq_create(pq->pq_maxlen, KM_SLEEP);
98 1.12 riastrad }
99 1.12 riastrad
100 1.12 riastrad static void
101 1.12 riastrad pktq_fini_cpu(void *vqp, void *vpq, struct cpu_info *ci)
102 1.12 riastrad {
103 1.12 riastrad struct pcq **qp = vqp, *q = *qp;
104 1.12 riastrad
105 1.12 riastrad KASSERT(pcq_peek(q) == NULL);
106 1.12 riastrad pcq_destroy(q);
107 1.12 riastrad *qp = NULL; /* paranoia */
108 1.12 riastrad }
109 1.12 riastrad
110 1.12 riastrad static struct pcq *
111 1.12 riastrad pktq_pcq(struct pktqueue *pq, struct cpu_info *ci)
112 1.12 riastrad {
113 1.12 riastrad struct pcq **qp, *q;
114 1.12 riastrad
115 1.12 riastrad /*
116 1.12 riastrad * As long as preemption is disabled, the xcall to swap percpu
117 1.12 riastrad * buffers can't complete, so it is safe to read the pointer.
118 1.12 riastrad */
119 1.12 riastrad KASSERT(kpreempt_disabled());
120 1.12 riastrad
121 1.12 riastrad qp = percpu_getptr_remote(pq->pq_pcq, ci);
122 1.12 riastrad q = *qp;
123 1.12 riastrad
124 1.12 riastrad return q;
125 1.12 riastrad }
126 1.1 rmind
127 1.1 rmind pktqueue_t *
128 1.5 ozaki pktq_create(size_t maxlen, void (*intrh)(void *), void *sc)
129 1.1 rmind {
130 1.1 rmind const u_int sflags = SOFTINT_NET | SOFTINT_MPSAFE | SOFTINT_RCPU;
131 1.1 rmind pktqueue_t *pq;
132 1.1 rmind percpu_t *pc;
133 1.1 rmind void *sih;
134 1.1 rmind
135 1.9 chs pc = percpu_alloc(sizeof(pktq_counters_t));
136 1.5 ozaki if ((sih = softint_establish(sflags, intrh, sc)) == NULL) {
137 1.1 rmind percpu_free(pc, sizeof(pktq_counters_t));
138 1.1 rmind return NULL;
139 1.1 rmind }
140 1.1 rmind
141 1.12 riastrad pq = kmem_zalloc(sizeof(*pq), KM_SLEEP);
142 1.1 rmind mutex_init(&pq->pq_lock, MUTEX_DEFAULT, IPL_NONE);
143 1.1 rmind pq->pq_maxlen = maxlen;
144 1.1 rmind pq->pq_counters = pc;
145 1.1 rmind pq->pq_sih = sih;
146 1.12 riastrad pq->pq_pcq = percpu_create(sizeof(struct pcq *),
147 1.12 riastrad pktq_init_cpu, pktq_fini_cpu, pq);
148 1.1 rmind
149 1.1 rmind return pq;
150 1.1 rmind }
151 1.1 rmind
152 1.1 rmind void
153 1.1 rmind pktq_destroy(pktqueue_t *pq)
154 1.1 rmind {
155 1.1 rmind
156 1.12 riastrad percpu_free(pq->pq_pcq, sizeof(struct pcq *));
157 1.1 rmind percpu_free(pq->pq_counters, sizeof(pktq_counters_t));
158 1.1 rmind softint_disestablish(pq->pq_sih);
159 1.1 rmind mutex_destroy(&pq->pq_lock);
160 1.12 riastrad kmem_free(pq, sizeof(*pq));
161 1.1 rmind }
162 1.1 rmind
163 1.1 rmind /*
164 1.1 rmind * - pktq_inc_counter: increment the counter given an ID.
165 1.1 rmind * - pktq_collect_counts: handler to sum up the counts from each CPU.
166 1.1 rmind * - pktq_getcount: return the effective count given an ID.
167 1.1 rmind */
168 1.1 rmind
169 1.1 rmind static inline void
170 1.1 rmind pktq_inc_count(pktqueue_t *pq, u_int i)
171 1.1 rmind {
172 1.1 rmind percpu_t *pc = pq->pq_counters;
173 1.1 rmind pktq_counters_t *c;
174 1.1 rmind
175 1.1 rmind c = percpu_getref(pc);
176 1.1 rmind c->count[i]++;
177 1.1 rmind percpu_putref(pc);
178 1.1 rmind }
179 1.1 rmind
180 1.1 rmind static void
181 1.1 rmind pktq_collect_counts(void *mem, void *arg, struct cpu_info *ci)
182 1.1 rmind {
183 1.1 rmind const pktq_counters_t *c = mem;
184 1.1 rmind pktq_counters_t *sum = arg;
185 1.1 rmind
186 1.11 thorpej int s = splnet();
187 1.11 thorpej
188 1.1 rmind for (u_int i = 0; i < PQCNT_NCOUNTERS; i++) {
189 1.1 rmind sum->count[i] += c->count[i];
190 1.1 rmind }
191 1.11 thorpej
192 1.11 thorpej splx(s);
193 1.1 rmind }
194 1.1 rmind
195 1.1 rmind uint64_t
196 1.1 rmind pktq_get_count(pktqueue_t *pq, pktq_count_t c)
197 1.1 rmind {
198 1.1 rmind pktq_counters_t sum;
199 1.1 rmind
200 1.1 rmind if (c != PKTQ_MAXLEN) {
201 1.1 rmind memset(&sum, 0, sizeof(sum));
202 1.11 thorpej percpu_foreach_xcall(pq->pq_counters,
203 1.11 thorpej XC_HIGHPRI_IPL(IPL_SOFTNET), pktq_collect_counts, &sum);
204 1.1 rmind }
205 1.1 rmind switch (c) {
206 1.1 rmind case PKTQ_NITEMS:
207 1.1 rmind return sum.count[PQCNT_ENQUEUE] - sum.count[PQCNT_DEQUEUE];
208 1.1 rmind case PKTQ_DROPS:
209 1.1 rmind return sum.count[PQCNT_DROP];
210 1.1 rmind case PKTQ_MAXLEN:
211 1.1 rmind return pq->pq_maxlen;
212 1.1 rmind }
213 1.1 rmind return 0;
214 1.1 rmind }
215 1.1 rmind
216 1.1 rmind uint32_t
217 1.1 rmind pktq_rps_hash(const struct mbuf *m __unused)
218 1.1 rmind {
219 1.1 rmind /*
220 1.1 rmind * XXX: No distribution yet; the softnet_lock contention
221 1.1 rmind * XXX: must be eliminated first.
222 1.1 rmind */
223 1.1 rmind return 0;
224 1.1 rmind }
225 1.1 rmind
226 1.1 rmind /*
227 1.1 rmind * pktq_enqueue: inject the packet into the end of the queue.
228 1.1 rmind *
229 1.1 rmind * => Must be called from the interrupt or with the preemption disabled.
230 1.1 rmind * => Consumes the packet and returns true on success.
231 1.1 rmind * => Returns false on failure; caller is responsible to free the packet.
232 1.1 rmind */
233 1.1 rmind bool
234 1.3 rmind pktq_enqueue(pktqueue_t *pq, struct mbuf *m, const u_int hash __unused)
235 1.1 rmind {
236 1.8 ozaki #if defined(_RUMPKERNEL) || defined(_RUMP_NATIVE_ABI)
237 1.12 riastrad struct cpu_info *ci = curcpu();
238 1.7 ozaki #else
239 1.12 riastrad struct cpu_info *ci = cpu_lookup(hash % ncpu);
240 1.7 ozaki #endif
241 1.1 rmind
242 1.1 rmind KASSERT(kpreempt_disabled());
243 1.1 rmind
244 1.12 riastrad if (__predict_false(!pcq_put(pktq_pcq(pq, ci), m))) {
245 1.1 rmind pktq_inc_count(pq, PQCNT_DROP);
246 1.1 rmind return false;
247 1.1 rmind }
248 1.12 riastrad softint_schedule_cpu(pq->pq_sih, ci);
249 1.1 rmind pktq_inc_count(pq, PQCNT_ENQUEUE);
250 1.1 rmind return true;
251 1.1 rmind }
252 1.1 rmind
253 1.1 rmind /*
254 1.1 rmind * pktq_dequeue: take a packet from the queue.
255 1.1 rmind *
256 1.1 rmind * => Must be called with preemption disabled.
257 1.1 rmind * => Must ensure there are not concurrent dequeue calls.
258 1.1 rmind */
259 1.1 rmind struct mbuf *
260 1.1 rmind pktq_dequeue(pktqueue_t *pq)
261 1.1 rmind {
262 1.12 riastrad struct cpu_info *ci = curcpu();
263 1.1 rmind struct mbuf *m;
264 1.1 rmind
265 1.12 riastrad KASSERT(kpreempt_disabled());
266 1.12 riastrad
267 1.12 riastrad m = pcq_get(pktq_pcq(pq, ci));
268 1.1 rmind if (__predict_false(m == PKTQ_MARKER)) {
269 1.1 rmind /* Note the marker entry. */
270 1.1 rmind atomic_inc_uint(&pq->pq_barrier);
271 1.1 rmind return NULL;
272 1.1 rmind }
273 1.1 rmind if (__predict_true(m != NULL)) {
274 1.1 rmind pktq_inc_count(pq, PQCNT_DEQUEUE);
275 1.1 rmind }
276 1.1 rmind return m;
277 1.1 rmind }
278 1.1 rmind
279 1.1 rmind /*
280 1.1 rmind * pktq_barrier: waits for a grace period when all packets enqueued at
281 1.1 rmind * the moment of calling this routine will be processed. This is used
282 1.1 rmind * to ensure that e.g. packets referencing some interface were drained.
283 1.1 rmind */
284 1.1 rmind void
285 1.1 rmind pktq_barrier(pktqueue_t *pq)
286 1.1 rmind {
287 1.12 riastrad CPU_INFO_ITERATOR cii;
288 1.12 riastrad struct cpu_info *ci;
289 1.1 rmind u_int pending = 0;
290 1.1 rmind
291 1.1 rmind mutex_enter(&pq->pq_lock);
292 1.1 rmind KASSERT(pq->pq_barrier == 0);
293 1.1 rmind
294 1.12 riastrad for (CPU_INFO_FOREACH(cii, ci)) {
295 1.12 riastrad struct pcq *q;
296 1.12 riastrad
297 1.12 riastrad kpreempt_disable();
298 1.12 riastrad q = pktq_pcq(pq, ci);
299 1.12 riastrad kpreempt_enable();
300 1.1 rmind
301 1.1 rmind /* If the queue is empty - nothing to do. */
302 1.1 rmind if (pcq_peek(q) == NULL) {
303 1.1 rmind continue;
304 1.1 rmind }
305 1.1 rmind /* Otherwise, put the marker and entry. */
306 1.1 rmind while (!pcq_put(q, PKTQ_MARKER)) {
307 1.1 rmind kpause("pktqsync", false, 1, NULL);
308 1.1 rmind }
309 1.1 rmind kpreempt_disable();
310 1.12 riastrad softint_schedule_cpu(pq->pq_sih, ci);
311 1.1 rmind kpreempt_enable();
312 1.1 rmind pending++;
313 1.1 rmind }
314 1.1 rmind
315 1.1 rmind /* Wait for each queue to process the markers. */
316 1.1 rmind while (pq->pq_barrier != pending) {
317 1.1 rmind kpause("pktqsync", false, 1, NULL);
318 1.1 rmind }
319 1.1 rmind pq->pq_barrier = 0;
320 1.1 rmind mutex_exit(&pq->pq_lock);
321 1.1 rmind }
322 1.1 rmind
323 1.1 rmind /*
324 1.1 rmind * pktq_flush: free mbufs in all queues.
325 1.1 rmind *
326 1.4 rmind * => The caller must ensure there are no concurrent writers or flush calls.
327 1.1 rmind */
328 1.1 rmind void
329 1.1 rmind pktq_flush(pktqueue_t *pq)
330 1.1 rmind {
331 1.12 riastrad CPU_INFO_ITERATOR cii;
332 1.12 riastrad struct cpu_info *ci;
333 1.1 rmind struct mbuf *m;
334 1.1 rmind
335 1.12 riastrad for (CPU_INFO_FOREACH(cii, ci)) {
336 1.12 riastrad struct pcq *q;
337 1.12 riastrad
338 1.12 riastrad kpreempt_disable();
339 1.12 riastrad q = pktq_pcq(pq, ci);
340 1.12 riastrad kpreempt_enable();
341 1.12 riastrad
342 1.12 riastrad /*
343 1.12 riastrad * XXX This can't be right -- if the softint is running
344 1.12 riastrad * then pcq_get isn't safe here.
345 1.12 riastrad */
346 1.12 riastrad while ((m = pcq_get(q)) != NULL) {
347 1.1 rmind pktq_inc_count(pq, PQCNT_DEQUEUE);
348 1.1 rmind m_freem(m);
349 1.1 rmind }
350 1.1 rmind }
351 1.1 rmind }
352 1.2 rmind
353 1.12 riastrad static void
354 1.12 riastrad pktq_set_maxlen_cpu(void *vpq, void *vqs)
355 1.12 riastrad {
356 1.12 riastrad struct pktqueue *pq = vpq;
357 1.12 riastrad struct pcq **qp, *q, **qs = vqs;
358 1.12 riastrad unsigned i = cpu_index(curcpu());
359 1.12 riastrad int s;
360 1.12 riastrad
361 1.12 riastrad s = splnet();
362 1.12 riastrad qp = percpu_getref(pq->pq_pcq);
363 1.12 riastrad q = *qp;
364 1.12 riastrad *qp = qs[i];
365 1.12 riastrad qs[i] = q;
366 1.12 riastrad percpu_putref(pq->pq_pcq);
367 1.12 riastrad splx(s);
368 1.12 riastrad }
369 1.12 riastrad
370 1.2 rmind /*
371 1.2 rmind * pktq_set_maxlen: create per-CPU queues using a new size and replace
372 1.2 rmind * the existing queues without losing any packets.
373 1.12 riastrad *
374 1.12 riastrad * XXX ncpu must remain stable throughout.
375 1.2 rmind */
376 1.2 rmind int
377 1.2 rmind pktq_set_maxlen(pktqueue_t *pq, size_t maxlen)
378 1.2 rmind {
379 1.2 rmind const u_int slotbytes = ncpu * sizeof(pcq_t *);
380 1.2 rmind pcq_t **qs;
381 1.2 rmind
382 1.2 rmind if (!maxlen || maxlen > PCQ_MAXLEN)
383 1.2 rmind return EINVAL;
384 1.2 rmind if (pq->pq_maxlen == maxlen)
385 1.2 rmind return 0;
386 1.2 rmind
387 1.12 riastrad /* First, allocate the new queues. */
388 1.2 rmind qs = kmem_zalloc(slotbytes, KM_SLEEP);
389 1.2 rmind for (u_int i = 0; i < ncpu; i++) {
390 1.2 rmind qs[i] = pcq_create(maxlen, KM_SLEEP);
391 1.2 rmind }
392 1.12 riastrad
393 1.12 riastrad /*
394 1.12 riastrad * Issue an xcall to replace the queue pointers on each CPU.
395 1.12 riastrad * This implies all the necessary memory barriers.
396 1.12 riastrad */
397 1.2 rmind mutex_enter(&pq->pq_lock);
398 1.12 riastrad xc_wait(xc_broadcast(XC_HIGHPRI, pktq_set_maxlen_cpu, pq, qs));
399 1.2 rmind pq->pq_maxlen = maxlen;
400 1.2 rmind mutex_exit(&pq->pq_lock);
401 1.2 rmind
402 1.2 rmind /*
403 1.2 rmind * At this point, the new packets are flowing into the new
404 1.4 rmind * queues. However, the old queues may have some packets
405 1.4 rmind * present which are no longer being processed. We are going
406 1.2 rmind * to re-enqueue them. This may change the order of packet
407 1.2 rmind * arrival, but it is not considered an issue.
408 1.2 rmind *
409 1.4 rmind * There may be in-flight interrupts calling pktq_dequeue()
410 1.2 rmind * which reference the old queues. Issue a barrier to ensure
411 1.2 rmind * that we are going to be the only pcq_get() callers on the
412 1.2 rmind * old queues.
413 1.2 rmind */
414 1.2 rmind pktq_barrier(pq);
415 1.2 rmind
416 1.2 rmind for (u_int i = 0; i < ncpu; i++) {
417 1.12 riastrad struct pcq *q;
418 1.2 rmind struct mbuf *m;
419 1.2 rmind
420 1.12 riastrad kpreempt_disable();
421 1.12 riastrad q = pktq_pcq(pq, cpu_lookup(i));
422 1.12 riastrad kpreempt_enable();
423 1.12 riastrad
424 1.2 rmind while ((m = pcq_get(qs[i])) != NULL) {
425 1.12 riastrad while (!pcq_put(q, m)) {
426 1.2 rmind kpause("pktqrenq", false, 1, NULL);
427 1.2 rmind }
428 1.2 rmind }
429 1.2 rmind pcq_destroy(qs[i]);
430 1.2 rmind }
431 1.2 rmind
432 1.2 rmind /* Well, that was fun. */
433 1.2 rmind kmem_free(qs, slotbytes);
434 1.2 rmind return 0;
435 1.2 rmind }
436 1.6 ozaki
437 1.6 ozaki int
438 1.6 ozaki sysctl_pktq_maxlen(SYSCTLFN_ARGS, pktqueue_t *pq)
439 1.6 ozaki {
440 1.6 ozaki u_int nmaxlen = pktq_get_count(pq, PKTQ_MAXLEN);
441 1.6 ozaki struct sysctlnode node = *rnode;
442 1.6 ozaki int error;
443 1.6 ozaki
444 1.6 ozaki node.sysctl_data = &nmaxlen;
445 1.6 ozaki error = sysctl_lookup(SYSCTLFN_CALL(&node));
446 1.6 ozaki if (error || newp == NULL)
447 1.6 ozaki return error;
448 1.6 ozaki return pktq_set_maxlen(pq, nmaxlen);
449 1.6 ozaki }
450 1.6 ozaki
451 1.6 ozaki int
452 1.6 ozaki sysctl_pktq_count(SYSCTLFN_ARGS, pktqueue_t *pq, u_int count_id)
453 1.6 ozaki {
454 1.10 msaitoh uint64_t count = pktq_get_count(pq, count_id);
455 1.6 ozaki struct sysctlnode node = *rnode;
456 1.10 msaitoh
457 1.6 ozaki node.sysctl_data = &count;
458 1.6 ozaki return sysctl_lookup(SYSCTLFN_CALL(&node));
459 1.6 ozaki }
460