Home | History | Annotate | Line # | Download | only in net
pktqueue.c revision 1.13
      1  1.13     skrll /*	$NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $	*/
      2   1.1     rmind 
      3   1.1     rmind /*-
      4   1.1     rmind  * Copyright (c) 2014 The NetBSD Foundation, Inc.
      5   1.1     rmind  * All rights reserved.
      6   1.1     rmind  *
      7   1.1     rmind  * This code is derived from software contributed to The NetBSD Foundation
      8   1.1     rmind  * by Mindaugas Rasiukevicius.
      9   1.1     rmind  *
     10   1.1     rmind  * Redistribution and use in source and binary forms, with or without
     11   1.1     rmind  * modification, are permitted provided that the following conditions
     12   1.1     rmind  * are met:
     13   1.1     rmind  * 1. Redistributions of source code must retain the above copyright
     14   1.1     rmind  *    notice, this list of conditions and the following disclaimer.
     15   1.1     rmind  * 2. Redistributions in binary form must reproduce the above copyright
     16   1.1     rmind  *    notice, this list of conditions and the following disclaimer in the
     17   1.1     rmind  *    documentation and/or other materials provided with the distribution.
     18   1.1     rmind  *
     19   1.1     rmind  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
     20   1.1     rmind  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
     21   1.1     rmind  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
     22   1.1     rmind  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
     23   1.1     rmind  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
     24   1.1     rmind  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
     25   1.1     rmind  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
     26   1.1     rmind  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
     27   1.1     rmind  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
     28   1.1     rmind  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
     29   1.1     rmind  * POSSIBILITY OF SUCH DAMAGE.
     30   1.1     rmind  */
     31   1.1     rmind 
     32   1.4     rmind /*
     33   1.4     rmind  * The packet queue (pktqueue) interface is a lockless IP input queue
     34   1.4     rmind  * which also abstracts and handles network ISR scheduling.  It provides
     35   1.4     rmind  * a mechanism to enable receiver-side packet steering (RPS).
     36   1.4     rmind  */
     37   1.4     rmind 
     38   1.1     rmind #include <sys/cdefs.h>
     39  1.13     skrll __KERNEL_RCSID(0, "$NetBSD: pktqueue.c,v 1.13 2021/03/25 08:18:03 skrll Exp $");
     40   1.1     rmind 
     41   1.1     rmind #include <sys/param.h>
     42   1.1     rmind #include <sys/types.h>
     43   1.1     rmind 
     44   1.1     rmind #include <sys/atomic.h>
     45   1.1     rmind #include <sys/cpu.h>
     46   1.1     rmind #include <sys/pcq.h>
     47   1.1     rmind #include <sys/intr.h>
     48   1.1     rmind #include <sys/mbuf.h>
     49   1.1     rmind #include <sys/proc.h>
     50   1.1     rmind #include <sys/percpu.h>
     51  1.11   thorpej #include <sys/xcall.h>
     52   1.1     rmind 
     53   1.1     rmind #include <net/pktqueue.h>
     54   1.1     rmind 
     55   1.1     rmind struct pktqueue {
     56   1.1     rmind 	/*
     57   1.1     rmind 	 * The lock used for a barrier mechanism.  The barrier counter,
     58   1.1     rmind 	 * as well as the drop counter, are managed atomically though.
     59   1.1     rmind 	 * Ensure this group is in a separate cache line.
     60   1.1     rmind 	 */
     61  1.13     skrll 	union {
     62  1.13     skrll 		struct {
     63  1.13     skrll 			kmutex_t	pq_lock;
     64  1.13     skrll 			volatile u_int	pq_barrier;
     65  1.13     skrll 		};
     66  1.13     skrll 		uint8_t	 _pad[COHERENCY_UNIT];
     67  1.13     skrll 	};
     68   1.1     rmind 
     69   1.1     rmind 	/* The size of the queue, counters and the interrupt handler. */
     70   1.1     rmind 	u_int		pq_maxlen;
     71   1.1     rmind 	percpu_t *	pq_counters;
     72   1.1     rmind 	void *		pq_sih;
     73   1.1     rmind 
     74   1.1     rmind 	/* Finally, per-CPU queues. */
     75  1.12  riastrad 	struct percpu *	pq_pcq;	/* struct pcq * */
     76   1.1     rmind };
     77   1.1     rmind 
     78   1.1     rmind /* The counters of the packet queue. */
     79   1.1     rmind #define	PQCNT_ENQUEUE	0
     80   1.1     rmind #define	PQCNT_DEQUEUE	1
     81   1.1     rmind #define	PQCNT_DROP	2
     82   1.1     rmind #define	PQCNT_NCOUNTERS	3
     83   1.1     rmind 
     84   1.1     rmind typedef struct {
     85   1.1     rmind 	uint64_t	count[PQCNT_NCOUNTERS];
     86   1.1     rmind } pktq_counters_t;
     87   1.1     rmind 
     88   1.1     rmind /* Special marker value used by pktq_barrier() mechanism. */
     89   1.1     rmind #define	PKTQ_MARKER	((void *)(~0ULL))
     90   1.1     rmind 
     91  1.12  riastrad static void
     92  1.12  riastrad pktq_init_cpu(void *vqp, void *vpq, struct cpu_info *ci)
     93  1.12  riastrad {
     94  1.12  riastrad 	struct pcq **qp = vqp;
     95  1.12  riastrad 	struct pktqueue *pq = vpq;
     96  1.12  riastrad 
     97  1.12  riastrad 	*qp = pcq_create(pq->pq_maxlen, KM_SLEEP);
     98  1.12  riastrad }
     99  1.12  riastrad 
    100  1.12  riastrad static void
    101  1.12  riastrad pktq_fini_cpu(void *vqp, void *vpq, struct cpu_info *ci)
    102  1.12  riastrad {
    103  1.12  riastrad 	struct pcq **qp = vqp, *q = *qp;
    104  1.12  riastrad 
    105  1.12  riastrad 	KASSERT(pcq_peek(q) == NULL);
    106  1.12  riastrad 	pcq_destroy(q);
    107  1.12  riastrad 	*qp = NULL;		/* paranoia */
    108  1.12  riastrad }
    109  1.12  riastrad 
    110  1.12  riastrad static struct pcq *
    111  1.12  riastrad pktq_pcq(struct pktqueue *pq, struct cpu_info *ci)
    112  1.12  riastrad {
    113  1.12  riastrad 	struct pcq **qp, *q;
    114  1.12  riastrad 
    115  1.12  riastrad 	/*
    116  1.12  riastrad 	 * As long as preemption is disabled, the xcall to swap percpu
    117  1.12  riastrad 	 * buffers can't complete, so it is safe to read the pointer.
    118  1.12  riastrad 	 */
    119  1.12  riastrad 	KASSERT(kpreempt_disabled());
    120  1.12  riastrad 
    121  1.12  riastrad 	qp = percpu_getptr_remote(pq->pq_pcq, ci);
    122  1.12  riastrad 	q = *qp;
    123  1.12  riastrad 
    124  1.12  riastrad 	return q;
    125  1.12  riastrad }
    126   1.1     rmind 
    127   1.1     rmind pktqueue_t *
    128   1.5     ozaki pktq_create(size_t maxlen, void (*intrh)(void *), void *sc)
    129   1.1     rmind {
    130   1.1     rmind 	const u_int sflags = SOFTINT_NET | SOFTINT_MPSAFE | SOFTINT_RCPU;
    131   1.1     rmind 	pktqueue_t *pq;
    132   1.1     rmind 	percpu_t *pc;
    133   1.1     rmind 	void *sih;
    134   1.1     rmind 
    135   1.9       chs 	pc = percpu_alloc(sizeof(pktq_counters_t));
    136   1.5     ozaki 	if ((sih = softint_establish(sflags, intrh, sc)) == NULL) {
    137   1.1     rmind 		percpu_free(pc, sizeof(pktq_counters_t));
    138   1.1     rmind 		return NULL;
    139   1.1     rmind 	}
    140   1.1     rmind 
    141  1.12  riastrad 	pq = kmem_zalloc(sizeof(*pq), KM_SLEEP);
    142   1.1     rmind 	mutex_init(&pq->pq_lock, MUTEX_DEFAULT, IPL_NONE);
    143   1.1     rmind 	pq->pq_maxlen = maxlen;
    144   1.1     rmind 	pq->pq_counters = pc;
    145   1.1     rmind 	pq->pq_sih = sih;
    146  1.12  riastrad 	pq->pq_pcq = percpu_create(sizeof(struct pcq *),
    147  1.12  riastrad 	    pktq_init_cpu, pktq_fini_cpu, pq);
    148   1.1     rmind 
    149   1.1     rmind 	return pq;
    150   1.1     rmind }
    151   1.1     rmind 
    152   1.1     rmind void
    153   1.1     rmind pktq_destroy(pktqueue_t *pq)
    154   1.1     rmind {
    155   1.1     rmind 
    156  1.12  riastrad 	percpu_free(pq->pq_pcq, sizeof(struct pcq *));
    157   1.1     rmind 	percpu_free(pq->pq_counters, sizeof(pktq_counters_t));
    158   1.1     rmind 	softint_disestablish(pq->pq_sih);
    159   1.1     rmind 	mutex_destroy(&pq->pq_lock);
    160  1.12  riastrad 	kmem_free(pq, sizeof(*pq));
    161   1.1     rmind }
    162   1.1     rmind 
    163   1.1     rmind /*
    164   1.1     rmind  * - pktq_inc_counter: increment the counter given an ID.
    165   1.1     rmind  * - pktq_collect_counts: handler to sum up the counts from each CPU.
    166   1.1     rmind  * - pktq_getcount: return the effective count given an ID.
    167   1.1     rmind  */
    168   1.1     rmind 
    169   1.1     rmind static inline void
    170   1.1     rmind pktq_inc_count(pktqueue_t *pq, u_int i)
    171   1.1     rmind {
    172   1.1     rmind 	percpu_t *pc = pq->pq_counters;
    173   1.1     rmind 	pktq_counters_t *c;
    174   1.1     rmind 
    175   1.1     rmind 	c = percpu_getref(pc);
    176   1.1     rmind 	c->count[i]++;
    177   1.1     rmind 	percpu_putref(pc);
    178   1.1     rmind }
    179   1.1     rmind 
    180   1.1     rmind static void
    181   1.1     rmind pktq_collect_counts(void *mem, void *arg, struct cpu_info *ci)
    182   1.1     rmind {
    183   1.1     rmind 	const pktq_counters_t *c = mem;
    184   1.1     rmind 	pktq_counters_t *sum = arg;
    185   1.1     rmind 
    186  1.11   thorpej 	int s = splnet();
    187  1.11   thorpej 
    188   1.1     rmind 	for (u_int i = 0; i < PQCNT_NCOUNTERS; i++) {
    189   1.1     rmind 		sum->count[i] += c->count[i];
    190   1.1     rmind 	}
    191  1.11   thorpej 
    192  1.11   thorpej 	splx(s);
    193   1.1     rmind }
    194   1.1     rmind 
    195   1.1     rmind uint64_t
    196   1.1     rmind pktq_get_count(pktqueue_t *pq, pktq_count_t c)
    197   1.1     rmind {
    198   1.1     rmind 	pktq_counters_t sum;
    199   1.1     rmind 
    200   1.1     rmind 	if (c != PKTQ_MAXLEN) {
    201   1.1     rmind 		memset(&sum, 0, sizeof(sum));
    202  1.11   thorpej 		percpu_foreach_xcall(pq->pq_counters,
    203  1.11   thorpej 		    XC_HIGHPRI_IPL(IPL_SOFTNET), pktq_collect_counts, &sum);
    204   1.1     rmind 	}
    205   1.1     rmind 	switch (c) {
    206   1.1     rmind 	case PKTQ_NITEMS:
    207   1.1     rmind 		return sum.count[PQCNT_ENQUEUE] - sum.count[PQCNT_DEQUEUE];
    208   1.1     rmind 	case PKTQ_DROPS:
    209   1.1     rmind 		return sum.count[PQCNT_DROP];
    210   1.1     rmind 	case PKTQ_MAXLEN:
    211   1.1     rmind 		return pq->pq_maxlen;
    212   1.1     rmind 	}
    213   1.1     rmind 	return 0;
    214   1.1     rmind }
    215   1.1     rmind 
    216   1.1     rmind uint32_t
    217   1.1     rmind pktq_rps_hash(const struct mbuf *m __unused)
    218   1.1     rmind {
    219   1.1     rmind 	/*
    220   1.1     rmind 	 * XXX: No distribution yet; the softnet_lock contention
    221   1.1     rmind 	 * XXX: must be eliminated first.
    222   1.1     rmind 	 */
    223   1.1     rmind 	return 0;
    224   1.1     rmind }
    225   1.1     rmind 
    226   1.1     rmind /*
    227   1.1     rmind  * pktq_enqueue: inject the packet into the end of the queue.
    228   1.1     rmind  *
    229   1.1     rmind  * => Must be called from the interrupt or with the preemption disabled.
    230   1.1     rmind  * => Consumes the packet and returns true on success.
    231   1.1     rmind  * => Returns false on failure; caller is responsible to free the packet.
    232   1.1     rmind  */
    233   1.1     rmind bool
    234   1.3     rmind pktq_enqueue(pktqueue_t *pq, struct mbuf *m, const u_int hash __unused)
    235   1.1     rmind {
    236   1.8     ozaki #if defined(_RUMPKERNEL) || defined(_RUMP_NATIVE_ABI)
    237  1.12  riastrad 	struct cpu_info *ci = curcpu();
    238   1.7     ozaki #else
    239  1.12  riastrad 	struct cpu_info *ci = cpu_lookup(hash % ncpu);
    240   1.7     ozaki #endif
    241   1.1     rmind 
    242   1.1     rmind 	KASSERT(kpreempt_disabled());
    243   1.1     rmind 
    244  1.12  riastrad 	if (__predict_false(!pcq_put(pktq_pcq(pq, ci), m))) {
    245   1.1     rmind 		pktq_inc_count(pq, PQCNT_DROP);
    246   1.1     rmind 		return false;
    247   1.1     rmind 	}
    248  1.12  riastrad 	softint_schedule_cpu(pq->pq_sih, ci);
    249   1.1     rmind 	pktq_inc_count(pq, PQCNT_ENQUEUE);
    250   1.1     rmind 	return true;
    251   1.1     rmind }
    252   1.1     rmind 
    253   1.1     rmind /*
    254   1.1     rmind  * pktq_dequeue: take a packet from the queue.
    255   1.1     rmind  *
    256   1.1     rmind  * => Must be called with preemption disabled.
    257   1.1     rmind  * => Must ensure there are not concurrent dequeue calls.
    258   1.1     rmind  */
    259   1.1     rmind struct mbuf *
    260   1.1     rmind pktq_dequeue(pktqueue_t *pq)
    261   1.1     rmind {
    262  1.12  riastrad 	struct cpu_info *ci = curcpu();
    263   1.1     rmind 	struct mbuf *m;
    264   1.1     rmind 
    265  1.12  riastrad 	KASSERT(kpreempt_disabled());
    266  1.12  riastrad 
    267  1.12  riastrad 	m = pcq_get(pktq_pcq(pq, ci));
    268   1.1     rmind 	if (__predict_false(m == PKTQ_MARKER)) {
    269   1.1     rmind 		/* Note the marker entry. */
    270   1.1     rmind 		atomic_inc_uint(&pq->pq_barrier);
    271   1.1     rmind 		return NULL;
    272   1.1     rmind 	}
    273   1.1     rmind 	if (__predict_true(m != NULL)) {
    274   1.1     rmind 		pktq_inc_count(pq, PQCNT_DEQUEUE);
    275   1.1     rmind 	}
    276   1.1     rmind 	return m;
    277   1.1     rmind }
    278   1.1     rmind 
    279   1.1     rmind /*
    280   1.1     rmind  * pktq_barrier: waits for a grace period when all packets enqueued at
    281   1.1     rmind  * the moment of calling this routine will be processed.  This is used
    282   1.1     rmind  * to ensure that e.g. packets referencing some interface were drained.
    283   1.1     rmind  */
    284   1.1     rmind void
    285   1.1     rmind pktq_barrier(pktqueue_t *pq)
    286   1.1     rmind {
    287  1.12  riastrad 	CPU_INFO_ITERATOR cii;
    288  1.12  riastrad 	struct cpu_info *ci;
    289   1.1     rmind 	u_int pending = 0;
    290   1.1     rmind 
    291   1.1     rmind 	mutex_enter(&pq->pq_lock);
    292   1.1     rmind 	KASSERT(pq->pq_barrier == 0);
    293   1.1     rmind 
    294  1.12  riastrad 	for (CPU_INFO_FOREACH(cii, ci)) {
    295  1.12  riastrad 		struct pcq *q;
    296  1.12  riastrad 
    297  1.12  riastrad 		kpreempt_disable();
    298  1.12  riastrad 		q = pktq_pcq(pq, ci);
    299  1.12  riastrad 		kpreempt_enable();
    300   1.1     rmind 
    301   1.1     rmind 		/* If the queue is empty - nothing to do. */
    302   1.1     rmind 		if (pcq_peek(q) == NULL) {
    303   1.1     rmind 			continue;
    304   1.1     rmind 		}
    305   1.1     rmind 		/* Otherwise, put the marker and entry. */
    306   1.1     rmind 		while (!pcq_put(q, PKTQ_MARKER)) {
    307   1.1     rmind 			kpause("pktqsync", false, 1, NULL);
    308   1.1     rmind 		}
    309   1.1     rmind 		kpreempt_disable();
    310  1.12  riastrad 		softint_schedule_cpu(pq->pq_sih, ci);
    311   1.1     rmind 		kpreempt_enable();
    312   1.1     rmind 		pending++;
    313   1.1     rmind 	}
    314   1.1     rmind 
    315   1.1     rmind 	/* Wait for each queue to process the markers. */
    316   1.1     rmind 	while (pq->pq_barrier != pending) {
    317   1.1     rmind 		kpause("pktqsync", false, 1, NULL);
    318   1.1     rmind 	}
    319   1.1     rmind 	pq->pq_barrier = 0;
    320   1.1     rmind 	mutex_exit(&pq->pq_lock);
    321   1.1     rmind }
    322   1.1     rmind 
    323   1.1     rmind /*
    324   1.1     rmind  * pktq_flush: free mbufs in all queues.
    325   1.1     rmind  *
    326   1.4     rmind  * => The caller must ensure there are no concurrent writers or flush calls.
    327   1.1     rmind  */
    328   1.1     rmind void
    329   1.1     rmind pktq_flush(pktqueue_t *pq)
    330   1.1     rmind {
    331  1.12  riastrad 	CPU_INFO_ITERATOR cii;
    332  1.12  riastrad 	struct cpu_info *ci;
    333   1.1     rmind 	struct mbuf *m;
    334   1.1     rmind 
    335  1.12  riastrad 	for (CPU_INFO_FOREACH(cii, ci)) {
    336  1.12  riastrad 		struct pcq *q;
    337  1.12  riastrad 
    338  1.12  riastrad 		kpreempt_disable();
    339  1.12  riastrad 		q = pktq_pcq(pq, ci);
    340  1.12  riastrad 		kpreempt_enable();
    341  1.12  riastrad 
    342  1.12  riastrad 		/*
    343  1.12  riastrad 		 * XXX This can't be right -- if the softint is running
    344  1.12  riastrad 		 * then pcq_get isn't safe here.
    345  1.12  riastrad 		 */
    346  1.12  riastrad 		while ((m = pcq_get(q)) != NULL) {
    347   1.1     rmind 			pktq_inc_count(pq, PQCNT_DEQUEUE);
    348   1.1     rmind 			m_freem(m);
    349   1.1     rmind 		}
    350   1.1     rmind 	}
    351   1.1     rmind }
    352   1.2     rmind 
    353  1.12  riastrad static void
    354  1.12  riastrad pktq_set_maxlen_cpu(void *vpq, void *vqs)
    355  1.12  riastrad {
    356  1.12  riastrad 	struct pktqueue *pq = vpq;
    357  1.12  riastrad 	struct pcq **qp, *q, **qs = vqs;
    358  1.12  riastrad 	unsigned i = cpu_index(curcpu());
    359  1.12  riastrad 	int s;
    360  1.12  riastrad 
    361  1.12  riastrad 	s = splnet();
    362  1.12  riastrad 	qp = percpu_getref(pq->pq_pcq);
    363  1.12  riastrad 	q = *qp;
    364  1.12  riastrad 	*qp = qs[i];
    365  1.12  riastrad 	qs[i] = q;
    366  1.12  riastrad 	percpu_putref(pq->pq_pcq);
    367  1.12  riastrad 	splx(s);
    368  1.12  riastrad }
    369  1.12  riastrad 
    370   1.2     rmind /*
    371   1.2     rmind  * pktq_set_maxlen: create per-CPU queues using a new size and replace
    372   1.2     rmind  * the existing queues without losing any packets.
    373  1.12  riastrad  *
    374  1.12  riastrad  * XXX ncpu must remain stable throughout.
    375   1.2     rmind  */
    376   1.2     rmind int
    377   1.2     rmind pktq_set_maxlen(pktqueue_t *pq, size_t maxlen)
    378   1.2     rmind {
    379   1.2     rmind 	const u_int slotbytes = ncpu * sizeof(pcq_t *);
    380   1.2     rmind 	pcq_t **qs;
    381   1.2     rmind 
    382   1.2     rmind 	if (!maxlen || maxlen > PCQ_MAXLEN)
    383   1.2     rmind 		return EINVAL;
    384   1.2     rmind 	if (pq->pq_maxlen == maxlen)
    385   1.2     rmind 		return 0;
    386   1.2     rmind 
    387  1.12  riastrad 	/* First, allocate the new queues. */
    388   1.2     rmind 	qs = kmem_zalloc(slotbytes, KM_SLEEP);
    389   1.2     rmind 	for (u_int i = 0; i < ncpu; i++) {
    390   1.2     rmind 		qs[i] = pcq_create(maxlen, KM_SLEEP);
    391   1.2     rmind 	}
    392  1.12  riastrad 
    393  1.12  riastrad 	/*
    394  1.12  riastrad 	 * Issue an xcall to replace the queue pointers on each CPU.
    395  1.12  riastrad 	 * This implies all the necessary memory barriers.
    396  1.12  riastrad 	 */
    397   1.2     rmind 	mutex_enter(&pq->pq_lock);
    398  1.12  riastrad 	xc_wait(xc_broadcast(XC_HIGHPRI, pktq_set_maxlen_cpu, pq, qs));
    399   1.2     rmind 	pq->pq_maxlen = maxlen;
    400   1.2     rmind 	mutex_exit(&pq->pq_lock);
    401   1.2     rmind 
    402   1.2     rmind 	/*
    403   1.2     rmind 	 * At this point, the new packets are flowing into the new
    404   1.4     rmind 	 * queues.  However, the old queues may have some packets
    405   1.4     rmind 	 * present which are no longer being processed.  We are going
    406   1.2     rmind 	 * to re-enqueue them.  This may change the order of packet
    407   1.2     rmind 	 * arrival, but it is not considered an issue.
    408   1.2     rmind 	 *
    409   1.4     rmind 	 * There may be in-flight interrupts calling pktq_dequeue()
    410   1.2     rmind 	 * which reference the old queues.  Issue a barrier to ensure
    411   1.2     rmind 	 * that we are going to be the only pcq_get() callers on the
    412   1.2     rmind 	 * old queues.
    413   1.2     rmind 	 */
    414   1.2     rmind 	pktq_barrier(pq);
    415   1.2     rmind 
    416   1.2     rmind 	for (u_int i = 0; i < ncpu; i++) {
    417  1.12  riastrad 		struct pcq *q;
    418   1.2     rmind 		struct mbuf *m;
    419   1.2     rmind 
    420  1.12  riastrad 		kpreempt_disable();
    421  1.12  riastrad 		q = pktq_pcq(pq, cpu_lookup(i));
    422  1.12  riastrad 		kpreempt_enable();
    423  1.12  riastrad 
    424   1.2     rmind 		while ((m = pcq_get(qs[i])) != NULL) {
    425  1.12  riastrad 			while (!pcq_put(q, m)) {
    426   1.2     rmind 				kpause("pktqrenq", false, 1, NULL);
    427   1.2     rmind 			}
    428   1.2     rmind 		}
    429   1.2     rmind 		pcq_destroy(qs[i]);
    430   1.2     rmind 	}
    431   1.2     rmind 
    432   1.2     rmind 	/* Well, that was fun. */
    433   1.2     rmind 	kmem_free(qs, slotbytes);
    434   1.2     rmind 	return 0;
    435   1.2     rmind }
    436   1.6     ozaki 
    437   1.6     ozaki int
    438   1.6     ozaki sysctl_pktq_maxlen(SYSCTLFN_ARGS, pktqueue_t *pq)
    439   1.6     ozaki {
    440   1.6     ozaki 	u_int nmaxlen = pktq_get_count(pq, PKTQ_MAXLEN);
    441   1.6     ozaki 	struct sysctlnode node = *rnode;
    442   1.6     ozaki 	int error;
    443   1.6     ozaki 
    444   1.6     ozaki 	node.sysctl_data = &nmaxlen;
    445   1.6     ozaki 	error = sysctl_lookup(SYSCTLFN_CALL(&node));
    446   1.6     ozaki 	if (error || newp == NULL)
    447   1.6     ozaki 		return error;
    448   1.6     ozaki 	return pktq_set_maxlen(pq, nmaxlen);
    449   1.6     ozaki }
    450   1.6     ozaki 
    451   1.6     ozaki int
    452   1.6     ozaki sysctl_pktq_count(SYSCTLFN_ARGS, pktqueue_t *pq, u_int count_id)
    453   1.6     ozaki {
    454  1.10   msaitoh 	uint64_t count = pktq_get_count(pq, count_id);
    455   1.6     ozaki 	struct sysctlnode node = *rnode;
    456  1.10   msaitoh 
    457   1.6     ozaki 	node.sysctl_data = &count;
    458   1.6     ozaki 	return sysctl_lookup(SYSCTLFN_CALL(&node));
    459   1.6     ozaki }
    460