if_vioif.c revision 1.112 1 1.112 riastrad /* $NetBSD: if_vioif.c,v 1.112 2024/06/29 12:11:12 riastradh Exp $ */
2 1.1 hannken
3 1.1 hannken /*
4 1.66 reinoud * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 1.1 hannken * Copyright (c) 2010 Minoura Makoto.
6 1.1 hannken * All rights reserved.
7 1.1 hannken *
8 1.1 hannken * Redistribution and use in source and binary forms, with or without
9 1.1 hannken * modification, are permitted provided that the following conditions
10 1.1 hannken * are met:
11 1.1 hannken * 1. Redistributions of source code must retain the above copyright
12 1.1 hannken * notice, this list of conditions and the following disclaimer.
13 1.1 hannken * 2. Redistributions in binary form must reproduce the above copyright
14 1.1 hannken * notice, this list of conditions and the following disclaimer in the
15 1.1 hannken * documentation and/or other materials provided with the distribution.
16 1.1 hannken *
17 1.1 hannken * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 1.1 hannken * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 1.1 hannken * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 1.1 hannken * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 1.1 hannken * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 1.1 hannken * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 1.1 hannken * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 1.1 hannken * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 1.1 hannken * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 1.1 hannken * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 1.1 hannken */
28 1.1 hannken
29 1.1 hannken #include <sys/cdefs.h>
30 1.112 riastrad __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.112 2024/06/29 12:11:12 riastradh Exp $");
31 1.15 ozaki
32 1.15 ozaki #ifdef _KERNEL_OPT
33 1.15 ozaki #include "opt_net_mpsafe.h"
34 1.15 ozaki #endif
35 1.1 hannken
36 1.1 hannken #include <sys/param.h>
37 1.1 hannken #include <sys/systm.h>
38 1.1 hannken #include <sys/kernel.h>
39 1.55 yamaguch #include <sys/atomic.h>
40 1.1 hannken #include <sys/bus.h>
41 1.1 hannken #include <sys/condvar.h>
42 1.1 hannken #include <sys/device.h>
43 1.63 yamaguch #include <sys/evcnt.h>
44 1.1 hannken #include <sys/intr.h>
45 1.1 hannken #include <sys/kmem.h>
46 1.1 hannken #include <sys/mbuf.h>
47 1.1 hannken #include <sys/mutex.h>
48 1.1 hannken #include <sys/sockio.h>
49 1.71 yamaguch #include <sys/syslog.h>
50 1.12 ozaki #include <sys/cpu.h>
51 1.26 pgoyette #include <sys/module.h>
52 1.46 yamaguch #include <sys/pcq.h>
53 1.55 yamaguch #include <sys/workqueue.h>
54 1.85 yamaguch #include <sys/xcall.h>
55 1.1 hannken
56 1.1 hannken #include <dev/pci/virtioreg.h>
57 1.1 hannken #include <dev/pci/virtiovar.h>
58 1.1 hannken
59 1.1 hannken #include <net/if.h>
60 1.74 yamaguch #include <net/if_dl.h>
61 1.1 hannken #include <net/if_media.h>
62 1.1 hannken #include <net/if_ether.h>
63 1.1 hannken
64 1.1 hannken #include <net/bpf.h>
65 1.1 hannken
66 1.26 pgoyette #include "ioconf.h"
67 1.1 hannken
68 1.7 ozaki #ifdef NET_MPSAFE
69 1.7 ozaki #define VIOIF_MPSAFE 1
70 1.46 yamaguch #define VIOIF_MULTIQ 1
71 1.7 ozaki #endif
72 1.7 ozaki
73 1.1 hannken /*
74 1.1 hannken * if_vioifreg.h:
75 1.1 hannken */
76 1.1 hannken /* Configuration registers */
77 1.66 reinoud #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
78 1.66 reinoud #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
79 1.66 reinoud #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
80 1.66 reinoud #define VIRTIO_NET_CONFIG_MTU 10 /* 16bit */
81 1.1 hannken
82 1.1 hannken /* Feature bits */
83 1.46 yamaguch #define VIRTIO_NET_F_CSUM __BIT(0)
84 1.46 yamaguch #define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
85 1.46 yamaguch #define VIRTIO_NET_F_MAC __BIT(5)
86 1.46 yamaguch #define VIRTIO_NET_F_GSO __BIT(6)
87 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
88 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
89 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ECN __BIT(9)
90 1.46 yamaguch #define VIRTIO_NET_F_GUEST_UFO __BIT(10)
91 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
92 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
93 1.46 yamaguch #define VIRTIO_NET_F_HOST_ECN __BIT(13)
94 1.46 yamaguch #define VIRTIO_NET_F_HOST_UFO __BIT(14)
95 1.46 yamaguch #define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
96 1.46 yamaguch #define VIRTIO_NET_F_STATUS __BIT(16)
97 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VQ __BIT(17)
98 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX __BIT(18)
99 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
100 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
101 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
102 1.46 yamaguch #define VIRTIO_NET_F_MQ __BIT(22)
103 1.78 yamaguch #define VIRTIO_NET_F_CTRL_MAC_ADDR __BIT(23)
104 1.1 hannken
105 1.79 uwe #define VIRTIO_NET_FLAG_BITS \
106 1.79 uwe VIRTIO_COMMON_FLAG_BITS \
107 1.79 uwe "b\x17" "CTRL_MAC\0" \
108 1.79 uwe "b\x16" "MQ\0" \
109 1.79 uwe "b\x15" "GUEST_ANNOUNCE\0" \
110 1.79 uwe "b\x14" "CTRL_RX_EXTRA\0" \
111 1.79 uwe "b\x13" "CTRL_VLAN\0" \
112 1.79 uwe "b\x12" "CTRL_RX\0" \
113 1.79 uwe "b\x11" "CTRL_VQ\0" \
114 1.79 uwe "b\x10" "STATUS\0" \
115 1.79 uwe "b\x0f" "MRG_RXBUF\0" \
116 1.79 uwe "b\x0e" "HOST_UFO\0" \
117 1.79 uwe "b\x0d" "HOST_ECN\0" \
118 1.79 uwe "b\x0c" "HOST_TSO6\0" \
119 1.79 uwe "b\x0b" "HOST_TSO4\0" \
120 1.79 uwe "b\x0a" "GUEST_UFO\0" \
121 1.79 uwe "b\x09" "GUEST_ECN\0" \
122 1.79 uwe "b\x08" "GUEST_TSO6\0" \
123 1.79 uwe "b\x07" "GUEST_TSO4\0" \
124 1.79 uwe "b\x06" "GSO\0" \
125 1.79 uwe "b\x05" "MAC\0" \
126 1.79 uwe "b\x01" "GUEST_CSUM\0" \
127 1.79 uwe "b\x00" "CSUM\0"
128 1.18 christos
129 1.1 hannken /* Status */
130 1.1 hannken #define VIRTIO_NET_S_LINK_UP 1
131 1.1 hannken
132 1.1 hannken /* Packet header structure */
133 1.1 hannken struct virtio_net_hdr {
134 1.1 hannken uint8_t flags;
135 1.1 hannken uint8_t gso_type;
136 1.1 hannken uint16_t hdr_len;
137 1.1 hannken uint16_t gso_size;
138 1.1 hannken uint16_t csum_start;
139 1.1 hannken uint16_t csum_offset;
140 1.66 reinoud
141 1.66 reinoud uint16_t num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */
142 1.1 hannken } __packed;
143 1.1 hannken
144 1.1 hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
145 1.1 hannken #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
146 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
147 1.1 hannken #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
148 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
149 1.1 hannken #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
150 1.1 hannken
151 1.1 hannken #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN)
152 1.1 hannken
153 1.1 hannken /* Control virtqueue */
154 1.1 hannken struct virtio_net_ctrl_cmd {
155 1.1 hannken uint8_t class;
156 1.1 hannken uint8_t command;
157 1.1 hannken } __packed;
158 1.1 hannken #define VIRTIO_NET_CTRL_RX 0
159 1.1 hannken # define VIRTIO_NET_CTRL_RX_PROMISC 0
160 1.1 hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI 1
161 1.1 hannken
162 1.1 hannken #define VIRTIO_NET_CTRL_MAC 1
163 1.1 hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
164 1.74 yamaguch # define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
165 1.1 hannken
166 1.1 hannken #define VIRTIO_NET_CTRL_VLAN 2
167 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_ADD 0
168 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_DEL 1
169 1.1 hannken
170 1.46 yamaguch #define VIRTIO_NET_CTRL_MQ 4
171 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
172 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
173 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
174 1.46 yamaguch
175 1.1 hannken struct virtio_net_ctrl_status {
176 1.1 hannken uint8_t ack;
177 1.1 hannken } __packed;
178 1.1 hannken #define VIRTIO_NET_OK 0
179 1.1 hannken #define VIRTIO_NET_ERR 1
180 1.1 hannken
181 1.1 hannken struct virtio_net_ctrl_rx {
182 1.1 hannken uint8_t onoff;
183 1.1 hannken } __packed;
184 1.1 hannken
185 1.1 hannken struct virtio_net_ctrl_mac_tbl {
186 1.1 hannken uint32_t nentries;
187 1.1 hannken uint8_t macs[][ETHER_ADDR_LEN];
188 1.1 hannken } __packed;
189 1.1 hannken
190 1.74 yamaguch struct virtio_net_ctrl_mac_addr {
191 1.74 yamaguch uint8_t mac[ETHER_ADDR_LEN];
192 1.74 yamaguch } __packed;
193 1.74 yamaguch
194 1.1 hannken struct virtio_net_ctrl_vlan {
195 1.1 hannken uint16_t id;
196 1.1 hannken } __packed;
197 1.1 hannken
198 1.46 yamaguch struct virtio_net_ctrl_mq {
199 1.46 yamaguch uint16_t virtqueue_pairs;
200 1.46 yamaguch } __packed;
201 1.46 yamaguch
202 1.1 hannken /*
203 1.1 hannken * if_vioifvar.h:
204 1.1 hannken */
205 1.43 yamaguch
206 1.43 yamaguch /*
207 1.43 yamaguch * Locking notes:
208 1.109 andvar * + a field in vioif_netqueue is protected by netq_lock (a spin mutex)
209 1.43 yamaguch * - more than one lock cannot be held at onece
210 1.98 yamaguch * + a field in vioif_tx_context and vioif_rx_context is also protected
211 1.98 yamaguch * by netq_lock.
212 1.43 yamaguch * + ctrlq_inuse is protected by ctrlq_wait_lock.
213 1.43 yamaguch * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
214 1.98 yamaguch * - netq_lock cannot be held along with ctrlq_wait_lock
215 1.62 yamaguch * + fields in vioif_softc except queues are protected by
216 1.62 yamaguch * sc->sc_lock(an adaptive mutex)
217 1.62 yamaguch * - the lock is held before acquisition of other locks
218 1.43 yamaguch */
219 1.43 yamaguch
220 1.66 reinoud struct vioif_ctrl_cmdspec {
221 1.66 reinoud bus_dmamap_t dmamap;
222 1.66 reinoud void *buf;
223 1.66 reinoud bus_size_t bufsize;
224 1.66 reinoud };
225 1.66 reinoud
226 1.55 yamaguch struct vioif_work {
227 1.55 yamaguch struct work cookie;
228 1.55 yamaguch void (*func)(void *);
229 1.55 yamaguch void *arg;
230 1.55 yamaguch unsigned int added;
231 1.55 yamaguch };
232 1.55 yamaguch
233 1.95 yamaguch struct vioif_net_map {
234 1.95 yamaguch struct virtio_net_hdr *vnm_hdr;
235 1.95 yamaguch bus_dmamap_t vnm_hdr_map;
236 1.95 yamaguch struct mbuf *vnm_mbuf;
237 1.95 yamaguch bus_dmamap_t vnm_mbuf_map;
238 1.95 yamaguch };
239 1.95 yamaguch
240 1.98 yamaguch #define VIOIF_NETQ_RX 0
241 1.98 yamaguch #define VIOIF_NETQ_TX 1
242 1.98 yamaguch #define VIOIF_NETQ_IDX 2
243 1.98 yamaguch #define VIOIF_NETQ_DIR(n) ((n) % VIOIF_NETQ_IDX)
244 1.98 yamaguch #define VIOIF_NETQ_PAIRIDX(n) ((n) / VIOIF_NETQ_IDX)
245 1.98 yamaguch #define VIOIF_NETQ_RXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX)
246 1.98 yamaguch #define VIOIF_NETQ_TXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX)
247 1.98 yamaguch
248 1.98 yamaguch struct vioif_netqueue {
249 1.98 yamaguch kmutex_t netq_lock;
250 1.98 yamaguch struct virtqueue *netq_vq;
251 1.98 yamaguch bool netq_stopping;
252 1.98 yamaguch bool netq_running_handle;
253 1.98 yamaguch void *netq_maps_kva;
254 1.98 yamaguch struct vioif_net_map *netq_maps;
255 1.98 yamaguch
256 1.98 yamaguch void *netq_softint;
257 1.98 yamaguch struct vioif_work netq_work;
258 1.98 yamaguch bool netq_workqueue;
259 1.98 yamaguch
260 1.98 yamaguch char netq_evgroup[32];
261 1.98 yamaguch struct evcnt netq_mbuf_load_failed;
262 1.99 yamaguch struct evcnt netq_enqueue_failed;
263 1.43 yamaguch
264 1.98 yamaguch void *netq_ctx;
265 1.43 yamaguch };
266 1.43 yamaguch
267 1.98 yamaguch struct vioif_tx_context {
268 1.98 yamaguch bool txc_link_active;
269 1.102 yamaguch bool txc_no_free_slots;
270 1.98 yamaguch pcq_t *txc_intrq;
271 1.98 yamaguch void *txc_deferred_transmit;
272 1.43 yamaguch
273 1.98 yamaguch struct evcnt txc_defrag_failed;
274 1.98 yamaguch };
275 1.43 yamaguch
276 1.98 yamaguch struct vioif_rx_context {
277 1.98 yamaguch struct evcnt rxc_mbuf_enobufs;
278 1.43 yamaguch };
279 1.43 yamaguch struct vioif_ctrlqueue {
280 1.43 yamaguch struct virtqueue *ctrlq_vq;
281 1.43 yamaguch enum {
282 1.43 yamaguch FREE, INUSE, DONE
283 1.43 yamaguch } ctrlq_inuse;
284 1.43 yamaguch kcondvar_t ctrlq_wait;
285 1.43 yamaguch kmutex_t ctrlq_wait_lock;
286 1.44 yamaguch struct lwp *ctrlq_owner;
287 1.43 yamaguch
288 1.43 yamaguch struct virtio_net_ctrl_cmd *ctrlq_cmd;
289 1.43 yamaguch struct virtio_net_ctrl_status *ctrlq_status;
290 1.43 yamaguch struct virtio_net_ctrl_rx *ctrlq_rx;
291 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
292 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
293 1.74 yamaguch struct virtio_net_ctrl_mac_addr *ctrlq_mac_addr;
294 1.46 yamaguch struct virtio_net_ctrl_mq *ctrlq_mq;
295 1.43 yamaguch
296 1.43 yamaguch bus_dmamap_t ctrlq_cmd_dmamap;
297 1.43 yamaguch bus_dmamap_t ctrlq_status_dmamap;
298 1.43 yamaguch bus_dmamap_t ctrlq_rx_dmamap;
299 1.43 yamaguch bus_dmamap_t ctrlq_tbl_uc_dmamap;
300 1.43 yamaguch bus_dmamap_t ctrlq_tbl_mc_dmamap;
301 1.74 yamaguch bus_dmamap_t ctrlq_mac_addr_dmamap;
302 1.46 yamaguch bus_dmamap_t ctrlq_mq_dmamap;
303 1.63 yamaguch
304 1.63 yamaguch struct evcnt ctrlq_cmd_load_failed;
305 1.63 yamaguch struct evcnt ctrlq_cmd_failed;
306 1.43 yamaguch };
307 1.43 yamaguch
308 1.1 hannken struct vioif_softc {
309 1.1 hannken device_t sc_dev;
310 1.62 yamaguch kmutex_t sc_lock;
311 1.55 yamaguch struct sysctllog *sc_sysctllog;
312 1.1 hannken
313 1.1 hannken struct virtio_softc *sc_virtio;
314 1.46 yamaguch struct virtqueue *sc_vqs;
315 1.66 reinoud u_int sc_hdr_size;
316 1.46 yamaguch
317 1.46 yamaguch int sc_max_nvq_pairs;
318 1.46 yamaguch int sc_req_nvq_pairs;
319 1.46 yamaguch int sc_act_nvq_pairs;
320 1.1 hannken
321 1.1 hannken uint8_t sc_mac[ETHER_ADDR_LEN];
322 1.1 hannken struct ethercom sc_ethercom;
323 1.82 knakahar int sc_link_state;
324 1.1 hannken
325 1.98 yamaguch struct vioif_netqueue *sc_netqs;
326 1.43 yamaguch
327 1.43 yamaguch bool sc_has_ctrl;
328 1.43 yamaguch struct vioif_ctrlqueue sc_ctrlq;
329 1.43 yamaguch
330 1.100 yamaguch bus_dma_segment_t sc_segs[1];
331 1.43 yamaguch void *sc_dmamem;
332 1.43 yamaguch void *sc_kmem;
333 1.1 hannken
334 1.101 yamaguch void *sc_cfg_softint;
335 1.55 yamaguch
336 1.55 yamaguch struct workqueue *sc_txrx_workqueue;
337 1.55 yamaguch bool sc_txrx_workqueue_sysctl;
338 1.55 yamaguch u_int sc_tx_intr_process_limit;
339 1.55 yamaguch u_int sc_tx_process_limit;
340 1.55 yamaguch u_int sc_rx_intr_process_limit;
341 1.55 yamaguch u_int sc_rx_process_limit;
342 1.1 hannken };
343 1.1 hannken #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
344 1.1 hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
345 1.1 hannken
346 1.55 yamaguch #define VIOIF_TX_INTR_PROCESS_LIMIT 256
347 1.55 yamaguch #define VIOIF_TX_PROCESS_LIMIT 256
348 1.55 yamaguch #define VIOIF_RX_INTR_PROCESS_LIMIT 0U
349 1.55 yamaguch #define VIOIF_RX_PROCESS_LIMIT 256
350 1.55 yamaguch
351 1.55 yamaguch #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET
352 1.82 knakahar #define VIOIF_IS_LINK_ACTIVE(_sc) ((_sc)->sc_link_state == LINK_STATE_UP ? \
353 1.82 knakahar true : false)
354 1.55 yamaguch
355 1.1 hannken /* cfattach interface functions */
356 1.1 hannken static int vioif_match(device_t, cfdata_t, void *);
357 1.1 hannken static void vioif_attach(device_t, device_t, void *);
358 1.55 yamaguch static int vioif_finalize_teardown(device_t);
359 1.1 hannken
360 1.1 hannken /* ifnet interface functions */
361 1.1 hannken static int vioif_init(struct ifnet *);
362 1.1 hannken static void vioif_stop(struct ifnet *, int);
363 1.1 hannken static void vioif_start(struct ifnet *);
364 1.46 yamaguch static int vioif_transmit(struct ifnet *, struct mbuf *);
365 1.1 hannken static int vioif_ioctl(struct ifnet *, u_long, void *);
366 1.1 hannken static void vioif_watchdog(struct ifnet *);
367 1.101 yamaguch static int vioif_ifflags(struct vioif_softc *);
368 1.75 yamaguch static int vioif_ifflags_cb(struct ethercom *);
369 1.1 hannken
370 1.98 yamaguch /* tx & rx */
371 1.101 yamaguch static int vioif_netqueue_init(struct vioif_softc *,
372 1.101 yamaguch struct virtio_softc *, size_t, u_int);
373 1.101 yamaguch static void vioif_netqueue_teardown(struct vioif_softc *,
374 1.101 yamaguch struct virtio_softc *, size_t);
375 1.99 yamaguch static void vioif_net_intr_enable(struct vioif_softc *,
376 1.99 yamaguch struct virtio_softc *);
377 1.99 yamaguch static void vioif_net_intr_disable(struct vioif_softc *,
378 1.99 yamaguch struct virtio_softc *);
379 1.101 yamaguch static void vioif_net_sched_handle(struct vioif_softc *,
380 1.101 yamaguch struct vioif_netqueue *);
381 1.98 yamaguch
382 1.1 hannken /* rx */
383 1.66 reinoud static void vioif_populate_rx_mbufs_locked(struct vioif_softc *,
384 1.98 yamaguch struct vioif_netqueue *);
385 1.54 yamaguch static int vioif_rx_intr(void *);
386 1.55 yamaguch static void vioif_rx_handle(void *);
387 1.101 yamaguch static void vioif_rx_queue_clear(struct vioif_softc *,
388 1.101 yamaguch struct virtio_softc *, struct vioif_netqueue *);
389 1.1 hannken
390 1.1 hannken /* tx */
391 1.101 yamaguch static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *);
392 1.101 yamaguch static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *);
393 1.101 yamaguch static void vioif_deferred_transmit(void *);
394 1.54 yamaguch static int vioif_tx_intr(void *);
395 1.55 yamaguch static void vioif_tx_handle(void *);
396 1.93 yamaguch static void vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *,
397 1.98 yamaguch struct vioif_netqueue *);
398 1.1 hannken
399 1.101 yamaguch /* controls */
400 1.101 yamaguch static int vioif_ctrl_intr(void *);
401 1.1 hannken static int vioif_ctrl_rx(struct vioif_softc *, int, bool);
402 1.1 hannken static int vioif_set_promisc(struct vioif_softc *, bool);
403 1.1 hannken static int vioif_set_allmulti(struct vioif_softc *, bool);
404 1.1 hannken static int vioif_set_rx_filter(struct vioif_softc *);
405 1.1 hannken static int vioif_rx_filter(struct vioif_softc *);
406 1.74 yamaguch static int vioif_set_mac_addr(struct vioif_softc *);
407 1.101 yamaguch static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
408 1.101 yamaguch
409 1.101 yamaguch /* config interrupt */
410 1.34 ozaki static int vioif_config_change(struct virtio_softc *);
411 1.101 yamaguch static void vioif_cfg_softint(void *);
412 1.101 yamaguch static void vioif_update_link_status(struct vioif_softc *);
413 1.101 yamaguch
414 1.101 yamaguch /* others */
415 1.101 yamaguch static void vioif_alloc_queues(struct vioif_softc *);
416 1.101 yamaguch static void vioif_free_queues(struct vioif_softc *);
417 1.101 yamaguch static int vioif_alloc_mems(struct vioif_softc *);
418 1.101 yamaguch static struct workqueue*
419 1.101 yamaguch vioif_workq_create(const char *, pri_t, int, int);
420 1.101 yamaguch static void vioif_workq_destroy(struct workqueue *);
421 1.101 yamaguch static void vioif_work_set(struct vioif_work *, void(*)(void *), void *);
422 1.101 yamaguch static void vioif_work_add(struct workqueue *, struct vioif_work *);
423 1.101 yamaguch static void vioif_work_wait(struct workqueue *, struct vioif_work *);
424 1.55 yamaguch static int vioif_setup_sysctl(struct vioif_softc *);
425 1.63 yamaguch static void vioif_setup_stats(struct vioif_softc *);
426 1.1 hannken
427 1.1 hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
428 1.1 hannken vioif_match, vioif_attach, NULL, NULL);
429 1.1 hannken
430 1.101 yamaguch static void
431 1.101 yamaguch vioif_intr_barrier(void)
432 1.101 yamaguch {
433 1.101 yamaguch
434 1.101 yamaguch /* wait for finish all interrupt handler */
435 1.101 yamaguch xc_barrier(0);
436 1.101 yamaguch }
437 1.101 yamaguch
438 1.101 yamaguch static void
439 1.101 yamaguch vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq)
440 1.101 yamaguch {
441 1.101 yamaguch
442 1.101 yamaguch virtio_enqueue_commit(vsc, vq, -1, true);
443 1.101 yamaguch }
444 1.101 yamaguch
445 1.1 hannken static int
446 1.1 hannken vioif_match(device_t parent, cfdata_t match, void *aux)
447 1.1 hannken {
448 1.32 jdolecek struct virtio_attach_args *va = aux;
449 1.1 hannken
450 1.66 reinoud if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK)
451 1.1 hannken return 1;
452 1.1 hannken
453 1.1 hannken return 0;
454 1.1 hannken }
455 1.1 hannken
456 1.101 yamaguch static void
457 1.101 yamaguch vioif_attach(device_t parent, device_t self, void *aux)
458 1.58 yamaguch {
459 1.101 yamaguch struct vioif_softc *sc = device_private(self);
460 1.101 yamaguch struct virtio_softc *vsc = device_private(parent);
461 1.101 yamaguch struct vioif_netqueue *txq0;
462 1.101 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
463 1.101 yamaguch uint64_t features, req_features;
464 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
465 1.101 yamaguch u_int softint_flags;
466 1.101 yamaguch int r, i, req_flags;
467 1.101 yamaguch char xnamebuf[MAXCOMLEN];
468 1.103 yamaguch size_t nvqs;
469 1.101 yamaguch
470 1.101 yamaguch if (virtio_child(vsc) != NULL) {
471 1.101 yamaguch aprint_normal(": child already attached for %s; "
472 1.101 yamaguch "something wrong...\n", device_xname(parent));
473 1.101 yamaguch return;
474 1.101 yamaguch }
475 1.101 yamaguch
476 1.101 yamaguch sc->sc_dev = self;
477 1.101 yamaguch sc->sc_virtio = vsc;
478 1.101 yamaguch sc->sc_link_state = LINK_STATE_UNKNOWN;
479 1.101 yamaguch
480 1.101 yamaguch sc->sc_max_nvq_pairs = 1;
481 1.101 yamaguch sc->sc_req_nvq_pairs = 1;
482 1.101 yamaguch sc->sc_act_nvq_pairs = 1;
483 1.101 yamaguch sc->sc_txrx_workqueue_sysctl = true;
484 1.101 yamaguch sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
485 1.101 yamaguch sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
486 1.101 yamaguch sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
487 1.101 yamaguch sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
488 1.101 yamaguch
489 1.101 yamaguch mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE);
490 1.101 yamaguch
491 1.101 yamaguch snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
492 1.101 yamaguch sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
493 1.101 yamaguch IPL_NET, WQ_PERCPU | WQ_MPSAFE);
494 1.101 yamaguch if (sc->sc_txrx_workqueue == NULL)
495 1.101 yamaguch goto err;
496 1.58 yamaguch
497 1.101 yamaguch req_flags = 0;
498 1.58 yamaguch
499 1.101 yamaguch #ifdef VIOIF_MPSAFE
500 1.101 yamaguch req_flags |= VIRTIO_F_INTR_MPSAFE;
501 1.101 yamaguch #endif
502 1.101 yamaguch req_flags |= VIRTIO_F_INTR_MSIX;
503 1.58 yamaguch
504 1.101 yamaguch req_features =
505 1.101 yamaguch VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
506 1.101 yamaguch VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
507 1.101 yamaguch req_features |= VIRTIO_F_RING_EVENT_IDX;
508 1.101 yamaguch req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR;
509 1.101 yamaguch #ifdef VIOIF_MULTIQ
510 1.101 yamaguch req_features |= VIRTIO_NET_F_MQ;
511 1.101 yamaguch #endif
512 1.103 yamaguch
513 1.103 yamaguch virtio_child_attach_start(vsc, self, IPL_NET,
514 1.101 yamaguch req_features, VIRTIO_NET_FLAG_BITS);
515 1.103 yamaguch features = virtio_features(vsc);
516 1.58 yamaguch
517 1.101 yamaguch if (features == 0)
518 1.101 yamaguch goto err;
519 1.58 yamaguch
520 1.101 yamaguch if (features & VIRTIO_NET_F_MAC) {
521 1.101 yamaguch for (i = 0; i < __arraycount(sc->sc_mac); i++) {
522 1.101 yamaguch sc->sc_mac[i] = virtio_read_device_config_1(vsc,
523 1.101 yamaguch VIRTIO_NET_CONFIG_MAC + i);
524 1.101 yamaguch }
525 1.101 yamaguch } else {
526 1.101 yamaguch /* code stolen from sys/net/if_tap.c */
527 1.101 yamaguch struct timeval tv;
528 1.101 yamaguch uint32_t ui;
529 1.101 yamaguch getmicrouptime(&tv);
530 1.101 yamaguch ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
531 1.101 yamaguch memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
532 1.101 yamaguch for (i = 0; i < __arraycount(sc->sc_mac); i++) {
533 1.101 yamaguch virtio_write_device_config_1(vsc,
534 1.101 yamaguch VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
535 1.101 yamaguch }
536 1.58 yamaguch }
537 1.58 yamaguch
538 1.101 yamaguch /* 'Ethernet' with capital follows other ethernet driver attachment */
539 1.101 yamaguch aprint_normal_dev(self, "Ethernet address %s\n",
540 1.101 yamaguch ether_sprintf(sc->sc_mac));
541 1.101 yamaguch
542 1.101 yamaguch if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) {
543 1.101 yamaguch sc->sc_hdr_size = sizeof(struct virtio_net_hdr);
544 1.101 yamaguch } else {
545 1.101 yamaguch sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers);
546 1.101 yamaguch }
547 1.58 yamaguch
548 1.101 yamaguch if ((features & VIRTIO_NET_F_CTRL_VQ) &&
549 1.101 yamaguch (features & VIRTIO_NET_F_CTRL_RX)) {
550 1.101 yamaguch sc->sc_has_ctrl = true;
551 1.58 yamaguch
552 1.101 yamaguch cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
553 1.101 yamaguch mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
554 1.101 yamaguch ctrlq->ctrlq_inuse = FREE;
555 1.101 yamaguch } else {
556 1.101 yamaguch sc->sc_has_ctrl = false;
557 1.58 yamaguch }
558 1.58 yamaguch
559 1.101 yamaguch if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
560 1.101 yamaguch sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
561 1.101 yamaguch VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
562 1.101 yamaguch
563 1.101 yamaguch if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
564 1.101 yamaguch goto err;
565 1.58 yamaguch
566 1.101 yamaguch /* Limit the number of queue pairs to use */
567 1.101 yamaguch sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
568 1.103 yamaguch
569 1.103 yamaguch if (sc->sc_max_nvq_pairs > 1)
570 1.103 yamaguch req_flags |= VIRTIO_F_INTR_PERVQ;
571 1.101 yamaguch }
572 1.58 yamaguch
573 1.101 yamaguch vioif_alloc_queues(sc);
574 1.58 yamaguch
575 1.101 yamaguch #ifdef VIOIF_MPSAFE
576 1.101 yamaguch softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
577 1.101 yamaguch #else
578 1.101 yamaguch softint_flags = SOFTINT_NET;
579 1.101 yamaguch #endif
580 1.58 yamaguch
581 1.101 yamaguch /*
582 1.101 yamaguch * Initialize network queues
583 1.101 yamaguch */
584 1.103 yamaguch nvqs = sc->sc_max_nvq_pairs * 2;
585 1.103 yamaguch for (i = 0; i < nvqs; i++) {
586 1.101 yamaguch r = vioif_netqueue_init(sc, vsc, i, softint_flags);
587 1.101 yamaguch if (r != 0)
588 1.101 yamaguch goto err;
589 1.101 yamaguch }
590 1.46 yamaguch
591 1.101 yamaguch if (sc->sc_has_ctrl) {
592 1.103 yamaguch int ctrlq_idx = nvqs;
593 1.103 yamaguch
594 1.103 yamaguch nvqs++;
595 1.101 yamaguch /*
596 1.101 yamaguch * Allocating a virtqueue for control channel
597 1.101 yamaguch */
598 1.101 yamaguch sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx];
599 1.104 yamaguch virtio_init_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx,
600 1.104 yamaguch vioif_ctrl_intr, ctrlq);
601 1.104 yamaguch
602 1.104 yamaguch r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, NBPG, 1, "control");
603 1.101 yamaguch if (r != 0) {
604 1.101 yamaguch aprint_error_dev(self, "failed to allocate "
605 1.101 yamaguch "a virtqueue for control channel, error code %d\n",
606 1.101 yamaguch r);
607 1.46 yamaguch
608 1.101 yamaguch sc->sc_has_ctrl = false;
609 1.101 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
610 1.101 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
611 1.101 yamaguch }
612 1.101 yamaguch }
613 1.46 yamaguch
614 1.101 yamaguch sc->sc_cfg_softint = softint_establish(softint_flags,
615 1.101 yamaguch vioif_cfg_softint, sc);
616 1.101 yamaguch if (sc->sc_cfg_softint == NULL) {
617 1.101 yamaguch aprint_error_dev(self, "cannot establish ctl softint\n");
618 1.101 yamaguch goto err;
619 1.101 yamaguch }
620 1.46 yamaguch
621 1.101 yamaguch if (vioif_alloc_mems(sc) < 0)
622 1.101 yamaguch goto err;
623 1.46 yamaguch
624 1.103 yamaguch r = virtio_child_attach_finish(vsc, sc->sc_vqs, nvqs,
625 1.104 yamaguch vioif_config_change, req_flags);
626 1.103 yamaguch if (r != 0)
627 1.101 yamaguch goto err;
628 1.46 yamaguch
629 1.101 yamaguch if (vioif_setup_sysctl(sc) != 0) {
630 1.101 yamaguch aprint_error_dev(self, "unable to create sysctl node\n");
631 1.101 yamaguch /* continue */
632 1.101 yamaguch }
633 1.98 yamaguch
634 1.101 yamaguch vioif_setup_stats(sc);
635 1.98 yamaguch
636 1.101 yamaguch strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
637 1.101 yamaguch ifp->if_softc = sc;
638 1.101 yamaguch ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
639 1.101 yamaguch #ifdef VIOIF_MPSAFE
640 1.101 yamaguch ifp->if_extflags = IFEF_MPSAFE;
641 1.101 yamaguch #endif
642 1.101 yamaguch ifp->if_start = vioif_start;
643 1.101 yamaguch if (sc->sc_req_nvq_pairs > 1)
644 1.101 yamaguch ifp->if_transmit = vioif_transmit;
645 1.101 yamaguch ifp->if_ioctl = vioif_ioctl;
646 1.101 yamaguch ifp->if_init = vioif_init;
647 1.101 yamaguch ifp->if_stop = vioif_stop;
648 1.101 yamaguch ifp->if_capabilities = 0;
649 1.101 yamaguch ifp->if_watchdog = vioif_watchdog;
650 1.101 yamaguch txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
651 1.101 yamaguch IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN));
652 1.101 yamaguch IFQ_SET_READY(&ifp->if_snd);
653 1.98 yamaguch
654 1.101 yamaguch sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
655 1.98 yamaguch
656 1.101 yamaguch if_attach(ifp);
657 1.101 yamaguch if_deferred_start_init(ifp, NULL);
658 1.101 yamaguch ether_ifattach(ifp, sc->sc_mac);
659 1.101 yamaguch ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb);
660 1.98 yamaguch
661 1.101 yamaguch return;
662 1.98 yamaguch
663 1.98 yamaguch err:
664 1.103 yamaguch nvqs = sc->sc_max_nvq_pairs * 2;
665 1.103 yamaguch for (i = 0; i < nvqs; i++) {
666 1.101 yamaguch vioif_netqueue_teardown(sc, vsc, i);
667 1.98 yamaguch }
668 1.98 yamaguch
669 1.101 yamaguch if (sc->sc_has_ctrl) {
670 1.101 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
671 1.101 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
672 1.101 yamaguch virtio_free_vq(vsc, ctrlq->ctrlq_vq);
673 1.101 yamaguch ctrlq->ctrlq_vq = NULL;
674 1.98 yamaguch }
675 1.98 yamaguch
676 1.101 yamaguch vioif_free_queues(sc);
677 1.101 yamaguch mutex_destroy(&sc->sc_lock);
678 1.101 yamaguch virtio_child_attach_failed(vsc);
679 1.101 yamaguch config_finalize_register(self, vioif_finalize_teardown);
680 1.98 yamaguch
681 1.101 yamaguch return;
682 1.98 yamaguch }
683 1.98 yamaguch
684 1.101 yamaguch static int
685 1.101 yamaguch vioif_finalize_teardown(device_t self)
686 1.98 yamaguch {
687 1.101 yamaguch struct vioif_softc *sc = device_private(self);
688 1.98 yamaguch
689 1.101 yamaguch if (sc->sc_txrx_workqueue != NULL) {
690 1.101 yamaguch vioif_workq_destroy(sc->sc_txrx_workqueue);
691 1.101 yamaguch sc->sc_txrx_workqueue = NULL;
692 1.46 yamaguch }
693 1.98 yamaguch
694 1.101 yamaguch return 0;
695 1.46 yamaguch }
696 1.46 yamaguch
697 1.1 hannken /*
698 1.101 yamaguch * Interface functions for ifnet
699 1.1 hannken */
700 1.1 hannken static int
701 1.101 yamaguch vioif_init(struct ifnet *ifp)
702 1.1 hannken {
703 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
704 1.1 hannken struct virtio_softc *vsc = sc->sc_virtio;
705 1.98 yamaguch struct vioif_netqueue *netq;
706 1.98 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
707 1.101 yamaguch int r, i;
708 1.101 yamaguch
709 1.101 yamaguch vioif_stop(ifp, 0);
710 1.101 yamaguch
711 1.101 yamaguch r = virtio_reinit_start(vsc);
712 1.101 yamaguch if (r != 0) {
713 1.101 yamaguch log(LOG_ERR, "%s: reset failed\n", ifp->if_xname);
714 1.101 yamaguch return EIO;
715 1.101 yamaguch }
716 1.1 hannken
717 1.101 yamaguch virtio_negotiate_features(vsc, virtio_features(vsc));
718 1.98 yamaguch
719 1.101 yamaguch for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
720 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
721 1.46 yamaguch
722 1.101 yamaguch mutex_enter(&netq->netq_lock);
723 1.101 yamaguch vioif_populate_rx_mbufs_locked(sc, netq);
724 1.101 yamaguch mutex_exit(&netq->netq_lock);
725 1.46 yamaguch }
726 1.98 yamaguch
727 1.101 yamaguch virtio_reinit_end(vsc);
728 1.101 yamaguch
729 1.101 yamaguch if (sc->sc_has_ctrl)
730 1.101 yamaguch virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
731 1.98 yamaguch
732 1.101 yamaguch r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
733 1.101 yamaguch if (r == 0)
734 1.101 yamaguch sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
735 1.101 yamaguch else
736 1.101 yamaguch sc->sc_act_nvq_pairs = 1;
737 1.101 yamaguch
738 1.101 yamaguch SET(ifp->if_flags, IFF_RUNNING);
739 1.42 yamaguch
740 1.101 yamaguch vioif_net_intr_enable(sc, vsc);
741 1.42 yamaguch
742 1.101 yamaguch vioif_update_link_status(sc);
743 1.101 yamaguch r = vioif_rx_filter(sc);
744 1.98 yamaguch
745 1.101 yamaguch return r;
746 1.101 yamaguch }
747 1.98 yamaguch
748 1.101 yamaguch static void
749 1.101 yamaguch vioif_stop(struct ifnet *ifp, int disable)
750 1.101 yamaguch {
751 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
752 1.101 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
753 1.101 yamaguch struct vioif_netqueue *netq;
754 1.101 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
755 1.101 yamaguch size_t i, act_qnum;
756 1.1 hannken
757 1.101 yamaguch act_qnum = sc->sc_act_nvq_pairs * 2;
758 1.46 yamaguch
759 1.101 yamaguch CLR(ifp->if_flags, IFF_RUNNING);
760 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
761 1.101 yamaguch netq = &sc->sc_netqs[i];
762 1.95 yamaguch
763 1.101 yamaguch mutex_enter(&netq->netq_lock);
764 1.101 yamaguch netq->netq_stopping = true;
765 1.101 yamaguch mutex_exit(&netq->netq_lock);
766 1.98 yamaguch }
767 1.46 yamaguch
768 1.101 yamaguch /* disable interrupts */
769 1.101 yamaguch vioif_net_intr_disable(sc, vsc);
770 1.101 yamaguch if (sc->sc_has_ctrl)
771 1.101 yamaguch virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
772 1.101 yamaguch
773 1.101 yamaguch /*
774 1.101 yamaguch * only way to stop interrupt, I/O and DMA is resetting...
775 1.101 yamaguch *
776 1.101 yamaguch * NOTE: Devices based on VirtIO draft specification can not
777 1.101 yamaguch * stop interrupt completely even if virtio_stop_vq_intr() is called.
778 1.101 yamaguch */
779 1.101 yamaguch virtio_reset(vsc);
780 1.101 yamaguch
781 1.101 yamaguch vioif_intr_barrier();
782 1.98 yamaguch
783 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
784 1.101 yamaguch netq = &sc->sc_netqs[i];
785 1.101 yamaguch vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work);
786 1.101 yamaguch }
787 1.42 yamaguch
788 1.101 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
789 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
790 1.101 yamaguch vioif_rx_queue_clear(sc, vsc, netq);
791 1.98 yamaguch
792 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
793 1.101 yamaguch vioif_tx_queue_clear(sc, vsc, netq);
794 1.98 yamaguch }
795 1.98 yamaguch
796 1.101 yamaguch /* all packet processing is stopped */
797 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
798 1.101 yamaguch netq = &sc->sc_netqs[i];
799 1.1 hannken
800 1.101 yamaguch mutex_enter(&netq->netq_lock);
801 1.101 yamaguch netq->netq_stopping = false;
802 1.101 yamaguch mutex_exit(&netq->netq_lock);
803 1.101 yamaguch }
804 1.101 yamaguch }
805 1.95 yamaguch
806 1.101 yamaguch static void
807 1.101 yamaguch vioif_start(struct ifnet *ifp)
808 1.101 yamaguch {
809 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
810 1.101 yamaguch struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
811 1.58 yamaguch
812 1.101 yamaguch #ifdef VIOIF_MPSAFE
813 1.101 yamaguch KASSERT(if_is_mpsafe(ifp));
814 1.101 yamaguch #endif
815 1.58 yamaguch
816 1.101 yamaguch mutex_enter(&txq0->netq_lock);
817 1.101 yamaguch vioif_start_locked(ifp, txq0);
818 1.101 yamaguch mutex_exit(&txq0->netq_lock);
819 1.101 yamaguch }
820 1.1 hannken
821 1.101 yamaguch static inline int
822 1.101 yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
823 1.101 yamaguch {
824 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
825 1.101 yamaguch u_int cpuid = cpu_index(curcpu());
826 1.58 yamaguch
827 1.101 yamaguch return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs);
828 1.101 yamaguch }
829 1.1 hannken
830 1.101 yamaguch static int
831 1.101 yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
832 1.101 yamaguch {
833 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
834 1.101 yamaguch struct vioif_netqueue *netq;
835 1.101 yamaguch struct vioif_tx_context *txc;
836 1.101 yamaguch int qid;
837 1.1 hannken
838 1.101 yamaguch qid = vioif_select_txqueue(ifp, m);
839 1.101 yamaguch netq = &sc->sc_netqs[qid];
840 1.101 yamaguch txc = netq->netq_ctx;
841 1.46 yamaguch
842 1.101 yamaguch if (__predict_false(!pcq_put(txc->txc_intrq, m))) {
843 1.101 yamaguch m_freem(m);
844 1.101 yamaguch return ENOBUFS;
845 1.101 yamaguch }
846 1.1 hannken
847 1.101 yamaguch net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
848 1.112 riastrad if_statadd_ref(ifp, nsr, if_obytes, m->m_pkthdr.len);
849 1.101 yamaguch if (m->m_flags & M_MCAST)
850 1.112 riastrad if_statinc_ref(ifp, nsr, if_omcasts);
851 1.101 yamaguch IF_STAT_PUTREF(ifp);
852 1.74 yamaguch
853 1.101 yamaguch if (mutex_tryenter(&netq->netq_lock)) {
854 1.101 yamaguch vioif_transmit_locked(ifp, netq);
855 1.101 yamaguch mutex_exit(&netq->netq_lock);
856 1.1 hannken }
857 1.1 hannken
858 1.1 hannken return 0;
859 1.101 yamaguch }
860 1.1 hannken
861 1.101 yamaguch void
862 1.101 yamaguch vioif_watchdog(struct ifnet *ifp)
863 1.101 yamaguch {
864 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
865 1.101 yamaguch struct vioif_netqueue *netq;
866 1.101 yamaguch int i;
867 1.101 yamaguch
868 1.102 yamaguch if (ISSET(ifp->if_flags, IFF_RUNNING)) {
869 1.102 yamaguch if (ISSET(ifp->if_flags, IFF_DEBUG)) {
870 1.102 yamaguch log(LOG_DEBUG, "%s: watchdog timed out\n",
871 1.102 yamaguch ifp->if_xname);
872 1.102 yamaguch }
873 1.102 yamaguch
874 1.101 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
875 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
876 1.46 yamaguch
877 1.101 yamaguch mutex_enter(&netq->netq_lock);
878 1.101 yamaguch if (!netq->netq_running_handle) {
879 1.101 yamaguch netq->netq_running_handle = true;
880 1.101 yamaguch vioif_net_sched_handle(sc, netq);
881 1.101 yamaguch }
882 1.101 yamaguch mutex_exit(&netq->netq_lock);
883 1.46 yamaguch }
884 1.1 hannken }
885 1.101 yamaguch }
886 1.101 yamaguch
887 1.101 yamaguch static int
888 1.101 yamaguch vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
889 1.101 yamaguch {
890 1.101 yamaguch int s, r;
891 1.101 yamaguch
892 1.101 yamaguch s = splnet();
893 1.101 yamaguch
894 1.101 yamaguch r = ether_ioctl(ifp, cmd, data);
895 1.101 yamaguch if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) {
896 1.101 yamaguch if (ifp->if_flags & IFF_RUNNING) {
897 1.101 yamaguch r = vioif_rx_filter(ifp->if_softc);
898 1.101 yamaguch } else {
899 1.101 yamaguch r = 0;
900 1.101 yamaguch }
901 1.1 hannken }
902 1.101 yamaguch
903 1.101 yamaguch splx(s);
904 1.101 yamaguch
905 1.101 yamaguch return r;
906 1.1 hannken }
907 1.1 hannken
908 1.101 yamaguch static int
909 1.101 yamaguch vioif_ifflags(struct vioif_softc *sc)
910 1.1 hannken {
911 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
912 1.101 yamaguch bool onoff;
913 1.101 yamaguch int r;
914 1.1 hannken
915 1.101 yamaguch if (!sc->sc_has_ctrl) {
916 1.101 yamaguch /* no ctrl vq; always promisc and allmulti */
917 1.101 yamaguch ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI);
918 1.101 yamaguch return 0;
919 1.1 hannken }
920 1.1 hannken
921 1.101 yamaguch onoff = ifp->if_flags & IFF_ALLMULTI ? true : false;
922 1.101 yamaguch r = vioif_set_allmulti(sc, onoff);
923 1.101 yamaguch if (r != 0) {
924 1.101 yamaguch log(LOG_WARNING,
925 1.101 yamaguch "%s: couldn't %sable ALLMULTI\n",
926 1.101 yamaguch ifp->if_xname, onoff ? "en" : "dis");
927 1.105 yamaguch if (onoff) {
928 1.105 yamaguch CLR(ifp->if_flags, IFF_ALLMULTI);
929 1.105 yamaguch } else {
930 1.105 yamaguch SET(ifp->if_flags, IFF_ALLMULTI);
931 1.101 yamaguch }
932 1.101 yamaguch }
933 1.1 hannken
934 1.101 yamaguch onoff = ifp->if_flags & IFF_PROMISC ? true : false;
935 1.101 yamaguch r = vioif_set_promisc(sc, onoff);
936 1.101 yamaguch if (r != 0) {
937 1.101 yamaguch log(LOG_WARNING,
938 1.101 yamaguch "%s: couldn't %sable PROMISC\n",
939 1.101 yamaguch ifp->if_xname, onoff ? "en" : "dis");
940 1.105 yamaguch if (onoff) {
941 1.105 yamaguch CLR(ifp->if_flags, IFF_PROMISC);
942 1.105 yamaguch } else {
943 1.105 yamaguch SET(ifp->if_flags, IFF_PROMISC);
944 1.101 yamaguch }
945 1.101 yamaguch }
946 1.55 yamaguch
947 1.101 yamaguch return 0;
948 1.101 yamaguch }
949 1.62 yamaguch
950 1.101 yamaguch static int
951 1.101 yamaguch vioif_ifflags_cb(struct ethercom *ec)
952 1.101 yamaguch {
953 1.101 yamaguch struct ifnet *ifp = &ec->ec_if;
954 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
955 1.46 yamaguch
956 1.101 yamaguch return vioif_ifflags(sc);
957 1.101 yamaguch }
958 1.1 hannken
959 1.101 yamaguch static int
960 1.101 yamaguch vioif_setup_sysctl(struct vioif_softc *sc)
961 1.101 yamaguch {
962 1.101 yamaguch const char *devname;
963 1.101 yamaguch struct sysctllog **log;
964 1.101 yamaguch const struct sysctlnode *rnode, *rxnode, *txnode;
965 1.101 yamaguch int error;
966 1.32 jdolecek
967 1.101 yamaguch log = &sc->sc_sysctllog;
968 1.101 yamaguch devname = device_xname(sc->sc_dev);
969 1.32 jdolecek
970 1.101 yamaguch error = sysctl_createv(log, 0, NULL, &rnode,
971 1.101 yamaguch 0, CTLTYPE_NODE, devname,
972 1.101 yamaguch SYSCTL_DESCR("virtio-net information and settings"),
973 1.101 yamaguch NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
974 1.101 yamaguch if (error)
975 1.101 yamaguch goto out;
976 1.7 ozaki
977 1.101 yamaguch error = sysctl_createv(log, 0, &rnode, NULL,
978 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
979 1.101 yamaguch SYSCTL_DESCR("Use workqueue for packet processing"),
980 1.101 yamaguch NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
981 1.101 yamaguch if (error)
982 1.101 yamaguch goto out;
983 1.32 jdolecek
984 1.101 yamaguch error = sysctl_createv(log, 0, &rnode, &rxnode,
985 1.101 yamaguch 0, CTLTYPE_NODE, "rx",
986 1.101 yamaguch SYSCTL_DESCR("virtio-net information and settings for Rx"),
987 1.101 yamaguch NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
988 1.101 yamaguch if (error)
989 1.101 yamaguch goto out;
990 1.1 hannken
991 1.101 yamaguch error = sysctl_createv(log, 0, &rxnode, NULL,
992 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
993 1.101 yamaguch SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
994 1.101 yamaguch NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
995 1.101 yamaguch if (error)
996 1.101 yamaguch goto out;
997 1.66 reinoud
998 1.101 yamaguch error = sysctl_createv(log, 0, &rxnode, NULL,
999 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
1000 1.101 yamaguch SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
1001 1.101 yamaguch NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
1002 1.101 yamaguch if (error)
1003 1.101 yamaguch goto out;
1004 1.46 yamaguch
1005 1.101 yamaguch error = sysctl_createv(log, 0, &rnode, &txnode,
1006 1.101 yamaguch 0, CTLTYPE_NODE, "tx",
1007 1.101 yamaguch SYSCTL_DESCR("virtio-net information and settings for Tx"),
1008 1.101 yamaguch NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
1009 1.101 yamaguch if (error)
1010 1.101 yamaguch goto out;
1011 1.46 yamaguch
1012 1.101 yamaguch error = sysctl_createv(log, 0, &txnode, NULL,
1013 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
1014 1.101 yamaguch SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
1015 1.101 yamaguch NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
1016 1.101 yamaguch if (error)
1017 1.101 yamaguch goto out;
1018 1.46 yamaguch
1019 1.101 yamaguch error = sysctl_createv(log, 0, &txnode, NULL,
1020 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
1021 1.101 yamaguch SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
1022 1.101 yamaguch NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
1023 1.46 yamaguch
1024 1.101 yamaguch out:
1025 1.101 yamaguch if (error)
1026 1.101 yamaguch sysctl_teardown(log);
1027 1.46 yamaguch
1028 1.101 yamaguch return error;
1029 1.101 yamaguch }
1030 1.46 yamaguch
1031 1.101 yamaguch static void
1032 1.101 yamaguch vioif_setup_stats(struct vioif_softc *sc)
1033 1.101 yamaguch {
1034 1.101 yamaguch struct vioif_netqueue *netq;
1035 1.101 yamaguch struct vioif_tx_context *txc;
1036 1.101 yamaguch struct vioif_rx_context *rxc;
1037 1.101 yamaguch size_t i, netq_num;
1038 1.7 ozaki
1039 1.98 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
1040 1.98 yamaguch for (i = 0; i < netq_num; i++) {
1041 1.101 yamaguch netq = &sc->sc_netqs[i];
1042 1.101 yamaguch evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC,
1043 1.101 yamaguch NULL, netq->netq_evgroup, "failed to load mbuf to DMA");
1044 1.101 yamaguch evcnt_attach_dynamic(&netq->netq_enqueue_failed,
1045 1.101 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
1046 1.101 yamaguch "virtqueue enqueue failed failed");
1047 1.17 ozaki
1048 1.101 yamaguch switch (VIOIF_NETQ_DIR(i)) {
1049 1.101 yamaguch case VIOIF_NETQ_RX:
1050 1.101 yamaguch rxc = netq->netq_ctx;
1051 1.101 yamaguch evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs,
1052 1.101 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
1053 1.101 yamaguch "no receive buffer");
1054 1.101 yamaguch break;
1055 1.101 yamaguch case VIOIF_NETQ_TX:
1056 1.101 yamaguch txc = netq->netq_ctx;
1057 1.101 yamaguch evcnt_attach_dynamic(&txc->txc_defrag_failed,
1058 1.101 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
1059 1.101 yamaguch "m_defrag() failed");
1060 1.101 yamaguch break;
1061 1.1 hannken }
1062 1.1 hannken }
1063 1.34 ozaki
1064 1.101 yamaguch evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC,
1065 1.101 yamaguch NULL, device_xname(sc->sc_dev), "control command dmamap load failed");
1066 1.101 yamaguch evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC,
1067 1.101 yamaguch NULL, device_xname(sc->sc_dev), "control command failed");
1068 1.101 yamaguch }
1069 1.1 hannken
1070 1.101 yamaguch /*
1071 1.101 yamaguch * allocate memory
1072 1.101 yamaguch */
1073 1.101 yamaguch static int
1074 1.101 yamaguch vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
1075 1.101 yamaguch bus_size_t size, int nsegs, const char *usage)
1076 1.101 yamaguch {
1077 1.101 yamaguch int r;
1078 1.1 hannken
1079 1.101 yamaguch r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
1080 1.101 yamaguch nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
1081 1.32 jdolecek
1082 1.101 yamaguch if (r != 0) {
1083 1.101 yamaguch aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
1084 1.101 yamaguch "error code %d\n", usage, r);
1085 1.55 yamaguch }
1086 1.55 yamaguch
1087 1.101 yamaguch return r;
1088 1.101 yamaguch }
1089 1.63 yamaguch
1090 1.101 yamaguch static void
1091 1.101 yamaguch vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
1092 1.101 yamaguch {
1093 1.1 hannken
1094 1.101 yamaguch if (*map) {
1095 1.101 yamaguch bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
1096 1.101 yamaguch *map = NULL;
1097 1.101 yamaguch }
1098 1.101 yamaguch }
1099 1.11 ozaki
1100 1.101 yamaguch static int
1101 1.101 yamaguch vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
1102 1.101 yamaguch void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
1103 1.101 yamaguch {
1104 1.101 yamaguch int r;
1105 1.1 hannken
1106 1.101 yamaguch r = vioif_dmamap_create(sc, map, size, nsegs, usage);
1107 1.101 yamaguch if (r != 0)
1108 1.101 yamaguch return 1;
1109 1.1 hannken
1110 1.101 yamaguch r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
1111 1.101 yamaguch size, NULL, rw | BUS_DMA_NOWAIT);
1112 1.101 yamaguch if (r != 0) {
1113 1.101 yamaguch vioif_dmamap_destroy(sc, map);
1114 1.101 yamaguch aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
1115 1.101 yamaguch "error code %d\n", usage, r);
1116 1.43 yamaguch }
1117 1.7 ozaki
1118 1.101 yamaguch return r;
1119 1.1 hannken }
1120 1.1 hannken
1121 1.101 yamaguch static void *
1122 1.101 yamaguch vioif_assign_mem(intptr_t *p, size_t size)
1123 1.55 yamaguch {
1124 1.101 yamaguch intptr_t rv;
1125 1.55 yamaguch
1126 1.101 yamaguch rv = *p;
1127 1.101 yamaguch *p += size;
1128 1.55 yamaguch
1129 1.101 yamaguch return (void *)rv;
1130 1.55 yamaguch }
1131 1.55 yamaguch
1132 1.1 hannken /*
1133 1.101 yamaguch * dma memory is used for:
1134 1.101 yamaguch * netq_maps_kva: metadata array for received frames (READ) and
1135 1.101 yamaguch * sent frames (WRITE)
1136 1.101 yamaguch * ctrlq_cmd: command to be sent via ctrl vq (WRITE)
1137 1.101 yamaguch * ctrlq_status: return value for a command via ctrl vq (READ)
1138 1.101 yamaguch * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command
1139 1.101 yamaguch * (WRITE)
1140 1.101 yamaguch * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1141 1.101 yamaguch * class command (WRITE)
1142 1.101 yamaguch * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1143 1.101 yamaguch * class command (WRITE)
1144 1.101 yamaguch * ctrlq_* structures are allocated only one each; they are protected by
1145 1.101 yamaguch * ctrlq_inuse variable and ctrlq_wait condvar.
1146 1.1 hannken */
1147 1.1 hannken static int
1148 1.101 yamaguch vioif_alloc_mems(struct vioif_softc *sc)
1149 1.1 hannken {
1150 1.33 ozaki struct virtio_softc *vsc = sc->sc_virtio;
1151 1.98 yamaguch struct vioif_netqueue *netq;
1152 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1153 1.101 yamaguch struct vioif_net_map *maps;
1154 1.101 yamaguch unsigned int vq_num;
1155 1.101 yamaguch int r, rsegs;
1156 1.101 yamaguch bus_size_t dmamemsize;
1157 1.101 yamaguch size_t qid, i, netq_num, kmemsize;
1158 1.101 yamaguch void *vaddr;
1159 1.101 yamaguch intptr_t p;
1160 1.101 yamaguch
1161 1.101 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
1162 1.1 hannken
1163 1.101 yamaguch /* allocate DMA memory */
1164 1.101 yamaguch dmamemsize = 0;
1165 1.7 ozaki
1166 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1167 1.101 yamaguch maps = sc->sc_netqs[qid].netq_maps;
1168 1.101 yamaguch vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
1169 1.101 yamaguch dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num;
1170 1.71 yamaguch }
1171 1.71 yamaguch
1172 1.101 yamaguch if (sc->sc_has_ctrl) {
1173 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_cmd);
1174 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_status);
1175 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_rx);
1176 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
1177 1.101 yamaguch + ETHER_ADDR_LEN;
1178 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
1179 1.101 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
1180 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr);
1181 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mq);
1182 1.101 yamaguch }
1183 1.46 yamaguch
1184 1.101 yamaguch r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0,
1185 1.101 yamaguch &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
1186 1.101 yamaguch if (r != 0) {
1187 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1188 1.107 nakayama "DMA memory allocation failed, size %" PRIuBUSSIZE ", "
1189 1.101 yamaguch "error code %d\n", dmamemsize, r);
1190 1.101 yamaguch goto err_none;
1191 1.101 yamaguch }
1192 1.101 yamaguch r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1,
1193 1.101 yamaguch dmamemsize, &vaddr, BUS_DMA_NOWAIT);
1194 1.101 yamaguch if (r != 0) {
1195 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1196 1.101 yamaguch "DMA memory map failed, error code %d\n", r);
1197 1.101 yamaguch goto err_dmamem_alloc;
1198 1.46 yamaguch }
1199 1.46 yamaguch
1200 1.101 yamaguch /* assign DMA memory */
1201 1.101 yamaguch memset(vaddr, 0, dmamemsize);
1202 1.101 yamaguch sc->sc_dmamem = vaddr;
1203 1.101 yamaguch p = (intptr_t) vaddr;
1204 1.47 yamaguch
1205 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1206 1.101 yamaguch netq = &sc->sc_netqs[qid];
1207 1.101 yamaguch maps = netq->netq_maps;
1208 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1209 1.46 yamaguch
1210 1.101 yamaguch netq->netq_maps_kva = vioif_assign_mem(&p,
1211 1.101 yamaguch sizeof(*maps[0].vnm_hdr) * vq_num);
1212 1.101 yamaguch }
1213 1.46 yamaguch
1214 1.101 yamaguch if (sc->sc_has_ctrl) {
1215 1.101 yamaguch ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
1216 1.101 yamaguch sizeof(*ctrlq->ctrlq_cmd));
1217 1.101 yamaguch ctrlq->ctrlq_status = vioif_assign_mem(&p,
1218 1.101 yamaguch sizeof(*ctrlq->ctrlq_status));
1219 1.101 yamaguch ctrlq->ctrlq_rx = vioif_assign_mem(&p,
1220 1.101 yamaguch sizeof(*ctrlq->ctrlq_rx));
1221 1.101 yamaguch ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
1222 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_uc)
1223 1.101 yamaguch + ETHER_ADDR_LEN);
1224 1.101 yamaguch ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
1225 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_mc)
1226 1.101 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
1227 1.101 yamaguch ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p,
1228 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_addr));
1229 1.101 yamaguch ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
1230 1.76 yamaguch }
1231 1.33 ozaki
1232 1.101 yamaguch /* allocate kmem */
1233 1.101 yamaguch kmemsize = 0;
1234 1.85 yamaguch
1235 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1236 1.101 yamaguch netq = &sc->sc_netqs[qid];
1237 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1238 1.55 yamaguch
1239 1.101 yamaguch kmemsize += sizeof(netq->netq_maps[0]) * vq_num;
1240 1.85 yamaguch }
1241 1.85 yamaguch
1242 1.101 yamaguch vaddr = kmem_zalloc(kmemsize, KM_SLEEP);
1243 1.101 yamaguch sc->sc_kmem = vaddr;
1244 1.98 yamaguch
1245 1.101 yamaguch /* assign allocated kmem */
1246 1.101 yamaguch p = (intptr_t) vaddr;
1247 1.46 yamaguch
1248 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1249 1.101 yamaguch netq = &sc->sc_netqs[qid];
1250 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1251 1.85 yamaguch
1252 1.101 yamaguch netq->netq_maps = vioif_assign_mem(&p,
1253 1.101 yamaguch sizeof(netq->netq_maps[0]) * vq_num);
1254 1.85 yamaguch }
1255 1.1 hannken
1256 1.101 yamaguch /* prepare dmamaps */
1257 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1258 1.101 yamaguch static const struct {
1259 1.101 yamaguch const char *msg_hdr;
1260 1.101 yamaguch const char *msg_payload;
1261 1.101 yamaguch int dma_flag;
1262 1.101 yamaguch bus_size_t dma_size;
1263 1.101 yamaguch int dma_nsegs;
1264 1.101 yamaguch } dmaparams[VIOIF_NETQ_IDX] = {
1265 1.101 yamaguch [VIOIF_NETQ_RX] = {
1266 1.101 yamaguch .msg_hdr = "rx header",
1267 1.101 yamaguch .msg_payload = "rx payload",
1268 1.101 yamaguch .dma_flag = BUS_DMA_READ,
1269 1.101 yamaguch .dma_size = MCLBYTES - ETHER_ALIGN,
1270 1.101 yamaguch .dma_nsegs = 1,
1271 1.101 yamaguch },
1272 1.101 yamaguch [VIOIF_NETQ_TX] = {
1273 1.101 yamaguch .msg_hdr = "tx header",
1274 1.101 yamaguch .msg_payload = "tx payload",
1275 1.101 yamaguch .dma_flag = BUS_DMA_WRITE,
1276 1.101 yamaguch .dma_size = ETHER_MAX_LEN,
1277 1.101 yamaguch .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS,
1278 1.101 yamaguch }
1279 1.101 yamaguch };
1280 1.1 hannken
1281 1.101 yamaguch struct virtio_net_hdr *hdrs;
1282 1.101 yamaguch int dir;
1283 1.111 isaki int nsegs;
1284 1.46 yamaguch
1285 1.101 yamaguch dir = VIOIF_NETQ_DIR(qid);
1286 1.101 yamaguch netq = &sc->sc_netqs[qid];
1287 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1288 1.101 yamaguch maps = netq->netq_maps;
1289 1.101 yamaguch hdrs = netq->netq_maps_kva;
1290 1.111 isaki nsegs = uimin(dmaparams[dir].dma_nsegs, vq_num - 1/*hdr*/);
1291 1.7 ozaki
1292 1.101 yamaguch for (i = 0; i < vq_num; i++) {
1293 1.101 yamaguch maps[i].vnm_hdr = &hdrs[i];
1294 1.101 yamaguch
1295 1.101 yamaguch r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map,
1296 1.101 yamaguch maps[i].vnm_hdr, sc->sc_hdr_size, 1,
1297 1.101 yamaguch dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr);
1298 1.101 yamaguch if (r != 0)
1299 1.101 yamaguch goto err_reqs;
1300 1.98 yamaguch
1301 1.101 yamaguch r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map,
1302 1.111 isaki dmaparams[dir].dma_size, nsegs,
1303 1.101 yamaguch dmaparams[dir].msg_payload);
1304 1.101 yamaguch if (r != 0)
1305 1.101 yamaguch goto err_reqs;
1306 1.101 yamaguch }
1307 1.101 yamaguch }
1308 1.7 ozaki
1309 1.101 yamaguch if (sc->sc_has_ctrl) {
1310 1.101 yamaguch /* control vq class & command */
1311 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
1312 1.101 yamaguch ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
1313 1.101 yamaguch BUS_DMA_WRITE, "control command");
1314 1.101 yamaguch if (r != 0)
1315 1.101 yamaguch goto err_reqs;
1316 1.1 hannken
1317 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
1318 1.101 yamaguch ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
1319 1.101 yamaguch BUS_DMA_READ, "control status");
1320 1.101 yamaguch if (r != 0)
1321 1.101 yamaguch goto err_reqs;
1322 1.1 hannken
1323 1.101 yamaguch /* control vq rx mode command parameter */
1324 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
1325 1.101 yamaguch ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
1326 1.101 yamaguch BUS_DMA_WRITE, "rx mode control command");
1327 1.101 yamaguch if (r != 0)
1328 1.101 yamaguch goto err_reqs;
1329 1.46 yamaguch
1330 1.101 yamaguch /* multiqueue set command */
1331 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
1332 1.101 yamaguch ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
1333 1.101 yamaguch BUS_DMA_WRITE, "multiqueue set command");
1334 1.101 yamaguch if (r != 0)
1335 1.101 yamaguch goto err_reqs;
1336 1.36 jdolecek
1337 1.101 yamaguch /* control vq MAC filter table for unicast */
1338 1.101 yamaguch /* do not load now since its length is variable */
1339 1.101 yamaguch r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
1340 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_uc)
1341 1.101 yamaguch + ETHER_ADDR_LEN, 1,
1342 1.101 yamaguch "unicast MAC address filter command");
1343 1.101 yamaguch if (r != 0)
1344 1.101 yamaguch goto err_reqs;
1345 1.95 yamaguch
1346 1.101 yamaguch /* control vq MAC filter table for multicast */
1347 1.101 yamaguch r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
1348 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_mc)
1349 1.101 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
1350 1.101 yamaguch "multicast MAC address filter command");
1351 1.101 yamaguch if (r != 0)
1352 1.101 yamaguch goto err_reqs;
1353 1.36 jdolecek
1354 1.101 yamaguch /* control vq MAC address set command */
1355 1.101 yamaguch r = vioif_dmamap_create_load(sc,
1356 1.101 yamaguch &ctrlq->ctrlq_mac_addr_dmamap,
1357 1.101 yamaguch ctrlq->ctrlq_mac_addr,
1358 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_addr), 1,
1359 1.101 yamaguch BUS_DMA_WRITE, "mac addr set command");
1360 1.101 yamaguch if (r != 0)
1361 1.101 yamaguch goto err_reqs;
1362 1.101 yamaguch }
1363 1.36 jdolecek
1364 1.101 yamaguch return 0;
1365 1.36 jdolecek
1366 1.101 yamaguch err_reqs:
1367 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
1368 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
1369 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
1370 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
1371 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
1372 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap);
1373 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1374 1.101 yamaguch vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
1375 1.101 yamaguch maps = sc->sc_netqs[qid].netq_maps;
1376 1.99 yamaguch
1377 1.101 yamaguch for (i = 0; i < vq_num; i++) {
1378 1.101 yamaguch vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map);
1379 1.101 yamaguch vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map);
1380 1.1 hannken }
1381 1.1 hannken }
1382 1.101 yamaguch if (sc->sc_kmem) {
1383 1.101 yamaguch kmem_free(sc->sc_kmem, kmemsize);
1384 1.101 yamaguch sc->sc_kmem = NULL;
1385 1.1 hannken }
1386 1.101 yamaguch bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize);
1387 1.101 yamaguch err_dmamem_alloc:
1388 1.101 yamaguch bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1);
1389 1.101 yamaguch err_none:
1390 1.101 yamaguch return -1;
1391 1.46 yamaguch }
1392 1.46 yamaguch
1393 1.46 yamaguch static void
1394 1.101 yamaguch vioif_alloc_queues(struct vioif_softc *sc)
1395 1.46 yamaguch {
1396 1.101 yamaguch int nvq_pairs = sc->sc_max_nvq_pairs;
1397 1.101 yamaguch size_t nvqs, netq_num;
1398 1.101 yamaguch
1399 1.101 yamaguch KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
1400 1.46 yamaguch
1401 1.101 yamaguch nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
1402 1.101 yamaguch if (sc->sc_has_ctrl)
1403 1.101 yamaguch nvqs++;
1404 1.46 yamaguch
1405 1.101 yamaguch sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
1406 1.106 yamaguch sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_netqs[0]) * netq_num,
1407 1.101 yamaguch KM_SLEEP);
1408 1.46 yamaguch }
1409 1.46 yamaguch
1410 1.46 yamaguch static void
1411 1.101 yamaguch vioif_free_queues(struct vioif_softc *sc)
1412 1.46 yamaguch {
1413 1.101 yamaguch size_t nvqs, netq_num;
1414 1.46 yamaguch
1415 1.101 yamaguch nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
1416 1.101 yamaguch if (sc->sc_ctrlq.ctrlq_vq)
1417 1.101 yamaguch nvqs++;
1418 1.46 yamaguch
1419 1.101 yamaguch kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num);
1420 1.101 yamaguch kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
1421 1.101 yamaguch sc->sc_netqs = NULL;
1422 1.101 yamaguch sc->sc_vqs = NULL;
1423 1.46 yamaguch }
1424 1.46 yamaguch
1425 1.101 yamaguch /*
1426 1.101 yamaguch * Network queues
1427 1.101 yamaguch */
1428 1.101 yamaguch static int
1429 1.101 yamaguch vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc,
1430 1.101 yamaguch size_t qid, u_int softint_flags)
1431 1.46 yamaguch {
1432 1.101 yamaguch static const struct {
1433 1.101 yamaguch const char *dirname;
1434 1.101 yamaguch int segsize;
1435 1.101 yamaguch int nsegs;
1436 1.101 yamaguch int (*intrhand)(void *);
1437 1.101 yamaguch void (*sihand)(void *);
1438 1.101 yamaguch } params[VIOIF_NETQ_IDX] = {
1439 1.101 yamaguch [VIOIF_NETQ_RX] = {
1440 1.101 yamaguch .dirname = "rx",
1441 1.101 yamaguch .segsize = MCLBYTES,
1442 1.101 yamaguch .nsegs = 2,
1443 1.101 yamaguch .intrhand = vioif_rx_intr,
1444 1.101 yamaguch .sihand = vioif_rx_handle,
1445 1.101 yamaguch },
1446 1.101 yamaguch [VIOIF_NETQ_TX] = {
1447 1.101 yamaguch .dirname = "tx",
1448 1.101 yamaguch .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN,
1449 1.101 yamaguch .nsegs = 2,
1450 1.101 yamaguch .intrhand = vioif_tx_intr,
1451 1.101 yamaguch .sihand = vioif_tx_handle,
1452 1.101 yamaguch }
1453 1.101 yamaguch };
1454 1.46 yamaguch
1455 1.101 yamaguch struct virtqueue *vq;
1456 1.101 yamaguch struct vioif_netqueue *netq;
1457 1.101 yamaguch struct vioif_tx_context *txc;
1458 1.101 yamaguch struct vioif_rx_context *rxc;
1459 1.101 yamaguch char qname[32];
1460 1.101 yamaguch int r, dir;
1461 1.46 yamaguch
1462 1.101 yamaguch txc = NULL;
1463 1.101 yamaguch rxc = NULL;
1464 1.101 yamaguch netq = &sc->sc_netqs[qid];
1465 1.101 yamaguch vq = &sc->sc_vqs[qid];
1466 1.101 yamaguch dir = VIOIF_NETQ_DIR(qid);
1467 1.46 yamaguch
1468 1.101 yamaguch netq->netq_vq = &sc->sc_vqs[qid];
1469 1.101 yamaguch netq->netq_stopping = false;
1470 1.101 yamaguch netq->netq_running_handle = false;
1471 1.46 yamaguch
1472 1.101 yamaguch snprintf(qname, sizeof(qname), "%s%zu",
1473 1.101 yamaguch params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid));
1474 1.101 yamaguch snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup),
1475 1.101 yamaguch "%s-%s", device_xname(sc->sc_dev), qname);
1476 1.46 yamaguch
1477 1.101 yamaguch mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET);
1478 1.104 yamaguch virtio_init_vq(vsc, vq, qid, params[dir].intrhand, netq);
1479 1.104 yamaguch
1480 1.104 yamaguch r = virtio_alloc_vq(vsc, vq,
1481 1.101 yamaguch params[dir].segsize + sc->sc_hdr_size,
1482 1.101 yamaguch params[dir].nsegs, qname);
1483 1.101 yamaguch if (r != 0)
1484 1.101 yamaguch goto err;
1485 1.101 yamaguch netq->netq_vq = vq;
1486 1.46 yamaguch
1487 1.101 yamaguch netq->netq_softint = softint_establish(softint_flags,
1488 1.101 yamaguch params[dir].sihand, netq);
1489 1.101 yamaguch if (netq->netq_softint == NULL) {
1490 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1491 1.101 yamaguch "couldn't establish %s softint\n",
1492 1.101 yamaguch params[dir].dirname);
1493 1.101 yamaguch goto err;
1494 1.46 yamaguch }
1495 1.101 yamaguch vioif_work_set(&netq->netq_work, params[dir].sihand, netq);
1496 1.46 yamaguch
1497 1.101 yamaguch switch (dir) {
1498 1.101 yamaguch case VIOIF_NETQ_RX:
1499 1.101 yamaguch rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP);
1500 1.101 yamaguch netq->netq_ctx = rxc;
1501 1.101 yamaguch /* nothing to do */
1502 1.101 yamaguch break;
1503 1.101 yamaguch case VIOIF_NETQ_TX:
1504 1.101 yamaguch txc = kmem_zalloc(sizeof(*txc), KM_SLEEP);
1505 1.101 yamaguch netq->netq_ctx = (void *)txc;
1506 1.101 yamaguch txc->txc_deferred_transmit = softint_establish(softint_flags,
1507 1.101 yamaguch vioif_deferred_transmit, netq);
1508 1.101 yamaguch if (txc->txc_deferred_transmit == NULL) {
1509 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1510 1.101 yamaguch "couldn't establish softint for "
1511 1.101 yamaguch "tx deferred transmit\n");
1512 1.101 yamaguch goto err;
1513 1.101 yamaguch }
1514 1.101 yamaguch txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc);
1515 1.102 yamaguch txc->txc_no_free_slots = false;
1516 1.101 yamaguch txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP);
1517 1.101 yamaguch break;
1518 1.46 yamaguch }
1519 1.46 yamaguch
1520 1.46 yamaguch return 0;
1521 1.46 yamaguch
1522 1.101 yamaguch err:
1523 1.101 yamaguch netq->netq_ctx = NULL;
1524 1.7 ozaki
1525 1.101 yamaguch if (rxc != NULL) {
1526 1.101 yamaguch kmem_free(rxc, sizeof(*rxc));
1527 1.101 yamaguch }
1528 1.1 hannken
1529 1.101 yamaguch if (txc != NULL) {
1530 1.101 yamaguch if (txc->txc_deferred_transmit != NULL)
1531 1.101 yamaguch softint_disestablish(txc->txc_deferred_transmit);
1532 1.101 yamaguch if (txc->txc_intrq != NULL)
1533 1.101 yamaguch pcq_destroy(txc->txc_intrq);
1534 1.101 yamaguch kmem_free(txc, sizeof(txc));
1535 1.101 yamaguch }
1536 1.1 hannken
1537 1.101 yamaguch vioif_work_set(&netq->netq_work, NULL, NULL);
1538 1.101 yamaguch if (netq->netq_softint != NULL) {
1539 1.101 yamaguch softint_disestablish(netq->netq_softint);
1540 1.101 yamaguch netq->netq_softint = NULL;
1541 1.1 hannken }
1542 1.1 hannken
1543 1.101 yamaguch virtio_free_vq(vsc, vq);
1544 1.101 yamaguch mutex_destroy(&netq->netq_lock);
1545 1.101 yamaguch netq->netq_vq = NULL;
1546 1.1 hannken
1547 1.101 yamaguch return -1;
1548 1.1 hannken }
1549 1.1 hannken
1550 1.101 yamaguch static void
1551 1.101 yamaguch vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc,
1552 1.101 yamaguch size_t qid)
1553 1.1 hannken {
1554 1.98 yamaguch struct vioif_netqueue *netq;
1555 1.101 yamaguch struct vioif_rx_context *rxc;
1556 1.101 yamaguch struct vioif_tx_context *txc;
1557 1.101 yamaguch int dir;
1558 1.101 yamaguch
1559 1.101 yamaguch netq = &sc->sc_netqs[qid];
1560 1.101 yamaguch
1561 1.101 yamaguch if (netq->netq_vq == NULL)
1562 1.101 yamaguch return;
1563 1.101 yamaguch
1564 1.101 yamaguch netq = &sc->sc_netqs[qid];
1565 1.101 yamaguch dir = VIOIF_NETQ_DIR(qid);
1566 1.101 yamaguch switch (dir) {
1567 1.101 yamaguch case VIOIF_NETQ_RX:
1568 1.101 yamaguch rxc = netq->netq_ctx;
1569 1.101 yamaguch netq->netq_ctx = NULL;
1570 1.101 yamaguch kmem_free(rxc, sizeof(*rxc));
1571 1.101 yamaguch break;
1572 1.101 yamaguch case VIOIF_NETQ_TX:
1573 1.101 yamaguch txc = netq->netq_ctx;
1574 1.101 yamaguch netq->netq_ctx = NULL;
1575 1.101 yamaguch softint_disestablish(txc->txc_deferred_transmit);
1576 1.101 yamaguch pcq_destroy(txc->txc_intrq);
1577 1.101 yamaguch kmem_free(txc, sizeof(*txc));
1578 1.101 yamaguch break;
1579 1.101 yamaguch }
1580 1.1 hannken
1581 1.101 yamaguch softint_disestablish(netq->netq_softint);
1582 1.101 yamaguch virtio_free_vq(vsc, netq->netq_vq);
1583 1.101 yamaguch mutex_destroy(&netq->netq_lock);
1584 1.101 yamaguch netq->netq_vq = NULL;
1585 1.1 hannken }
1586 1.1 hannken
1587 1.98 yamaguch static void
1588 1.98 yamaguch vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq)
1589 1.98 yamaguch {
1590 1.98 yamaguch
1591 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1592 1.98 yamaguch KASSERT(!netq->netq_stopping);
1593 1.98 yamaguch
1594 1.98 yamaguch if (netq->netq_workqueue) {
1595 1.98 yamaguch vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work);
1596 1.98 yamaguch } else {
1597 1.98 yamaguch softint_schedule(netq->netq_softint);
1598 1.98 yamaguch }
1599 1.98 yamaguch }
1600 1.98 yamaguch
1601 1.99 yamaguch static int
1602 1.99 yamaguch vioif_net_load_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map,
1603 1.99 yamaguch struct mbuf *m, int dma_flags)
1604 1.99 yamaguch {
1605 1.99 yamaguch int r;
1606 1.99 yamaguch
1607 1.99 yamaguch KASSERT(map->vnm_mbuf == NULL);
1608 1.99 yamaguch
1609 1.99 yamaguch r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1610 1.99 yamaguch map->vnm_mbuf_map, m, dma_flags | BUS_DMA_NOWAIT);
1611 1.99 yamaguch if (r == 0) {
1612 1.99 yamaguch map->vnm_mbuf = m;
1613 1.99 yamaguch }
1614 1.99 yamaguch
1615 1.99 yamaguch return r;
1616 1.99 yamaguch }
1617 1.99 yamaguch
1618 1.99 yamaguch static void
1619 1.99 yamaguch vioif_net_unload_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map)
1620 1.99 yamaguch {
1621 1.99 yamaguch
1622 1.99 yamaguch KASSERT(map->vnm_mbuf != NULL);
1623 1.99 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1624 1.99 yamaguch map->vnm_mbuf = NULL;
1625 1.99 yamaguch }
1626 1.99 yamaguch
1627 1.99 yamaguch static int
1628 1.99 yamaguch vioif_net_enqueue(struct virtio_softc *vsc, struct virtqueue *vq,
1629 1.99 yamaguch int slot, struct vioif_net_map *map, int dma_ops, bool is_write)
1630 1.99 yamaguch {
1631 1.99 yamaguch int r;
1632 1.99 yamaguch
1633 1.99 yamaguch KASSERT(map->vnm_mbuf != NULL);
1634 1.99 yamaguch
1635 1.99 yamaguch /* This should actually never fail */
1636 1.99 yamaguch r = virtio_enqueue_reserve(vsc, vq, slot,
1637 1.99 yamaguch map->vnm_mbuf_map->dm_nsegs + 1);
1638 1.99 yamaguch if (r != 0) {
1639 1.99 yamaguch /* slot already freed by virtio_enqueue_reserve */
1640 1.99 yamaguch return r;
1641 1.99 yamaguch }
1642 1.99 yamaguch
1643 1.99 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1644 1.99 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, dma_ops);
1645 1.99 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1646 1.99 yamaguch 0, map->vnm_hdr_map->dm_mapsize, dma_ops);
1647 1.99 yamaguch
1648 1.99 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, is_write);
1649 1.99 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, is_write);
1650 1.99 yamaguch virtio_enqueue_commit(vsc, vq, slot, false);
1651 1.99 yamaguch
1652 1.99 yamaguch return 0;
1653 1.99 yamaguch }
1654 1.99 yamaguch
1655 1.99 yamaguch static int
1656 1.99 yamaguch vioif_net_enqueue_tx(struct virtio_softc *vsc, struct virtqueue *vq,
1657 1.99 yamaguch int slot, struct vioif_net_map *map)
1658 1.99 yamaguch {
1659 1.99 yamaguch
1660 1.99 yamaguch return vioif_net_enqueue(vsc, vq, slot, map,
1661 1.99 yamaguch BUS_DMASYNC_PREWRITE, true);
1662 1.99 yamaguch }
1663 1.99 yamaguch
1664 1.99 yamaguch static int
1665 1.99 yamaguch vioif_net_enqueue_rx(struct virtio_softc *vsc, struct virtqueue *vq,
1666 1.99 yamaguch int slot, struct vioif_net_map *map)
1667 1.99 yamaguch {
1668 1.99 yamaguch
1669 1.99 yamaguch return vioif_net_enqueue(vsc, vq, slot, map,
1670 1.99 yamaguch BUS_DMASYNC_PREREAD, false);
1671 1.99 yamaguch }
1672 1.99 yamaguch
1673 1.101 yamaguch static struct mbuf *
1674 1.101 yamaguch vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq,
1675 1.101 yamaguch int slot, struct vioif_net_map *map, int dma_flags)
1676 1.101 yamaguch {
1677 1.101 yamaguch struct mbuf *m;
1678 1.101 yamaguch
1679 1.101 yamaguch m = map->vnm_mbuf;
1680 1.101 yamaguch KASSERT(m != NULL);
1681 1.101 yamaguch map->vnm_mbuf = NULL;
1682 1.101 yamaguch
1683 1.101 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1684 1.101 yamaguch 0, map->vnm_hdr_map->dm_mapsize, dma_flags);
1685 1.101 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1686 1.101 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, dma_flags);
1687 1.101 yamaguch
1688 1.101 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1689 1.101 yamaguch virtio_dequeue_commit(vsc, vq, slot);
1690 1.101 yamaguch
1691 1.101 yamaguch return m;
1692 1.101 yamaguch }
1693 1.101 yamaguch
1694 1.101 yamaguch static void
1695 1.101 yamaguch vioif_net_intr_enable(struct vioif_softc *sc, struct virtio_softc *vsc)
1696 1.101 yamaguch {
1697 1.101 yamaguch struct vioif_netqueue *netq;
1698 1.101 yamaguch size_t i, act_qnum;
1699 1.101 yamaguch int enqueued;
1700 1.101 yamaguch
1701 1.101 yamaguch act_qnum = sc->sc_act_nvq_pairs * 2;
1702 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
1703 1.101 yamaguch netq = &sc->sc_netqs[i];
1704 1.101 yamaguch
1705 1.101 yamaguch KASSERT(!netq->netq_stopping);
1706 1.101 yamaguch KASSERT(!netq->netq_running_handle);
1707 1.101 yamaguch
1708 1.101 yamaguch enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
1709 1.101 yamaguch if (enqueued != 0) {
1710 1.101 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1711 1.101 yamaguch
1712 1.101 yamaguch mutex_enter(&netq->netq_lock);
1713 1.101 yamaguch netq->netq_running_handle = true;
1714 1.101 yamaguch vioif_net_sched_handle(sc, netq);
1715 1.101 yamaguch mutex_exit(&netq->netq_lock);
1716 1.101 yamaguch }
1717 1.101 yamaguch }
1718 1.101 yamaguch }
1719 1.101 yamaguch
1720 1.101 yamaguch static void
1721 1.101 yamaguch vioif_net_intr_disable(struct vioif_softc *sc, struct virtio_softc *vsc)
1722 1.101 yamaguch {
1723 1.101 yamaguch struct vioif_netqueue *netq;
1724 1.101 yamaguch size_t i, act_qnum;
1725 1.101 yamaguch
1726 1.101 yamaguch act_qnum = sc->sc_act_nvq_pairs * 2;
1727 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
1728 1.101 yamaguch netq = &sc->sc_netqs[i];
1729 1.101 yamaguch
1730 1.101 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1731 1.101 yamaguch }
1732 1.101 yamaguch }
1733 1.101 yamaguch
1734 1.101 yamaguch /*
1735 1.101 yamaguch * Receive implementation
1736 1.101 yamaguch */
1737 1.101 yamaguch /* enqueue mbufs to receive slots */
1738 1.101 yamaguch static void
1739 1.101 yamaguch vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq)
1740 1.101 yamaguch {
1741 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1742 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1743 1.101 yamaguch struct vioif_rx_context *rxc;
1744 1.101 yamaguch struct vioif_net_map *map;
1745 1.101 yamaguch struct mbuf *m;
1746 1.101 yamaguch int i, r, ndone = 0;
1747 1.101 yamaguch
1748 1.101 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1749 1.101 yamaguch
1750 1.101 yamaguch rxc = netq->netq_ctx;
1751 1.101 yamaguch
1752 1.101 yamaguch for (i = 0; i < vq->vq_num; i++) {
1753 1.101 yamaguch int slot;
1754 1.101 yamaguch r = virtio_enqueue_prep(vsc, vq, &slot);
1755 1.101 yamaguch if (r == EAGAIN)
1756 1.101 yamaguch break;
1757 1.101 yamaguch if (__predict_false(r != 0))
1758 1.101 yamaguch panic("enqueue_prep for rx buffers");
1759 1.101 yamaguch
1760 1.101 yamaguch MGETHDR(m, M_DONTWAIT, MT_DATA);
1761 1.101 yamaguch if (m == NULL) {
1762 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1763 1.101 yamaguch rxc->rxc_mbuf_enobufs.ev_count++;
1764 1.101 yamaguch break;
1765 1.101 yamaguch }
1766 1.101 yamaguch MCLGET(m, M_DONTWAIT);
1767 1.101 yamaguch if ((m->m_flags & M_EXT) == 0) {
1768 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1769 1.101 yamaguch m_freem(m);
1770 1.101 yamaguch rxc->rxc_mbuf_enobufs.ev_count++;
1771 1.101 yamaguch break;
1772 1.101 yamaguch }
1773 1.101 yamaguch
1774 1.101 yamaguch m->m_len = m->m_pkthdr.len = MCLBYTES;
1775 1.101 yamaguch m_adj(m, ETHER_ALIGN);
1776 1.101 yamaguch
1777 1.101 yamaguch map = &netq->netq_maps[slot];
1778 1.101 yamaguch r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_READ);
1779 1.101 yamaguch if (r != 0) {
1780 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1781 1.101 yamaguch m_freem(m);
1782 1.101 yamaguch netq->netq_mbuf_load_failed.ev_count++;
1783 1.101 yamaguch break;
1784 1.101 yamaguch }
1785 1.101 yamaguch
1786 1.101 yamaguch r = vioif_net_enqueue_rx(vsc, vq, slot, map);
1787 1.101 yamaguch if (r != 0) {
1788 1.101 yamaguch vioif_net_unload_mbuf(vsc, map);
1789 1.101 yamaguch netq->netq_enqueue_failed.ev_count++;
1790 1.101 yamaguch m_freem(m);
1791 1.101 yamaguch /* slot already freed by vioif_net_enqueue_rx */
1792 1.101 yamaguch break;
1793 1.101 yamaguch }
1794 1.101 yamaguch
1795 1.101 yamaguch ndone++;
1796 1.101 yamaguch }
1797 1.101 yamaguch
1798 1.101 yamaguch if (ndone > 0)
1799 1.101 yamaguch vioif_notify(vsc, vq);
1800 1.101 yamaguch }
1801 1.101 yamaguch
1802 1.101 yamaguch /* dequeue received packets */
1803 1.101 yamaguch static bool
1804 1.101 yamaguch vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1805 1.101 yamaguch struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
1806 1.101 yamaguch {
1807 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1808 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1809 1.101 yamaguch struct vioif_net_map *map;
1810 1.101 yamaguch struct mbuf *m;
1811 1.101 yamaguch int slot, len;
1812 1.101 yamaguch bool more;
1813 1.101 yamaguch size_t ndeq;
1814 1.101 yamaguch
1815 1.101 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1816 1.101 yamaguch
1817 1.101 yamaguch more = false;
1818 1.101 yamaguch ndeq = 0;
1819 1.101 yamaguch
1820 1.101 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
1821 1.101 yamaguch goto done;
1822 1.101 yamaguch
1823 1.101 yamaguch for (;;ndeq++) {
1824 1.101 yamaguch if (ndeq >= limit) {
1825 1.101 yamaguch more = true;
1826 1.101 yamaguch break;
1827 1.101 yamaguch }
1828 1.101 yamaguch
1829 1.101 yamaguch if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1830 1.101 yamaguch break;
1831 1.101 yamaguch
1832 1.101 yamaguch map = &netq->netq_maps[slot];
1833 1.101 yamaguch KASSERT(map->vnm_mbuf != NULL);
1834 1.101 yamaguch m = vioif_net_dequeue_commit(vsc, vq, slot,
1835 1.101 yamaguch map, BUS_DMASYNC_POSTREAD);
1836 1.101 yamaguch KASSERT(m != NULL);
1837 1.101 yamaguch
1838 1.101 yamaguch m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size;
1839 1.101 yamaguch m_set_rcvif(m, ifp);
1840 1.101 yamaguch if_percpuq_enqueue(ifp->if_percpuq, m);
1841 1.101 yamaguch }
1842 1.101 yamaguch
1843 1.101 yamaguch done:
1844 1.101 yamaguch if (ndeqp != NULL)
1845 1.101 yamaguch *ndeqp = ndeq;
1846 1.101 yamaguch
1847 1.101 yamaguch return more;
1848 1.101 yamaguch }
1849 1.101 yamaguch
1850 1.101 yamaguch static void
1851 1.101 yamaguch vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
1852 1.101 yamaguch struct vioif_netqueue *netq)
1853 1.101 yamaguch {
1854 1.101 yamaguch struct vioif_net_map *map;
1855 1.101 yamaguch struct mbuf *m;
1856 1.101 yamaguch unsigned int i, vq_num;
1857 1.101 yamaguch bool more;
1858 1.101 yamaguch
1859 1.101 yamaguch mutex_enter(&netq->netq_lock);
1860 1.101 yamaguch
1861 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1862 1.101 yamaguch for (;;) {
1863 1.101 yamaguch more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL);
1864 1.101 yamaguch if (more == false)
1865 1.101 yamaguch break;
1866 1.101 yamaguch }
1867 1.101 yamaguch
1868 1.101 yamaguch for (i = 0; i < vq_num; i++) {
1869 1.101 yamaguch map = &netq->netq_maps[i];
1870 1.101 yamaguch
1871 1.101 yamaguch m = map->vnm_mbuf;
1872 1.101 yamaguch if (m == NULL)
1873 1.101 yamaguch continue;
1874 1.101 yamaguch
1875 1.101 yamaguch vioif_net_unload_mbuf(vsc, map);
1876 1.101 yamaguch m_freem(m);
1877 1.101 yamaguch }
1878 1.101 yamaguch mutex_exit(&netq->netq_lock);
1879 1.101 yamaguch }
1880 1.101 yamaguch
1881 1.101 yamaguch static void
1882 1.101 yamaguch vioif_rx_handle_locked(void *xnetq, u_int limit)
1883 1.101 yamaguch {
1884 1.101 yamaguch struct vioif_netqueue *netq = xnetq;
1885 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1886 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1887 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1888 1.101 yamaguch bool more;
1889 1.101 yamaguch int enqueued;
1890 1.101 yamaguch size_t ndeq;
1891 1.101 yamaguch
1892 1.101 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1893 1.101 yamaguch KASSERT(!netq->netq_stopping);
1894 1.101 yamaguch
1895 1.101 yamaguch more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq);
1896 1.101 yamaguch if (ndeq > 0)
1897 1.101 yamaguch vioif_populate_rx_mbufs_locked(sc, netq);
1898 1.101 yamaguch
1899 1.101 yamaguch if (more) {
1900 1.101 yamaguch vioif_net_sched_handle(sc, netq);
1901 1.101 yamaguch return;
1902 1.101 yamaguch }
1903 1.101 yamaguch
1904 1.101 yamaguch enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
1905 1.101 yamaguch if (enqueued != 0) {
1906 1.101 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1907 1.101 yamaguch vioif_net_sched_handle(sc, netq);
1908 1.101 yamaguch return;
1909 1.101 yamaguch }
1910 1.101 yamaguch
1911 1.101 yamaguch netq->netq_running_handle = false;
1912 1.101 yamaguch }
1913 1.101 yamaguch
1914 1.101 yamaguch static int
1915 1.101 yamaguch vioif_rx_intr(void *arg)
1916 1.99 yamaguch {
1917 1.101 yamaguch struct vioif_netqueue *netq = arg;
1918 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1919 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1920 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1921 1.101 yamaguch u_int limit;
1922 1.99 yamaguch
1923 1.101 yamaguch mutex_enter(&netq->netq_lock);
1924 1.99 yamaguch
1925 1.101 yamaguch /* handler is already running in softint/workqueue */
1926 1.101 yamaguch if (netq->netq_running_handle)
1927 1.101 yamaguch goto done;
1928 1.99 yamaguch
1929 1.108 yamaguch if (netq->netq_stopping)
1930 1.108 yamaguch goto done;
1931 1.108 yamaguch
1932 1.101 yamaguch netq->netq_running_handle = true;
1933 1.99 yamaguch
1934 1.101 yamaguch limit = sc->sc_rx_intr_process_limit;
1935 1.101 yamaguch virtio_stop_vq_intr(vsc, vq);
1936 1.101 yamaguch vioif_rx_handle_locked(netq, limit);
1937 1.99 yamaguch
1938 1.101 yamaguch done:
1939 1.101 yamaguch mutex_exit(&netq->netq_lock);
1940 1.101 yamaguch return 1;
1941 1.99 yamaguch }
1942 1.99 yamaguch
1943 1.99 yamaguch static void
1944 1.101 yamaguch vioif_rx_handle(void *xnetq)
1945 1.99 yamaguch {
1946 1.101 yamaguch struct vioif_netqueue *netq = xnetq;
1947 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1948 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1949 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1950 1.101 yamaguch u_int limit;
1951 1.99 yamaguch
1952 1.101 yamaguch mutex_enter(&netq->netq_lock);
1953 1.99 yamaguch
1954 1.101 yamaguch KASSERT(netq->netq_running_handle);
1955 1.99 yamaguch
1956 1.101 yamaguch if (netq->netq_stopping) {
1957 1.101 yamaguch netq->netq_running_handle = false;
1958 1.101 yamaguch goto done;
1959 1.99 yamaguch }
1960 1.99 yamaguch
1961 1.101 yamaguch limit = sc->sc_rx_process_limit;
1962 1.101 yamaguch vioif_rx_handle_locked(netq, limit);
1963 1.99 yamaguch
1964 1.101 yamaguch done:
1965 1.101 yamaguch mutex_exit(&netq->netq_lock);
1966 1.99 yamaguch }
1967 1.99 yamaguch
1968 1.1 hannken /*
1969 1.110 andvar * Transmission implementation
1970 1.1 hannken */
1971 1.101 yamaguch /* enqueue mbufs to send */
1972 1.1 hannken static void
1973 1.101 yamaguch vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq,
1974 1.101 yamaguch bool is_transmit)
1975 1.12 ozaki {
1976 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
1977 1.101 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1978 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1979 1.101 yamaguch struct vioif_tx_context *txc;
1980 1.95 yamaguch struct vioif_net_map *map;
1981 1.89 yamaguch struct mbuf *m;
1982 1.101 yamaguch int queued = 0;
1983 1.1 hannken
1984 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1985 1.98 yamaguch
1986 1.101 yamaguch if (netq->netq_stopping ||
1987 1.101 yamaguch !ISSET(ifp->if_flags, IFF_RUNNING))
1988 1.101 yamaguch return;
1989 1.101 yamaguch
1990 1.101 yamaguch txc = netq->netq_ctx;
1991 1.101 yamaguch
1992 1.102 yamaguch if (!txc->txc_link_active ||
1993 1.102 yamaguch txc->txc_no_free_slots)
1994 1.101 yamaguch return;
1995 1.7 ozaki
1996 1.101 yamaguch for (;;) {
1997 1.101 yamaguch int slot, r;
1998 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
1999 1.102 yamaguch if (r == EAGAIN) {
2000 1.102 yamaguch txc->txc_no_free_slots = true;
2001 1.1 hannken break;
2002 1.102 yamaguch }
2003 1.97 yamaguch if (__predict_false(r != 0))
2004 1.101 yamaguch panic("enqueue_prep for tx buffers");
2005 1.101 yamaguch
2006 1.101 yamaguch if (is_transmit)
2007 1.101 yamaguch m = pcq_get(txc->txc_intrq);
2008 1.101 yamaguch else
2009 1.101 yamaguch IFQ_DEQUEUE(&ifp->if_snd, m);
2010 1.89 yamaguch
2011 1.89 yamaguch if (m == NULL) {
2012 1.91 yamaguch virtio_enqueue_abort(vsc, vq, slot);
2013 1.91 yamaguch break;
2014 1.91 yamaguch }
2015 1.89 yamaguch
2016 1.101 yamaguch map = &netq->netq_maps[slot];
2017 1.101 yamaguch KASSERT(map->vnm_mbuf == NULL);
2018 1.89 yamaguch
2019 1.101 yamaguch r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE);
2020 1.91 yamaguch if (r != 0) {
2021 1.101 yamaguch /* maybe just too fragmented */
2022 1.101 yamaguch struct mbuf *newm;
2023 1.101 yamaguch
2024 1.101 yamaguch newm = m_defrag(m, M_NOWAIT);
2025 1.101 yamaguch if (newm != NULL) {
2026 1.101 yamaguch m = newm;
2027 1.101 yamaguch r = vioif_net_load_mbuf(vsc, map, m,
2028 1.101 yamaguch BUS_DMA_WRITE);
2029 1.101 yamaguch } else {
2030 1.101 yamaguch txc->txc_defrag_failed.ev_count++;
2031 1.101 yamaguch r = -1;
2032 1.101 yamaguch }
2033 1.101 yamaguch
2034 1.101 yamaguch if (r != 0) {
2035 1.101 yamaguch netq->netq_mbuf_load_failed.ev_count++;
2036 1.101 yamaguch m_freem(m);
2037 1.101 yamaguch if_statinc(ifp, if_oerrors);
2038 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
2039 1.101 yamaguch continue;
2040 1.101 yamaguch }
2041 1.1 hannken }
2042 1.89 yamaguch
2043 1.101 yamaguch memset(map->vnm_hdr, 0, sc->sc_hdr_size);
2044 1.101 yamaguch
2045 1.101 yamaguch r = vioif_net_enqueue_tx(vsc, vq, slot, map);
2046 1.1 hannken if (r != 0) {
2047 1.101 yamaguch netq->netq_enqueue_failed.ev_count++;
2048 1.99 yamaguch vioif_net_unload_mbuf(vsc, map);
2049 1.89 yamaguch m_freem(m);
2050 1.101 yamaguch /* slot already freed by vioif_net_enqueue_tx */
2051 1.101 yamaguch
2052 1.101 yamaguch if_statinc(ifp, if_oerrors);
2053 1.101 yamaguch continue;
2054 1.1 hannken }
2055 1.95 yamaguch
2056 1.101 yamaguch queued++;
2057 1.101 yamaguch bpf_mtap(ifp, m, BPF_D_OUT);
2058 1.1 hannken }
2059 1.99 yamaguch
2060 1.101 yamaguch if (queued > 0) {
2061 1.99 yamaguch vioif_notify(vsc, vq);
2062 1.101 yamaguch ifp->if_timer = 5;
2063 1.93 yamaguch }
2064 1.7 ozaki }
2065 1.7 ozaki
2066 1.101 yamaguch /* dequeue sent mbufs */
2067 1.55 yamaguch static bool
2068 1.101 yamaguch vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
2069 1.102 yamaguch struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
2070 1.7 ozaki {
2071 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2072 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2073 1.95 yamaguch struct vioif_net_map *map;
2074 1.1 hannken struct mbuf *m;
2075 1.1 hannken int slot, len;
2076 1.102 yamaguch bool more;
2077 1.102 yamaguch size_t ndeq;
2078 1.1 hannken
2079 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
2080 1.7 ozaki
2081 1.102 yamaguch more = false;
2082 1.102 yamaguch ndeq = 0;
2083 1.102 yamaguch
2084 1.54 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
2085 1.102 yamaguch goto done;
2086 1.55 yamaguch
2087 1.102 yamaguch for (;;ndeq++) {
2088 1.101 yamaguch if (limit-- == 0) {
2089 1.55 yamaguch more = true;
2090 1.55 yamaguch break;
2091 1.55 yamaguch }
2092 1.55 yamaguch
2093 1.55 yamaguch if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
2094 1.55 yamaguch break;
2095 1.55 yamaguch
2096 1.98 yamaguch map = &netq->netq_maps[slot];
2097 1.95 yamaguch KASSERT(map->vnm_mbuf != NULL);
2098 1.99 yamaguch m = vioif_net_dequeue_commit(vsc, vq, slot,
2099 1.101 yamaguch map, BUS_DMASYNC_POSTWRITE);
2100 1.99 yamaguch KASSERT(m != NULL);
2101 1.95 yamaguch
2102 1.101 yamaguch if_statinc(ifp, if_opackets);
2103 1.101 yamaguch m_freem(m);
2104 1.70 skrll }
2105 1.87 yamaguch
2106 1.102 yamaguch done:
2107 1.102 yamaguch if (ndeqp != NULL)
2108 1.102 yamaguch *ndeqp = ndeq;
2109 1.101 yamaguch return more;
2110 1.66 reinoud }
2111 1.66 reinoud
2112 1.101 yamaguch static void
2113 1.101 yamaguch vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
2114 1.101 yamaguch struct vioif_netqueue *netq)
2115 1.1 hannken {
2116 1.102 yamaguch struct vioif_tx_context *txc;
2117 1.101 yamaguch struct vioif_net_map *map;
2118 1.101 yamaguch struct mbuf *m;
2119 1.101 yamaguch unsigned int i, vq_num;
2120 1.101 yamaguch bool more;
2121 1.55 yamaguch
2122 1.98 yamaguch mutex_enter(&netq->netq_lock);
2123 1.7 ozaki
2124 1.102 yamaguch txc = netq->netq_ctx;
2125 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
2126 1.102 yamaguch
2127 1.101 yamaguch for (;;) {
2128 1.102 yamaguch more = vioif_tx_deq_locked(sc, vsc, netq, vq_num, NULL);
2129 1.101 yamaguch if (more == false)
2130 1.101 yamaguch break;
2131 1.101 yamaguch }
2132 1.55 yamaguch
2133 1.101 yamaguch for (i = 0; i < vq_num; i++) {
2134 1.101 yamaguch map = &netq->netq_maps[i];
2135 1.55 yamaguch
2136 1.101 yamaguch m = map->vnm_mbuf;
2137 1.101 yamaguch if (m == NULL)
2138 1.101 yamaguch continue;
2139 1.55 yamaguch
2140 1.101 yamaguch vioif_net_unload_mbuf(vsc, map);
2141 1.101 yamaguch m_freem(m);
2142 1.101 yamaguch }
2143 1.102 yamaguch
2144 1.102 yamaguch txc->txc_no_free_slots = false;
2145 1.102 yamaguch
2146 1.98 yamaguch mutex_exit(&netq->netq_lock);
2147 1.55 yamaguch }
2148 1.55 yamaguch
2149 1.55 yamaguch static void
2150 1.101 yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
2151 1.55 yamaguch {
2152 1.55 yamaguch
2153 1.101 yamaguch /*
2154 1.101 yamaguch * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
2155 1.101 yamaguch */
2156 1.101 yamaguch vioif_send_common_locked(ifp, netq, false);
2157 1.101 yamaguch
2158 1.101 yamaguch }
2159 1.85 yamaguch
2160 1.101 yamaguch static void
2161 1.101 yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
2162 1.101 yamaguch {
2163 1.55 yamaguch
2164 1.101 yamaguch vioif_send_common_locked(ifp, netq, true);
2165 1.101 yamaguch }
2166 1.1 hannken
2167 1.101 yamaguch static void
2168 1.101 yamaguch vioif_deferred_transmit(void *arg)
2169 1.101 yamaguch {
2170 1.101 yamaguch struct vioif_netqueue *netq = arg;
2171 1.101 yamaguch struct virtio_softc *vsc = netq->netq_vq->vq_owner;
2172 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
2173 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2174 1.1 hannken
2175 1.101 yamaguch mutex_enter(&netq->netq_lock);
2176 1.101 yamaguch vioif_send_common_locked(ifp, netq, true);
2177 1.98 yamaguch mutex_exit(&netq->netq_lock);
2178 1.1 hannken }
2179 1.1 hannken
2180 1.66 reinoud static void
2181 1.98 yamaguch vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit)
2182 1.66 reinoud {
2183 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2184 1.98 yamaguch struct vioif_tx_context *txc = netq->netq_ctx;
2185 1.66 reinoud struct virtio_softc *vsc = vq->vq_owner;
2186 1.66 reinoud struct vioif_softc *sc = device_private(virtio_child(vsc));
2187 1.66 reinoud struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2188 1.66 reinoud bool more;
2189 1.87 yamaguch int enqueued;
2190 1.102 yamaguch size_t ndeq;
2191 1.66 reinoud
2192 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
2193 1.98 yamaguch KASSERT(!netq->netq_stopping);
2194 1.66 reinoud
2195 1.102 yamaguch more = vioif_tx_deq_locked(sc, vsc, netq, limit, &ndeq);
2196 1.102 yamaguch if (txc->txc_no_free_slots && ndeq > 0) {
2197 1.102 yamaguch txc->txc_no_free_slots = false;
2198 1.102 yamaguch softint_schedule(txc->txc_deferred_transmit);
2199 1.102 yamaguch }
2200 1.102 yamaguch
2201 1.66 reinoud if (more) {
2202 1.98 yamaguch vioif_net_sched_handle(sc, netq);
2203 1.66 reinoud return;
2204 1.66 reinoud }
2205 1.66 reinoud
2206 1.87 yamaguch enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ?
2207 1.87 yamaguch virtio_postpone_intr_smart(vsc, vq):
2208 1.87 yamaguch virtio_start_vq_intr(vsc, vq);
2209 1.87 yamaguch if (enqueued != 0) {
2210 1.87 yamaguch virtio_stop_vq_intr(vsc, vq);
2211 1.98 yamaguch vioif_net_sched_handle(sc, netq);
2212 1.66 reinoud return;
2213 1.66 reinoud }
2214 1.66 reinoud
2215 1.98 yamaguch netq->netq_running_handle = false;
2216 1.84 yamaguch
2217 1.66 reinoud /* for ALTQ */
2218 1.102 yamaguch if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)])
2219 1.66 reinoud if_schedule_deferred_start(ifp);
2220 1.102 yamaguch
2221 1.98 yamaguch softint_schedule(txc->txc_deferred_transmit);
2222 1.66 reinoud }
2223 1.66 reinoud
2224 1.1 hannken static int
2225 1.54 yamaguch vioif_tx_intr(void *arg)
2226 1.1 hannken {
2227 1.98 yamaguch struct vioif_netqueue *netq = arg;
2228 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2229 1.1 hannken struct virtio_softc *vsc = vq->vq_owner;
2230 1.32 jdolecek struct vioif_softc *sc = device_private(virtio_child(vsc));
2231 1.55 yamaguch u_int limit;
2232 1.55 yamaguch
2233 1.98 yamaguch mutex_enter(&netq->netq_lock);
2234 1.7 ozaki
2235 1.84 yamaguch /* tx handler is already running in softint/workqueue */
2236 1.98 yamaguch if (netq->netq_running_handle)
2237 1.84 yamaguch goto done;
2238 1.7 ozaki
2239 1.98 yamaguch if (netq->netq_stopping)
2240 1.84 yamaguch goto done;
2241 1.55 yamaguch
2242 1.98 yamaguch netq->netq_running_handle = true;
2243 1.55 yamaguch
2244 1.84 yamaguch virtio_stop_vq_intr(vsc, vq);
2245 1.98 yamaguch netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl;
2246 1.98 yamaguch limit = sc->sc_tx_intr_process_limit;
2247 1.98 yamaguch vioif_tx_handle_locked(netq, limit);
2248 1.7 ozaki
2249 1.84 yamaguch done:
2250 1.98 yamaguch mutex_exit(&netq->netq_lock);
2251 1.55 yamaguch return 1;
2252 1.55 yamaguch }
2253 1.55 yamaguch
2254 1.55 yamaguch static void
2255 1.98 yamaguch vioif_tx_handle(void *xnetq)
2256 1.55 yamaguch {
2257 1.98 yamaguch struct vioif_netqueue *netq = xnetq;
2258 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2259 1.55 yamaguch struct virtio_softc *vsc = vq->vq_owner;
2260 1.55 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
2261 1.55 yamaguch u_int limit;
2262 1.55 yamaguch
2263 1.98 yamaguch mutex_enter(&netq->netq_lock);
2264 1.85 yamaguch
2265 1.98 yamaguch KASSERT(netq->netq_running_handle);
2266 1.85 yamaguch
2267 1.98 yamaguch if (netq->netq_stopping) {
2268 1.98 yamaguch netq->netq_running_handle = false;
2269 1.85 yamaguch goto done;
2270 1.85 yamaguch }
2271 1.85 yamaguch
2272 1.55 yamaguch limit = sc->sc_tx_process_limit;
2273 1.98 yamaguch vioif_tx_handle_locked(netq, limit);
2274 1.55 yamaguch
2275 1.85 yamaguch done:
2276 1.98 yamaguch mutex_exit(&netq->netq_lock);
2277 1.7 ozaki }
2278 1.7 ozaki
2279 1.1 hannken /*
2280 1.1 hannken * Control vq
2281 1.1 hannken */
2282 1.1 hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
2283 1.44 yamaguch static void
2284 1.44 yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
2285 1.1 hannken {
2286 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2287 1.1 hannken
2288 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2289 1.43 yamaguch while (ctrlq->ctrlq_inuse != FREE)
2290 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
2291 1.43 yamaguch ctrlq->ctrlq_inuse = INUSE;
2292 1.44 yamaguch ctrlq->ctrlq_owner = curlwp;
2293 1.44 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2294 1.44 yamaguch }
2295 1.44 yamaguch
2296 1.44 yamaguch static void
2297 1.44 yamaguch vioif_ctrl_release(struct vioif_softc *sc)
2298 1.44 yamaguch {
2299 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2300 1.44 yamaguch
2301 1.44 yamaguch KASSERT(ctrlq->ctrlq_inuse != FREE);
2302 1.44 yamaguch KASSERT(ctrlq->ctrlq_owner == curlwp);
2303 1.44 yamaguch
2304 1.44 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2305 1.44 yamaguch ctrlq->ctrlq_inuse = FREE;
2306 1.46 yamaguch ctrlq->ctrlq_owner = NULL;
2307 1.44 yamaguch cv_signal(&ctrlq->ctrlq_wait);
2308 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2309 1.44 yamaguch }
2310 1.44 yamaguch
2311 1.44 yamaguch static int
2312 1.44 yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
2313 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2314 1.44 yamaguch {
2315 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2316 1.44 yamaguch int i, r, loaded;
2317 1.44 yamaguch
2318 1.44 yamaguch loaded = 0;
2319 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2320 1.44 yamaguch r = bus_dmamap_load(virtio_dmat(vsc),
2321 1.44 yamaguch specs[i].dmamap, specs[i].buf, specs[i].bufsize,
2322 1.48 msaitoh NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
2323 1.44 yamaguch if (r) {
2324 1.63 yamaguch sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++;
2325 1.44 yamaguch goto err;
2326 1.44 yamaguch }
2327 1.44 yamaguch loaded++;
2328 1.44 yamaguch
2329 1.44 yamaguch }
2330 1.44 yamaguch
2331 1.44 yamaguch return r;
2332 1.44 yamaguch
2333 1.44 yamaguch err:
2334 1.44 yamaguch for (i = 0; i < loaded; i++) {
2335 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2336 1.44 yamaguch }
2337 1.44 yamaguch
2338 1.44 yamaguch return r;
2339 1.44 yamaguch }
2340 1.44 yamaguch
2341 1.44 yamaguch static void
2342 1.44 yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
2343 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2344 1.44 yamaguch {
2345 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2346 1.44 yamaguch int i;
2347 1.44 yamaguch
2348 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2349 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2350 1.44 yamaguch }
2351 1.44 yamaguch }
2352 1.44 yamaguch
2353 1.44 yamaguch static int
2354 1.44 yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
2355 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2356 1.44 yamaguch {
2357 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2358 1.44 yamaguch struct virtqueue *vq = ctrlq->ctrlq_vq;
2359 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2360 1.44 yamaguch int i, r, slot;
2361 1.43 yamaguch
2362 1.44 yamaguch ctrlq->ctrlq_cmd->class = class;
2363 1.43 yamaguch ctrlq->ctrlq_cmd->command = cmd;
2364 1.1 hannken
2365 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
2366 1.50 christos 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
2367 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2368 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
2369 1.50 christos 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
2370 1.44 yamaguch }
2371 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
2372 1.50 christos 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
2373 1.1 hannken
2374 1.66 reinoud /* we need to explicitly (re)start vq intr when using RING EVENT IDX */
2375 1.66 reinoud if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
2376 1.66 reinoud virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
2377 1.66 reinoud
2378 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
2379 1.1 hannken if (r != 0)
2380 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2381 1.44 yamaguch r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
2382 1.1 hannken if (r != 0)
2383 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2384 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
2385 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2386 1.44 yamaguch virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
2387 1.44 yamaguch }
2388 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
2389 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, true);
2390 1.1 hannken
2391 1.1 hannken /* wait for done */
2392 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2393 1.43 yamaguch while (ctrlq->ctrlq_inuse != DONE)
2394 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
2395 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2396 1.109 andvar /* already dequeued */
2397 1.1 hannken
2398 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
2399 1.50 christos sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
2400 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2401 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
2402 1.50 christos specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
2403 1.44 yamaguch }
2404 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
2405 1.50 christos sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
2406 1.1 hannken
2407 1.43 yamaguch if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
2408 1.1 hannken r = 0;
2409 1.1 hannken else {
2410 1.63 yamaguch device_printf(sc->sc_dev, "failed setting rx mode\n");
2411 1.63 yamaguch sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++;
2412 1.1 hannken r = EIO;
2413 1.1 hannken }
2414 1.1 hannken
2415 1.44 yamaguch return r;
2416 1.44 yamaguch }
2417 1.44 yamaguch
2418 1.101 yamaguch /* ctrl vq interrupt; wake up the command issuer */
2419 1.101 yamaguch static int
2420 1.101 yamaguch vioif_ctrl_intr(void *arg)
2421 1.101 yamaguch {
2422 1.101 yamaguch struct vioif_ctrlqueue *ctrlq = arg;
2423 1.101 yamaguch struct virtqueue *vq = ctrlq->ctrlq_vq;
2424 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
2425 1.101 yamaguch int r, slot;
2426 1.101 yamaguch
2427 1.101 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
2428 1.101 yamaguch return 0;
2429 1.101 yamaguch
2430 1.101 yamaguch r = virtio_dequeue(vsc, vq, &slot, NULL);
2431 1.101 yamaguch if (r == ENOENT)
2432 1.101 yamaguch return 0;
2433 1.101 yamaguch virtio_dequeue_commit(vsc, vq, slot);
2434 1.101 yamaguch
2435 1.101 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2436 1.101 yamaguch ctrlq->ctrlq_inuse = DONE;
2437 1.101 yamaguch cv_signal(&ctrlq->ctrlq_wait);
2438 1.101 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2439 1.101 yamaguch
2440 1.101 yamaguch return 1;
2441 1.101 yamaguch }
2442 1.101 yamaguch
2443 1.44 yamaguch static int
2444 1.44 yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
2445 1.44 yamaguch {
2446 1.44 yamaguch struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
2447 1.44 yamaguch struct vioif_ctrl_cmdspec specs[1];
2448 1.44 yamaguch int r;
2449 1.44 yamaguch
2450 1.44 yamaguch if (!sc->sc_has_ctrl)
2451 1.44 yamaguch return ENOTSUP;
2452 1.44 yamaguch
2453 1.44 yamaguch vioif_ctrl_acquire(sc);
2454 1.44 yamaguch
2455 1.44 yamaguch rx->onoff = onoff;
2456 1.44 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
2457 1.44 yamaguch specs[0].buf = rx;
2458 1.44 yamaguch specs[0].bufsize = sizeof(*rx);
2459 1.44 yamaguch
2460 1.44 yamaguch r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
2461 1.44 yamaguch specs, __arraycount(specs));
2462 1.3 christos
2463 1.44 yamaguch vioif_ctrl_release(sc);
2464 1.1 hannken return r;
2465 1.1 hannken }
2466 1.1 hannken
2467 1.1 hannken static int
2468 1.1 hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
2469 1.1 hannken {
2470 1.50 christos return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
2471 1.1 hannken }
2472 1.1 hannken
2473 1.1 hannken static int
2474 1.1 hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
2475 1.1 hannken {
2476 1.50 christos return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
2477 1.1 hannken }
2478 1.1 hannken
2479 1.1 hannken static int
2480 1.101 yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
2481 1.1 hannken {
2482 1.101 yamaguch struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
2483 1.101 yamaguch struct vioif_ctrl_cmdspec specs[1];
2484 1.44 yamaguch int r;
2485 1.44 yamaguch
2486 1.32 jdolecek if (!sc->sc_has_ctrl)
2487 1.1 hannken return ENOTSUP;
2488 1.1 hannken
2489 1.101 yamaguch if (nvq_pairs <= 1)
2490 1.101 yamaguch return EINVAL;
2491 1.101 yamaguch
2492 1.44 yamaguch vioif_ctrl_acquire(sc);
2493 1.1 hannken
2494 1.101 yamaguch mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs);
2495 1.101 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
2496 1.101 yamaguch specs[0].buf = mq;
2497 1.101 yamaguch specs[0].bufsize = sizeof(*mq);
2498 1.1 hannken
2499 1.44 yamaguch r = vioif_ctrl_send_command(sc,
2500 1.101 yamaguch VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
2501 1.101 yamaguch specs, __arraycount(specs));
2502 1.1 hannken
2503 1.44 yamaguch vioif_ctrl_release(sc);
2504 1.3 christos
2505 1.1 hannken return r;
2506 1.1 hannken }
2507 1.1 hannken
2508 1.46 yamaguch static int
2509 1.74 yamaguch vioif_set_mac_addr(struct vioif_softc *sc)
2510 1.74 yamaguch {
2511 1.74 yamaguch struct virtio_net_ctrl_mac_addr *ma =
2512 1.74 yamaguch sc->sc_ctrlq.ctrlq_mac_addr;
2513 1.74 yamaguch struct vioif_ctrl_cmdspec specs[1];
2514 1.74 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2515 1.74 yamaguch int nspecs = __arraycount(specs);
2516 1.78 yamaguch uint64_t features;
2517 1.74 yamaguch int r;
2518 1.78 yamaguch size_t i;
2519 1.74 yamaguch
2520 1.74 yamaguch if (!sc->sc_has_ctrl)
2521 1.74 yamaguch return ENOTSUP;
2522 1.74 yamaguch
2523 1.78 yamaguch if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac,
2524 1.78 yamaguch ETHER_ADDR_LEN) == 0) {
2525 1.78 yamaguch return 0;
2526 1.78 yamaguch }
2527 1.74 yamaguch
2528 1.78 yamaguch memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2529 1.74 yamaguch
2530 1.78 yamaguch features = virtio_features(sc->sc_virtio);
2531 1.78 yamaguch if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) {
2532 1.78 yamaguch vioif_ctrl_acquire(sc);
2533 1.78 yamaguch
2534 1.78 yamaguch memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN);
2535 1.78 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap;
2536 1.78 yamaguch specs[0].buf = ma;
2537 1.78 yamaguch specs[0].bufsize = sizeof(*ma);
2538 1.78 yamaguch
2539 1.78 yamaguch r = vioif_ctrl_send_command(sc,
2540 1.78 yamaguch VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET,
2541 1.78 yamaguch specs, nspecs);
2542 1.74 yamaguch
2543 1.78 yamaguch vioif_ctrl_release(sc);
2544 1.78 yamaguch } else {
2545 1.78 yamaguch for (i = 0; i < __arraycount(sc->sc_mac); i++) {
2546 1.78 yamaguch virtio_write_device_config_1(sc->sc_virtio,
2547 1.78 yamaguch VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
2548 1.78 yamaguch }
2549 1.78 yamaguch r = 0;
2550 1.78 yamaguch }
2551 1.74 yamaguch
2552 1.74 yamaguch return r;
2553 1.74 yamaguch }
2554 1.74 yamaguch
2555 1.74 yamaguch static int
2556 1.101 yamaguch vioif_set_rx_filter(struct vioif_softc *sc)
2557 1.1 hannken {
2558 1.101 yamaguch /* filter already set in ctrlq->ctrlq_mac_tbl */
2559 1.101 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2560 1.101 yamaguch struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
2561 1.101 yamaguch struct vioif_ctrl_cmdspec specs[2];
2562 1.101 yamaguch int nspecs = __arraycount(specs);
2563 1.101 yamaguch int r;
2564 1.1 hannken
2565 1.101 yamaguch mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
2566 1.101 yamaguch mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
2567 1.54 yamaguch
2568 1.101 yamaguch if (!sc->sc_has_ctrl)
2569 1.101 yamaguch return ENOTSUP;
2570 1.1 hannken
2571 1.101 yamaguch vioif_ctrl_acquire(sc);
2572 1.1 hannken
2573 1.101 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
2574 1.101 yamaguch specs[0].buf = mac_tbl_uc;
2575 1.101 yamaguch specs[0].bufsize = sizeof(*mac_tbl_uc)
2576 1.101 yamaguch + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries));
2577 1.1 hannken
2578 1.101 yamaguch specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
2579 1.101 yamaguch specs[1].buf = mac_tbl_mc;
2580 1.101 yamaguch specs[1].bufsize = sizeof(*mac_tbl_mc)
2581 1.101 yamaguch + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries));
2582 1.75 yamaguch
2583 1.101 yamaguch r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
2584 1.101 yamaguch if (r != 0)
2585 1.101 yamaguch goto out;
2586 1.75 yamaguch
2587 1.101 yamaguch r = vioif_ctrl_send_command(sc,
2588 1.101 yamaguch VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
2589 1.101 yamaguch specs, nspecs);
2590 1.75 yamaguch
2591 1.101 yamaguch vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
2592 1.75 yamaguch
2593 1.101 yamaguch out:
2594 1.101 yamaguch vioif_ctrl_release(sc);
2595 1.75 yamaguch
2596 1.101 yamaguch return r;
2597 1.75 yamaguch }
2598 1.75 yamaguch
2599 1.1 hannken /*
2600 1.1 hannken * If multicast filter small enough (<=MAXENTRIES) set rx filter
2601 1.1 hannken * If large multicast filter exist use ALLMULTI
2602 1.1 hannken * If setting rx filter fails fall back to ALLMULTI
2603 1.1 hannken */
2604 1.1 hannken static int
2605 1.1 hannken vioif_rx_filter(struct vioif_softc *sc)
2606 1.1 hannken {
2607 1.66 reinoud struct virtio_softc *vsc = sc->sc_virtio;
2608 1.48 msaitoh struct ethercom *ec = &sc->sc_ethercom;
2609 1.48 msaitoh struct ifnet *ifp = &ec->ec_if;
2610 1.1 hannken struct ether_multi *enm;
2611 1.1 hannken struct ether_multistep step;
2612 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2613 1.1 hannken int nentries;
2614 1.75 yamaguch bool allmulti = 0;
2615 1.1 hannken int r;
2616 1.1 hannken
2617 1.75 yamaguch if (!sc->sc_has_ctrl) {
2618 1.75 yamaguch goto set_ifflags;
2619 1.1 hannken }
2620 1.1 hannken
2621 1.74 yamaguch memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0],
2622 1.74 yamaguch CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2623 1.74 yamaguch
2624 1.75 yamaguch nentries = 0;
2625 1.75 yamaguch allmulti = false;
2626 1.75 yamaguch
2627 1.48 msaitoh ETHER_LOCK(ec);
2628 1.75 yamaguch for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
2629 1.75 yamaguch ETHER_NEXT_MULTI(step, enm)) {
2630 1.1 hannken if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
2631 1.75 yamaguch allmulti = true;
2632 1.75 yamaguch break;
2633 1.1 hannken }
2634 1.50 christos if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
2635 1.75 yamaguch allmulti = true;
2636 1.75 yamaguch break;
2637 1.1 hannken }
2638 1.75 yamaguch
2639 1.43 yamaguch memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
2640 1.50 christos enm->enm_addrlo, ETHER_ADDR_LEN);
2641 1.75 yamaguch nentries++;
2642 1.1 hannken }
2643 1.48 msaitoh ETHER_UNLOCK(ec);
2644 1.30 ozaki
2645 1.74 yamaguch r = vioif_set_mac_addr(sc);
2646 1.74 yamaguch if (r != 0) {
2647 1.74 yamaguch log(LOG_WARNING, "%s: couldn't set MAC address\n",
2648 1.74 yamaguch ifp->if_xname);
2649 1.74 yamaguch }
2650 1.74 yamaguch
2651 1.75 yamaguch if (!allmulti) {
2652 1.74 yamaguch ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1);
2653 1.66 reinoud ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries);
2654 1.1 hannken r = vioif_set_rx_filter(sc);
2655 1.1 hannken if (r != 0) {
2656 1.75 yamaguch allmulti = true; /* fallback */
2657 1.1 hannken }
2658 1.75 yamaguch }
2659 1.75 yamaguch
2660 1.75 yamaguch if (allmulti) {
2661 1.66 reinoud ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0);
2662 1.66 reinoud ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0);
2663 1.1 hannken r = vioif_set_rx_filter(sc);
2664 1.1 hannken if (r != 0) {
2665 1.75 yamaguch log(LOG_DEBUG, "%s: couldn't clear RX filter\n",
2666 1.75 yamaguch ifp->if_xname);
2667 1.75 yamaguch /* what to do on failure? */
2668 1.1 hannken }
2669 1.75 yamaguch
2670 1.75 yamaguch ifp->if_flags |= IFF_ALLMULTI;
2671 1.1 hannken }
2672 1.75 yamaguch
2673 1.75 yamaguch set_ifflags:
2674 1.75 yamaguch r = vioif_ifflags(sc);
2675 1.1 hannken
2676 1.1 hannken return r;
2677 1.1 hannken }
2678 1.1 hannken
2679 1.101 yamaguch /*
2680 1.101 yamaguch * VM configuration changes
2681 1.101 yamaguch */
2682 1.101 yamaguch static int
2683 1.101 yamaguch vioif_config_change(struct virtio_softc *vsc)
2684 1.101 yamaguch {
2685 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
2686 1.101 yamaguch
2687 1.101 yamaguch softint_schedule(sc->sc_cfg_softint);
2688 1.101 yamaguch return 0;
2689 1.101 yamaguch }
2690 1.101 yamaguch
2691 1.101 yamaguch static void
2692 1.101 yamaguch vioif_cfg_softint(void *arg)
2693 1.101 yamaguch {
2694 1.101 yamaguch struct vioif_softc *sc = arg;
2695 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2696 1.101 yamaguch
2697 1.101 yamaguch vioif_update_link_status(sc);
2698 1.101 yamaguch vioif_start(ifp);
2699 1.101 yamaguch }
2700 1.101 yamaguch
2701 1.82 knakahar static int
2702 1.82 knakahar vioif_get_link_status(struct vioif_softc *sc)
2703 1.34 ozaki {
2704 1.34 ozaki struct virtio_softc *vsc = sc->sc_virtio;
2705 1.34 ozaki uint16_t status;
2706 1.34 ozaki
2707 1.34 ozaki if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
2708 1.34 ozaki status = virtio_read_device_config_2(vsc,
2709 1.34 ozaki VIRTIO_NET_CONFIG_STATUS);
2710 1.34 ozaki else
2711 1.34 ozaki status = VIRTIO_NET_S_LINK_UP;
2712 1.34 ozaki
2713 1.82 knakahar if ((status & VIRTIO_NET_S_LINK_UP) != 0)
2714 1.82 knakahar return LINK_STATE_UP;
2715 1.82 knakahar
2716 1.82 knakahar return LINK_STATE_DOWN;
2717 1.34 ozaki }
2718 1.34 ozaki
2719 1.34 ozaki static void
2720 1.34 ozaki vioif_update_link_status(struct vioif_softc *sc)
2721 1.34 ozaki {
2722 1.34 ozaki struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2723 1.98 yamaguch struct vioif_netqueue *netq;
2724 1.98 yamaguch struct vioif_tx_context *txc;
2725 1.82 knakahar bool active;
2726 1.46 yamaguch int link, i;
2727 1.34 ozaki
2728 1.62 yamaguch mutex_enter(&sc->sc_lock);
2729 1.62 yamaguch
2730 1.82 knakahar link = vioif_get_link_status(sc);
2731 1.34 ozaki
2732 1.82 knakahar if (link == sc->sc_link_state)
2733 1.82 knakahar goto done;
2734 1.34 ozaki
2735 1.82 knakahar sc->sc_link_state = link;
2736 1.82 knakahar
2737 1.82 knakahar active = VIOIF_IS_LINK_ACTIVE(sc);
2738 1.82 knakahar for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
2739 1.98 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
2740 1.34 ozaki
2741 1.98 yamaguch mutex_enter(&netq->netq_lock);
2742 1.98 yamaguch txc = netq->netq_ctx;
2743 1.98 yamaguch txc->txc_link_active = active;
2744 1.98 yamaguch mutex_exit(&netq->netq_lock);
2745 1.34 ozaki }
2746 1.34 ozaki
2747 1.82 knakahar if_link_state_change(ifp, sc->sc_link_state);
2748 1.62 yamaguch
2749 1.82 knakahar done:
2750 1.62 yamaguch mutex_exit(&sc->sc_lock);
2751 1.34 ozaki }
2752 1.34 ozaki
2753 1.101 yamaguch static void
2754 1.101 yamaguch vioif_workq_work(struct work *wk, void *context)
2755 1.34 ozaki {
2756 1.101 yamaguch struct vioif_work *work;
2757 1.34 ozaki
2758 1.101 yamaguch work = container_of(wk, struct vioif_work, cookie);
2759 1.34 ozaki
2760 1.101 yamaguch atomic_store_relaxed(&work->added, 0);
2761 1.101 yamaguch work->func(work->arg);
2762 1.34 ozaki }
2763 1.34 ozaki
2764 1.55 yamaguch static struct workqueue *
2765 1.55 yamaguch vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
2766 1.55 yamaguch {
2767 1.55 yamaguch struct workqueue *wq;
2768 1.55 yamaguch int error;
2769 1.55 yamaguch
2770 1.55 yamaguch error = workqueue_create(&wq, name, vioif_workq_work, NULL,
2771 1.55 yamaguch prio, ipl, flags);
2772 1.55 yamaguch
2773 1.55 yamaguch if (error)
2774 1.55 yamaguch return NULL;
2775 1.55 yamaguch
2776 1.55 yamaguch return wq;
2777 1.55 yamaguch }
2778 1.55 yamaguch
2779 1.55 yamaguch static void
2780 1.55 yamaguch vioif_workq_destroy(struct workqueue *wq)
2781 1.55 yamaguch {
2782 1.55 yamaguch
2783 1.55 yamaguch workqueue_destroy(wq);
2784 1.55 yamaguch }
2785 1.55 yamaguch
2786 1.55 yamaguch static void
2787 1.55 yamaguch vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
2788 1.55 yamaguch {
2789 1.55 yamaguch
2790 1.55 yamaguch memset(work, 0, sizeof(*work));
2791 1.55 yamaguch work->func = func;
2792 1.55 yamaguch work->arg = arg;
2793 1.55 yamaguch }
2794 1.55 yamaguch
2795 1.55 yamaguch static void
2796 1.55 yamaguch vioif_work_add(struct workqueue *wq, struct vioif_work *work)
2797 1.55 yamaguch {
2798 1.55 yamaguch
2799 1.55 yamaguch if (atomic_load_relaxed(&work->added) != 0)
2800 1.55 yamaguch return;
2801 1.55 yamaguch
2802 1.55 yamaguch atomic_store_relaxed(&work->added, 1);
2803 1.55 yamaguch kpreempt_disable();
2804 1.55 yamaguch workqueue_enqueue(wq, &work->cookie, NULL);
2805 1.55 yamaguch kpreempt_enable();
2806 1.55 yamaguch }
2807 1.55 yamaguch
2808 1.55 yamaguch static void
2809 1.55 yamaguch vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
2810 1.55 yamaguch {
2811 1.55 yamaguch
2812 1.55 yamaguch workqueue_wait(wq, &work->cookie);
2813 1.55 yamaguch }
2814 1.55 yamaguch
2815 1.26 pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
2816 1.48 msaitoh
2817 1.26 pgoyette #ifdef _MODULE
2818 1.26 pgoyette #include "ioconf.c"
2819 1.26 pgoyette #endif
2820 1.48 msaitoh
2821 1.48 msaitoh static int
2822 1.26 pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
2823 1.26 pgoyette {
2824 1.26 pgoyette int error = 0;
2825 1.48 msaitoh
2826 1.26 pgoyette #ifdef _MODULE
2827 1.26 pgoyette switch (cmd) {
2828 1.26 pgoyette case MODULE_CMD_INIT:
2829 1.48 msaitoh error = config_init_component(cfdriver_ioconf_if_vioif,
2830 1.48 msaitoh cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2831 1.26 pgoyette break;
2832 1.26 pgoyette case MODULE_CMD_FINI:
2833 1.26 pgoyette error = config_fini_component(cfdriver_ioconf_if_vioif,
2834 1.26 pgoyette cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2835 1.26 pgoyette break;
2836 1.26 pgoyette default:
2837 1.26 pgoyette error = ENOTTY;
2838 1.48 msaitoh break;
2839 1.26 pgoyette }
2840 1.26 pgoyette #endif
2841 1.48 msaitoh
2842 1.26 pgoyette return error;
2843 1.26 pgoyette }
2844