if_vioif.c revision 1.101 1 1.101 yamaguch /* $NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $ */
2 1.1 hannken
3 1.1 hannken /*
4 1.66 reinoud * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 1.1 hannken * Copyright (c) 2010 Minoura Makoto.
6 1.1 hannken * All rights reserved.
7 1.1 hannken *
8 1.1 hannken * Redistribution and use in source and binary forms, with or without
9 1.1 hannken * modification, are permitted provided that the following conditions
10 1.1 hannken * are met:
11 1.1 hannken * 1. Redistributions of source code must retain the above copyright
12 1.1 hannken * notice, this list of conditions and the following disclaimer.
13 1.1 hannken * 2. Redistributions in binary form must reproduce the above copyright
14 1.1 hannken * notice, this list of conditions and the following disclaimer in the
15 1.1 hannken * documentation and/or other materials provided with the distribution.
16 1.1 hannken *
17 1.1 hannken * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 1.1 hannken * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 1.1 hannken * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 1.1 hannken * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 1.1 hannken * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 1.1 hannken * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 1.1 hannken * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 1.1 hannken * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 1.1 hannken * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 1.1 hannken * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 1.1 hannken */
28 1.1 hannken
29 1.1 hannken #include <sys/cdefs.h>
30 1.101 yamaguch __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $");
31 1.15 ozaki
32 1.15 ozaki #ifdef _KERNEL_OPT
33 1.15 ozaki #include "opt_net_mpsafe.h"
34 1.15 ozaki #endif
35 1.1 hannken
36 1.1 hannken #include <sys/param.h>
37 1.1 hannken #include <sys/systm.h>
38 1.1 hannken #include <sys/kernel.h>
39 1.55 yamaguch #include <sys/atomic.h>
40 1.1 hannken #include <sys/bus.h>
41 1.1 hannken #include <sys/condvar.h>
42 1.1 hannken #include <sys/device.h>
43 1.63 yamaguch #include <sys/evcnt.h>
44 1.1 hannken #include <sys/intr.h>
45 1.1 hannken #include <sys/kmem.h>
46 1.1 hannken #include <sys/mbuf.h>
47 1.1 hannken #include <sys/mutex.h>
48 1.1 hannken #include <sys/sockio.h>
49 1.71 yamaguch #include <sys/syslog.h>
50 1.12 ozaki #include <sys/cpu.h>
51 1.26 pgoyette #include <sys/module.h>
52 1.46 yamaguch #include <sys/pcq.h>
53 1.55 yamaguch #include <sys/workqueue.h>
54 1.85 yamaguch #include <sys/xcall.h>
55 1.1 hannken
56 1.1 hannken #include <dev/pci/virtioreg.h>
57 1.1 hannken #include <dev/pci/virtiovar.h>
58 1.1 hannken
59 1.1 hannken #include <net/if.h>
60 1.74 yamaguch #include <net/if_dl.h>
61 1.1 hannken #include <net/if_media.h>
62 1.1 hannken #include <net/if_ether.h>
63 1.1 hannken
64 1.1 hannken #include <net/bpf.h>
65 1.1 hannken
66 1.26 pgoyette #include "ioconf.h"
67 1.1 hannken
68 1.7 ozaki #ifdef NET_MPSAFE
69 1.7 ozaki #define VIOIF_MPSAFE 1
70 1.46 yamaguch #define VIOIF_MULTIQ 1
71 1.7 ozaki #endif
72 1.7 ozaki
73 1.1 hannken /*
74 1.1 hannken * if_vioifreg.h:
75 1.1 hannken */
76 1.1 hannken /* Configuration registers */
77 1.66 reinoud #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
78 1.66 reinoud #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
79 1.66 reinoud #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
80 1.66 reinoud #define VIRTIO_NET_CONFIG_MTU 10 /* 16bit */
81 1.1 hannken
82 1.1 hannken /* Feature bits */
83 1.46 yamaguch #define VIRTIO_NET_F_CSUM __BIT(0)
84 1.46 yamaguch #define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
85 1.46 yamaguch #define VIRTIO_NET_F_MAC __BIT(5)
86 1.46 yamaguch #define VIRTIO_NET_F_GSO __BIT(6)
87 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
88 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
89 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ECN __BIT(9)
90 1.46 yamaguch #define VIRTIO_NET_F_GUEST_UFO __BIT(10)
91 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
92 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
93 1.46 yamaguch #define VIRTIO_NET_F_HOST_ECN __BIT(13)
94 1.46 yamaguch #define VIRTIO_NET_F_HOST_UFO __BIT(14)
95 1.46 yamaguch #define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
96 1.46 yamaguch #define VIRTIO_NET_F_STATUS __BIT(16)
97 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VQ __BIT(17)
98 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX __BIT(18)
99 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
100 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
101 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
102 1.46 yamaguch #define VIRTIO_NET_F_MQ __BIT(22)
103 1.78 yamaguch #define VIRTIO_NET_F_CTRL_MAC_ADDR __BIT(23)
104 1.1 hannken
105 1.79 uwe #define VIRTIO_NET_FLAG_BITS \
106 1.79 uwe VIRTIO_COMMON_FLAG_BITS \
107 1.79 uwe "b\x17" "CTRL_MAC\0" \
108 1.79 uwe "b\x16" "MQ\0" \
109 1.79 uwe "b\x15" "GUEST_ANNOUNCE\0" \
110 1.79 uwe "b\x14" "CTRL_RX_EXTRA\0" \
111 1.79 uwe "b\x13" "CTRL_VLAN\0" \
112 1.79 uwe "b\x12" "CTRL_RX\0" \
113 1.79 uwe "b\x11" "CTRL_VQ\0" \
114 1.79 uwe "b\x10" "STATUS\0" \
115 1.79 uwe "b\x0f" "MRG_RXBUF\0" \
116 1.79 uwe "b\x0e" "HOST_UFO\0" \
117 1.79 uwe "b\x0d" "HOST_ECN\0" \
118 1.79 uwe "b\x0c" "HOST_TSO6\0" \
119 1.79 uwe "b\x0b" "HOST_TSO4\0" \
120 1.79 uwe "b\x0a" "GUEST_UFO\0" \
121 1.79 uwe "b\x09" "GUEST_ECN\0" \
122 1.79 uwe "b\x08" "GUEST_TSO6\0" \
123 1.79 uwe "b\x07" "GUEST_TSO4\0" \
124 1.79 uwe "b\x06" "GSO\0" \
125 1.79 uwe "b\x05" "MAC\0" \
126 1.79 uwe "b\x01" "GUEST_CSUM\0" \
127 1.79 uwe "b\x00" "CSUM\0"
128 1.18 christos
129 1.1 hannken /* Status */
130 1.1 hannken #define VIRTIO_NET_S_LINK_UP 1
131 1.1 hannken
132 1.1 hannken /* Packet header structure */
133 1.1 hannken struct virtio_net_hdr {
134 1.1 hannken uint8_t flags;
135 1.1 hannken uint8_t gso_type;
136 1.1 hannken uint16_t hdr_len;
137 1.1 hannken uint16_t gso_size;
138 1.1 hannken uint16_t csum_start;
139 1.1 hannken uint16_t csum_offset;
140 1.66 reinoud
141 1.66 reinoud uint16_t num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */
142 1.1 hannken } __packed;
143 1.1 hannken
144 1.1 hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
145 1.1 hannken #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
146 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
147 1.1 hannken #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
148 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
149 1.1 hannken #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
150 1.1 hannken
151 1.1 hannken #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN)
152 1.1 hannken
153 1.1 hannken /* Control virtqueue */
154 1.1 hannken struct virtio_net_ctrl_cmd {
155 1.1 hannken uint8_t class;
156 1.1 hannken uint8_t command;
157 1.1 hannken } __packed;
158 1.1 hannken #define VIRTIO_NET_CTRL_RX 0
159 1.1 hannken # define VIRTIO_NET_CTRL_RX_PROMISC 0
160 1.1 hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI 1
161 1.1 hannken
162 1.1 hannken #define VIRTIO_NET_CTRL_MAC 1
163 1.1 hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
164 1.74 yamaguch # define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
165 1.1 hannken
166 1.1 hannken #define VIRTIO_NET_CTRL_VLAN 2
167 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_ADD 0
168 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_DEL 1
169 1.1 hannken
170 1.46 yamaguch #define VIRTIO_NET_CTRL_MQ 4
171 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
172 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
173 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
174 1.46 yamaguch
175 1.1 hannken struct virtio_net_ctrl_status {
176 1.1 hannken uint8_t ack;
177 1.1 hannken } __packed;
178 1.1 hannken #define VIRTIO_NET_OK 0
179 1.1 hannken #define VIRTIO_NET_ERR 1
180 1.1 hannken
181 1.1 hannken struct virtio_net_ctrl_rx {
182 1.1 hannken uint8_t onoff;
183 1.1 hannken } __packed;
184 1.1 hannken
185 1.1 hannken struct virtio_net_ctrl_mac_tbl {
186 1.1 hannken uint32_t nentries;
187 1.1 hannken uint8_t macs[][ETHER_ADDR_LEN];
188 1.1 hannken } __packed;
189 1.1 hannken
190 1.74 yamaguch struct virtio_net_ctrl_mac_addr {
191 1.74 yamaguch uint8_t mac[ETHER_ADDR_LEN];
192 1.74 yamaguch } __packed;
193 1.74 yamaguch
194 1.1 hannken struct virtio_net_ctrl_vlan {
195 1.1 hannken uint16_t id;
196 1.1 hannken } __packed;
197 1.1 hannken
198 1.46 yamaguch struct virtio_net_ctrl_mq {
199 1.46 yamaguch uint16_t virtqueue_pairs;
200 1.46 yamaguch } __packed;
201 1.46 yamaguch
202 1.1 hannken /*
203 1.1 hannken * if_vioifvar.h:
204 1.1 hannken */
205 1.43 yamaguch
206 1.43 yamaguch /*
207 1.43 yamaguch * Locking notes:
208 1.98 yamaguch * + a field in vioif_netueue is protected by netq_lock (a spin mutex)
209 1.43 yamaguch * - more than one lock cannot be held at onece
210 1.98 yamaguch * + a field in vioif_tx_context and vioif_rx_context is also protected
211 1.98 yamaguch * by netq_lock.
212 1.43 yamaguch * + ctrlq_inuse is protected by ctrlq_wait_lock.
213 1.43 yamaguch * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
214 1.98 yamaguch * - netq_lock cannot be held along with ctrlq_wait_lock
215 1.62 yamaguch * + fields in vioif_softc except queues are protected by
216 1.62 yamaguch * sc->sc_lock(an adaptive mutex)
217 1.62 yamaguch * - the lock is held before acquisition of other locks
218 1.43 yamaguch */
219 1.43 yamaguch
220 1.66 reinoud struct vioif_ctrl_cmdspec {
221 1.66 reinoud bus_dmamap_t dmamap;
222 1.66 reinoud void *buf;
223 1.66 reinoud bus_size_t bufsize;
224 1.66 reinoud };
225 1.66 reinoud
226 1.55 yamaguch struct vioif_work {
227 1.55 yamaguch struct work cookie;
228 1.55 yamaguch void (*func)(void *);
229 1.55 yamaguch void *arg;
230 1.55 yamaguch unsigned int added;
231 1.55 yamaguch };
232 1.55 yamaguch
233 1.95 yamaguch struct vioif_net_map {
234 1.95 yamaguch struct virtio_net_hdr *vnm_hdr;
235 1.95 yamaguch bus_dmamap_t vnm_hdr_map;
236 1.95 yamaguch struct mbuf *vnm_mbuf;
237 1.95 yamaguch bus_dmamap_t vnm_mbuf_map;
238 1.95 yamaguch };
239 1.95 yamaguch
240 1.98 yamaguch #define VIOIF_NETQ_RX 0
241 1.98 yamaguch #define VIOIF_NETQ_TX 1
242 1.98 yamaguch #define VIOIF_NETQ_IDX 2
243 1.98 yamaguch #define VIOIF_NETQ_DIR(n) ((n) % VIOIF_NETQ_IDX)
244 1.98 yamaguch #define VIOIF_NETQ_PAIRIDX(n) ((n) / VIOIF_NETQ_IDX)
245 1.98 yamaguch #define VIOIF_NETQ_RXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX)
246 1.98 yamaguch #define VIOIF_NETQ_TXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX)
247 1.98 yamaguch
248 1.98 yamaguch struct vioif_netqueue {
249 1.98 yamaguch kmutex_t netq_lock;
250 1.98 yamaguch struct virtqueue *netq_vq;
251 1.98 yamaguch bool netq_stopping;
252 1.98 yamaguch bool netq_running_handle;
253 1.98 yamaguch void *netq_maps_kva;
254 1.98 yamaguch struct vioif_net_map *netq_maps;
255 1.98 yamaguch
256 1.98 yamaguch void *netq_softint;
257 1.98 yamaguch struct vioif_work netq_work;
258 1.98 yamaguch bool netq_workqueue;
259 1.98 yamaguch
260 1.98 yamaguch char netq_evgroup[32];
261 1.98 yamaguch struct evcnt netq_mbuf_load_failed;
262 1.99 yamaguch struct evcnt netq_enqueue_failed;
263 1.43 yamaguch
264 1.98 yamaguch void *netq_ctx;
265 1.43 yamaguch };
266 1.43 yamaguch
267 1.98 yamaguch struct vioif_tx_context {
268 1.98 yamaguch bool txc_link_active;
269 1.98 yamaguch pcq_t *txc_intrq;
270 1.98 yamaguch void *txc_deferred_transmit;
271 1.43 yamaguch
272 1.98 yamaguch struct evcnt txc_defrag_failed;
273 1.98 yamaguch };
274 1.43 yamaguch
275 1.98 yamaguch struct vioif_rx_context {
276 1.98 yamaguch struct evcnt rxc_mbuf_enobufs;
277 1.43 yamaguch };
278 1.43 yamaguch struct vioif_ctrlqueue {
279 1.43 yamaguch struct virtqueue *ctrlq_vq;
280 1.43 yamaguch enum {
281 1.43 yamaguch FREE, INUSE, DONE
282 1.43 yamaguch } ctrlq_inuse;
283 1.43 yamaguch kcondvar_t ctrlq_wait;
284 1.43 yamaguch kmutex_t ctrlq_wait_lock;
285 1.44 yamaguch struct lwp *ctrlq_owner;
286 1.43 yamaguch
287 1.43 yamaguch struct virtio_net_ctrl_cmd *ctrlq_cmd;
288 1.43 yamaguch struct virtio_net_ctrl_status *ctrlq_status;
289 1.43 yamaguch struct virtio_net_ctrl_rx *ctrlq_rx;
290 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
291 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
292 1.74 yamaguch struct virtio_net_ctrl_mac_addr *ctrlq_mac_addr;
293 1.46 yamaguch struct virtio_net_ctrl_mq *ctrlq_mq;
294 1.43 yamaguch
295 1.43 yamaguch bus_dmamap_t ctrlq_cmd_dmamap;
296 1.43 yamaguch bus_dmamap_t ctrlq_status_dmamap;
297 1.43 yamaguch bus_dmamap_t ctrlq_rx_dmamap;
298 1.43 yamaguch bus_dmamap_t ctrlq_tbl_uc_dmamap;
299 1.43 yamaguch bus_dmamap_t ctrlq_tbl_mc_dmamap;
300 1.74 yamaguch bus_dmamap_t ctrlq_mac_addr_dmamap;
301 1.46 yamaguch bus_dmamap_t ctrlq_mq_dmamap;
302 1.63 yamaguch
303 1.63 yamaguch struct evcnt ctrlq_cmd_load_failed;
304 1.63 yamaguch struct evcnt ctrlq_cmd_failed;
305 1.43 yamaguch };
306 1.43 yamaguch
307 1.1 hannken struct vioif_softc {
308 1.1 hannken device_t sc_dev;
309 1.62 yamaguch kmutex_t sc_lock;
310 1.55 yamaguch struct sysctllog *sc_sysctllog;
311 1.1 hannken
312 1.1 hannken struct virtio_softc *sc_virtio;
313 1.46 yamaguch struct virtqueue *sc_vqs;
314 1.66 reinoud u_int sc_hdr_size;
315 1.46 yamaguch
316 1.46 yamaguch int sc_max_nvq_pairs;
317 1.46 yamaguch int sc_req_nvq_pairs;
318 1.46 yamaguch int sc_act_nvq_pairs;
319 1.1 hannken
320 1.1 hannken uint8_t sc_mac[ETHER_ADDR_LEN];
321 1.1 hannken struct ethercom sc_ethercom;
322 1.82 knakahar int sc_link_state;
323 1.1 hannken
324 1.98 yamaguch struct vioif_netqueue *sc_netqs;
325 1.43 yamaguch
326 1.43 yamaguch bool sc_has_ctrl;
327 1.43 yamaguch struct vioif_ctrlqueue sc_ctrlq;
328 1.43 yamaguch
329 1.100 yamaguch bus_dma_segment_t sc_segs[1];
330 1.43 yamaguch void *sc_dmamem;
331 1.43 yamaguch void *sc_kmem;
332 1.1 hannken
333 1.101 yamaguch void *sc_cfg_softint;
334 1.55 yamaguch
335 1.55 yamaguch struct workqueue *sc_txrx_workqueue;
336 1.55 yamaguch bool sc_txrx_workqueue_sysctl;
337 1.55 yamaguch u_int sc_tx_intr_process_limit;
338 1.55 yamaguch u_int sc_tx_process_limit;
339 1.55 yamaguch u_int sc_rx_intr_process_limit;
340 1.55 yamaguch u_int sc_rx_process_limit;
341 1.1 hannken };
342 1.1 hannken #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
343 1.1 hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
344 1.1 hannken
345 1.55 yamaguch #define VIOIF_TX_INTR_PROCESS_LIMIT 256
346 1.55 yamaguch #define VIOIF_TX_PROCESS_LIMIT 256
347 1.55 yamaguch #define VIOIF_RX_INTR_PROCESS_LIMIT 0U
348 1.55 yamaguch #define VIOIF_RX_PROCESS_LIMIT 256
349 1.55 yamaguch
350 1.55 yamaguch #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET
351 1.82 knakahar #define VIOIF_IS_LINK_ACTIVE(_sc) ((_sc)->sc_link_state == LINK_STATE_UP ? \
352 1.82 knakahar true : false)
353 1.55 yamaguch
354 1.1 hannken /* cfattach interface functions */
355 1.1 hannken static int vioif_match(device_t, cfdata_t, void *);
356 1.1 hannken static void vioif_attach(device_t, device_t, void *);
357 1.55 yamaguch static int vioif_finalize_teardown(device_t);
358 1.1 hannken
359 1.1 hannken /* ifnet interface functions */
360 1.1 hannken static int vioif_init(struct ifnet *);
361 1.1 hannken static void vioif_stop(struct ifnet *, int);
362 1.1 hannken static void vioif_start(struct ifnet *);
363 1.46 yamaguch static int vioif_transmit(struct ifnet *, struct mbuf *);
364 1.1 hannken static int vioif_ioctl(struct ifnet *, u_long, void *);
365 1.1 hannken static void vioif_watchdog(struct ifnet *);
366 1.101 yamaguch static int vioif_ifflags(struct vioif_softc *);
367 1.75 yamaguch static int vioif_ifflags_cb(struct ethercom *);
368 1.1 hannken
369 1.98 yamaguch /* tx & rx */
370 1.101 yamaguch static int vioif_netqueue_init(struct vioif_softc *,
371 1.101 yamaguch struct virtio_softc *, size_t, u_int);
372 1.101 yamaguch static void vioif_netqueue_teardown(struct vioif_softc *,
373 1.101 yamaguch struct virtio_softc *, size_t);
374 1.99 yamaguch static void vioif_net_intr_enable(struct vioif_softc *,
375 1.99 yamaguch struct virtio_softc *);
376 1.99 yamaguch static void vioif_net_intr_disable(struct vioif_softc *,
377 1.99 yamaguch struct virtio_softc *);
378 1.101 yamaguch static void vioif_net_sched_handle(struct vioif_softc *,
379 1.101 yamaguch struct vioif_netqueue *);
380 1.98 yamaguch
381 1.1 hannken /* rx */
382 1.66 reinoud static void vioif_populate_rx_mbufs_locked(struct vioif_softc *,
383 1.98 yamaguch struct vioif_netqueue *);
384 1.54 yamaguch static int vioif_rx_intr(void *);
385 1.55 yamaguch static void vioif_rx_handle(void *);
386 1.101 yamaguch static void vioif_rx_queue_clear(struct vioif_softc *,
387 1.101 yamaguch struct virtio_softc *, struct vioif_netqueue *);
388 1.1 hannken
389 1.1 hannken /* tx */
390 1.101 yamaguch static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *);
391 1.101 yamaguch static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *);
392 1.101 yamaguch static void vioif_deferred_transmit(void *);
393 1.54 yamaguch static int vioif_tx_intr(void *);
394 1.55 yamaguch static void vioif_tx_handle(void *);
395 1.93 yamaguch static void vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *,
396 1.98 yamaguch struct vioif_netqueue *);
397 1.1 hannken
398 1.101 yamaguch /* controls */
399 1.101 yamaguch static int vioif_ctrl_intr(void *);
400 1.1 hannken static int vioif_ctrl_rx(struct vioif_softc *, int, bool);
401 1.1 hannken static int vioif_set_promisc(struct vioif_softc *, bool);
402 1.1 hannken static int vioif_set_allmulti(struct vioif_softc *, bool);
403 1.1 hannken static int vioif_set_rx_filter(struct vioif_softc *);
404 1.1 hannken static int vioif_rx_filter(struct vioif_softc *);
405 1.74 yamaguch static int vioif_set_mac_addr(struct vioif_softc *);
406 1.101 yamaguch static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
407 1.101 yamaguch
408 1.101 yamaguch /* config interrupt */
409 1.34 ozaki static int vioif_config_change(struct virtio_softc *);
410 1.101 yamaguch static void vioif_cfg_softint(void *);
411 1.101 yamaguch static void vioif_update_link_status(struct vioif_softc *);
412 1.101 yamaguch
413 1.101 yamaguch /* others */
414 1.101 yamaguch static void vioif_alloc_queues(struct vioif_softc *);
415 1.101 yamaguch static void vioif_free_queues(struct vioif_softc *);
416 1.101 yamaguch static int vioif_alloc_mems(struct vioif_softc *);
417 1.101 yamaguch static struct workqueue*
418 1.101 yamaguch vioif_workq_create(const char *, pri_t, int, int);
419 1.101 yamaguch static void vioif_workq_destroy(struct workqueue *);
420 1.101 yamaguch static void vioif_work_set(struct vioif_work *, void(*)(void *), void *);
421 1.101 yamaguch static void vioif_work_add(struct workqueue *, struct vioif_work *);
422 1.101 yamaguch static void vioif_work_wait(struct workqueue *, struct vioif_work *);
423 1.55 yamaguch static int vioif_setup_sysctl(struct vioif_softc *);
424 1.63 yamaguch static void vioif_setup_stats(struct vioif_softc *);
425 1.1 hannken
426 1.1 hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
427 1.1 hannken vioif_match, vioif_attach, NULL, NULL);
428 1.1 hannken
429 1.101 yamaguch static void
430 1.101 yamaguch vioif_intr_barrier(void)
431 1.101 yamaguch {
432 1.101 yamaguch
433 1.101 yamaguch /* wait for finish all interrupt handler */
434 1.101 yamaguch xc_barrier(0);
435 1.101 yamaguch }
436 1.101 yamaguch
437 1.101 yamaguch static void
438 1.101 yamaguch vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq)
439 1.101 yamaguch {
440 1.101 yamaguch
441 1.101 yamaguch virtio_enqueue_commit(vsc, vq, -1, true);
442 1.101 yamaguch }
443 1.101 yamaguch
444 1.1 hannken static int
445 1.1 hannken vioif_match(device_t parent, cfdata_t match, void *aux)
446 1.1 hannken {
447 1.32 jdolecek struct virtio_attach_args *va = aux;
448 1.1 hannken
449 1.66 reinoud if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK)
450 1.1 hannken return 1;
451 1.1 hannken
452 1.1 hannken return 0;
453 1.1 hannken }
454 1.1 hannken
455 1.101 yamaguch static void
456 1.101 yamaguch vioif_attach(device_t parent, device_t self, void *aux)
457 1.58 yamaguch {
458 1.101 yamaguch struct vioif_softc *sc = device_private(self);
459 1.101 yamaguch struct virtio_softc *vsc = device_private(parent);
460 1.101 yamaguch struct vioif_netqueue *txq0;
461 1.101 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
462 1.101 yamaguch uint64_t features, req_features;
463 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
464 1.101 yamaguch u_int softint_flags;
465 1.101 yamaguch int r, i, req_flags;
466 1.101 yamaguch char xnamebuf[MAXCOMLEN];
467 1.101 yamaguch size_t netq_num;
468 1.101 yamaguch
469 1.101 yamaguch if (virtio_child(vsc) != NULL) {
470 1.101 yamaguch aprint_normal(": child already attached for %s; "
471 1.101 yamaguch "something wrong...\n", device_xname(parent));
472 1.101 yamaguch return;
473 1.101 yamaguch }
474 1.101 yamaguch
475 1.101 yamaguch sc->sc_dev = self;
476 1.101 yamaguch sc->sc_virtio = vsc;
477 1.101 yamaguch sc->sc_link_state = LINK_STATE_UNKNOWN;
478 1.101 yamaguch
479 1.101 yamaguch sc->sc_max_nvq_pairs = 1;
480 1.101 yamaguch sc->sc_req_nvq_pairs = 1;
481 1.101 yamaguch sc->sc_act_nvq_pairs = 1;
482 1.101 yamaguch sc->sc_txrx_workqueue_sysctl = true;
483 1.101 yamaguch sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
484 1.101 yamaguch sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
485 1.101 yamaguch sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
486 1.101 yamaguch sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
487 1.101 yamaguch
488 1.101 yamaguch mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE);
489 1.101 yamaguch
490 1.101 yamaguch snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
491 1.101 yamaguch sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
492 1.101 yamaguch IPL_NET, WQ_PERCPU | WQ_MPSAFE);
493 1.101 yamaguch if (sc->sc_txrx_workqueue == NULL)
494 1.101 yamaguch goto err;
495 1.58 yamaguch
496 1.101 yamaguch req_flags = 0;
497 1.58 yamaguch
498 1.101 yamaguch #ifdef VIOIF_MPSAFE
499 1.101 yamaguch req_flags |= VIRTIO_F_INTR_MPSAFE;
500 1.101 yamaguch #endif
501 1.101 yamaguch req_flags |= VIRTIO_F_INTR_MSIX;
502 1.58 yamaguch
503 1.101 yamaguch req_features =
504 1.101 yamaguch VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
505 1.101 yamaguch VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
506 1.101 yamaguch req_features |= VIRTIO_F_RING_EVENT_IDX;
507 1.101 yamaguch req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR;
508 1.101 yamaguch #ifdef VIOIF_MULTIQ
509 1.101 yamaguch req_features |= VIRTIO_NET_F_MQ;
510 1.101 yamaguch #endif
511 1.101 yamaguch virtio_child_attach_start(vsc, self, IPL_NET, NULL,
512 1.101 yamaguch vioif_config_change, virtio_vq_intrhand, req_flags,
513 1.101 yamaguch req_features, VIRTIO_NET_FLAG_BITS);
514 1.58 yamaguch
515 1.101 yamaguch features = virtio_features(vsc);
516 1.101 yamaguch if (features == 0)
517 1.101 yamaguch goto err;
518 1.58 yamaguch
519 1.101 yamaguch if (features & VIRTIO_NET_F_MAC) {
520 1.101 yamaguch for (i = 0; i < __arraycount(sc->sc_mac); i++) {
521 1.101 yamaguch sc->sc_mac[i] = virtio_read_device_config_1(vsc,
522 1.101 yamaguch VIRTIO_NET_CONFIG_MAC + i);
523 1.101 yamaguch }
524 1.101 yamaguch } else {
525 1.101 yamaguch /* code stolen from sys/net/if_tap.c */
526 1.101 yamaguch struct timeval tv;
527 1.101 yamaguch uint32_t ui;
528 1.101 yamaguch getmicrouptime(&tv);
529 1.101 yamaguch ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
530 1.101 yamaguch memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
531 1.101 yamaguch for (i = 0; i < __arraycount(sc->sc_mac); i++) {
532 1.101 yamaguch virtio_write_device_config_1(vsc,
533 1.101 yamaguch VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
534 1.101 yamaguch }
535 1.58 yamaguch }
536 1.58 yamaguch
537 1.101 yamaguch /* 'Ethernet' with capital follows other ethernet driver attachment */
538 1.101 yamaguch aprint_normal_dev(self, "Ethernet address %s\n",
539 1.101 yamaguch ether_sprintf(sc->sc_mac));
540 1.101 yamaguch
541 1.101 yamaguch if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) {
542 1.101 yamaguch sc->sc_hdr_size = sizeof(struct virtio_net_hdr);
543 1.101 yamaguch } else {
544 1.101 yamaguch sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers);
545 1.101 yamaguch }
546 1.58 yamaguch
547 1.101 yamaguch if ((features & VIRTIO_NET_F_CTRL_VQ) &&
548 1.101 yamaguch (features & VIRTIO_NET_F_CTRL_RX)) {
549 1.101 yamaguch sc->sc_has_ctrl = true;
550 1.58 yamaguch
551 1.101 yamaguch cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
552 1.101 yamaguch mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
553 1.101 yamaguch ctrlq->ctrlq_inuse = FREE;
554 1.101 yamaguch } else {
555 1.101 yamaguch sc->sc_has_ctrl = false;
556 1.58 yamaguch }
557 1.58 yamaguch
558 1.101 yamaguch if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
559 1.101 yamaguch sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
560 1.101 yamaguch VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
561 1.101 yamaguch
562 1.101 yamaguch if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
563 1.101 yamaguch goto err;
564 1.58 yamaguch
565 1.101 yamaguch /* Limit the number of queue pairs to use */
566 1.101 yamaguch sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
567 1.101 yamaguch }
568 1.58 yamaguch
569 1.101 yamaguch vioif_alloc_queues(sc);
570 1.101 yamaguch virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
571 1.58 yamaguch
572 1.101 yamaguch #ifdef VIOIF_MPSAFE
573 1.101 yamaguch softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
574 1.101 yamaguch #else
575 1.101 yamaguch softint_flags = SOFTINT_NET;
576 1.101 yamaguch #endif
577 1.58 yamaguch
578 1.101 yamaguch /*
579 1.101 yamaguch * Initialize network queues
580 1.101 yamaguch */
581 1.101 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
582 1.101 yamaguch for (i = 0; i < netq_num; i++) {
583 1.101 yamaguch r = vioif_netqueue_init(sc, vsc, i, softint_flags);
584 1.101 yamaguch if (r != 0)
585 1.101 yamaguch goto err;
586 1.101 yamaguch }
587 1.46 yamaguch
588 1.101 yamaguch if (sc->sc_has_ctrl) {
589 1.101 yamaguch int ctrlq_idx = sc->sc_max_nvq_pairs * 2;
590 1.101 yamaguch /*
591 1.101 yamaguch * Allocating a virtqueue for control channel
592 1.101 yamaguch */
593 1.101 yamaguch sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx];
594 1.101 yamaguch r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx,
595 1.101 yamaguch NBPG, 1, "control");
596 1.101 yamaguch if (r != 0) {
597 1.101 yamaguch aprint_error_dev(self, "failed to allocate "
598 1.101 yamaguch "a virtqueue for control channel, error code %d\n",
599 1.101 yamaguch r);
600 1.46 yamaguch
601 1.101 yamaguch sc->sc_has_ctrl = false;
602 1.101 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
603 1.101 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
604 1.101 yamaguch } else {
605 1.101 yamaguch ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
606 1.101 yamaguch ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
607 1.101 yamaguch }
608 1.101 yamaguch }
609 1.46 yamaguch
610 1.101 yamaguch sc->sc_cfg_softint = softint_establish(softint_flags,
611 1.101 yamaguch vioif_cfg_softint, sc);
612 1.101 yamaguch if (sc->sc_cfg_softint == NULL) {
613 1.101 yamaguch aprint_error_dev(self, "cannot establish ctl softint\n");
614 1.101 yamaguch goto err;
615 1.101 yamaguch }
616 1.46 yamaguch
617 1.101 yamaguch if (vioif_alloc_mems(sc) < 0)
618 1.101 yamaguch goto err;
619 1.46 yamaguch
620 1.101 yamaguch if (virtio_child_attach_finish(vsc) != 0)
621 1.101 yamaguch goto err;
622 1.46 yamaguch
623 1.101 yamaguch if (vioif_setup_sysctl(sc) != 0) {
624 1.101 yamaguch aprint_error_dev(self, "unable to create sysctl node\n");
625 1.101 yamaguch /* continue */
626 1.101 yamaguch }
627 1.98 yamaguch
628 1.101 yamaguch vioif_setup_stats(sc);
629 1.98 yamaguch
630 1.101 yamaguch strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
631 1.101 yamaguch ifp->if_softc = sc;
632 1.101 yamaguch ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
633 1.101 yamaguch #ifdef VIOIF_MPSAFE
634 1.101 yamaguch ifp->if_extflags = IFEF_MPSAFE;
635 1.101 yamaguch #endif
636 1.101 yamaguch ifp->if_start = vioif_start;
637 1.101 yamaguch if (sc->sc_req_nvq_pairs > 1)
638 1.101 yamaguch ifp->if_transmit = vioif_transmit;
639 1.101 yamaguch ifp->if_ioctl = vioif_ioctl;
640 1.101 yamaguch ifp->if_init = vioif_init;
641 1.101 yamaguch ifp->if_stop = vioif_stop;
642 1.101 yamaguch ifp->if_capabilities = 0;
643 1.101 yamaguch ifp->if_watchdog = vioif_watchdog;
644 1.101 yamaguch txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
645 1.101 yamaguch IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN));
646 1.101 yamaguch IFQ_SET_READY(&ifp->if_snd);
647 1.98 yamaguch
648 1.101 yamaguch sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
649 1.98 yamaguch
650 1.101 yamaguch if_attach(ifp);
651 1.101 yamaguch if_deferred_start_init(ifp, NULL);
652 1.101 yamaguch ether_ifattach(ifp, sc->sc_mac);
653 1.101 yamaguch ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb);
654 1.98 yamaguch
655 1.101 yamaguch return;
656 1.98 yamaguch
657 1.98 yamaguch err:
658 1.101 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
659 1.101 yamaguch for (i = 0; i < netq_num; i++) {
660 1.101 yamaguch vioif_netqueue_teardown(sc, vsc, i);
661 1.98 yamaguch }
662 1.98 yamaguch
663 1.101 yamaguch if (sc->sc_has_ctrl) {
664 1.101 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
665 1.101 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
666 1.101 yamaguch virtio_free_vq(vsc, ctrlq->ctrlq_vq);
667 1.101 yamaguch ctrlq->ctrlq_vq = NULL;
668 1.98 yamaguch }
669 1.98 yamaguch
670 1.101 yamaguch vioif_free_queues(sc);
671 1.101 yamaguch mutex_destroy(&sc->sc_lock);
672 1.101 yamaguch virtio_child_attach_failed(vsc);
673 1.101 yamaguch config_finalize_register(self, vioif_finalize_teardown);
674 1.98 yamaguch
675 1.101 yamaguch return;
676 1.98 yamaguch }
677 1.98 yamaguch
678 1.101 yamaguch static int
679 1.101 yamaguch vioif_finalize_teardown(device_t self)
680 1.98 yamaguch {
681 1.101 yamaguch struct vioif_softc *sc = device_private(self);
682 1.98 yamaguch
683 1.101 yamaguch if (sc->sc_txrx_workqueue != NULL) {
684 1.101 yamaguch vioif_workq_destroy(sc->sc_txrx_workqueue);
685 1.101 yamaguch sc->sc_txrx_workqueue = NULL;
686 1.46 yamaguch }
687 1.98 yamaguch
688 1.101 yamaguch return 0;
689 1.46 yamaguch }
690 1.46 yamaguch
691 1.1 hannken /*
692 1.101 yamaguch * Interface functions for ifnet
693 1.1 hannken */
694 1.1 hannken static int
695 1.101 yamaguch vioif_init(struct ifnet *ifp)
696 1.1 hannken {
697 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
698 1.1 hannken struct virtio_softc *vsc = sc->sc_virtio;
699 1.98 yamaguch struct vioif_netqueue *netq;
700 1.98 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
701 1.101 yamaguch int r, i;
702 1.101 yamaguch
703 1.101 yamaguch vioif_stop(ifp, 0);
704 1.101 yamaguch
705 1.101 yamaguch r = virtio_reinit_start(vsc);
706 1.101 yamaguch if (r != 0) {
707 1.101 yamaguch log(LOG_ERR, "%s: reset failed\n", ifp->if_xname);
708 1.101 yamaguch return EIO;
709 1.101 yamaguch }
710 1.1 hannken
711 1.101 yamaguch virtio_negotiate_features(vsc, virtio_features(vsc));
712 1.98 yamaguch
713 1.101 yamaguch for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
714 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
715 1.46 yamaguch
716 1.101 yamaguch mutex_enter(&netq->netq_lock);
717 1.101 yamaguch vioif_populate_rx_mbufs_locked(sc, netq);
718 1.101 yamaguch mutex_exit(&netq->netq_lock);
719 1.46 yamaguch }
720 1.98 yamaguch
721 1.101 yamaguch virtio_reinit_end(vsc);
722 1.101 yamaguch
723 1.101 yamaguch if (sc->sc_has_ctrl)
724 1.101 yamaguch virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
725 1.98 yamaguch
726 1.101 yamaguch r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
727 1.101 yamaguch if (r == 0)
728 1.101 yamaguch sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
729 1.101 yamaguch else
730 1.101 yamaguch sc->sc_act_nvq_pairs = 1;
731 1.101 yamaguch
732 1.101 yamaguch SET(ifp->if_flags, IFF_RUNNING);
733 1.101 yamaguch CLR(ifp->if_flags, IFF_OACTIVE);
734 1.42 yamaguch
735 1.101 yamaguch vioif_net_intr_enable(sc, vsc);
736 1.42 yamaguch
737 1.101 yamaguch vioif_update_link_status(sc);
738 1.101 yamaguch r = vioif_rx_filter(sc);
739 1.98 yamaguch
740 1.101 yamaguch return r;
741 1.101 yamaguch }
742 1.98 yamaguch
743 1.101 yamaguch static void
744 1.101 yamaguch vioif_stop(struct ifnet *ifp, int disable)
745 1.101 yamaguch {
746 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
747 1.101 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
748 1.101 yamaguch struct vioif_netqueue *netq;
749 1.101 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
750 1.101 yamaguch size_t i, act_qnum;
751 1.1 hannken
752 1.101 yamaguch act_qnum = sc->sc_act_nvq_pairs * 2;
753 1.46 yamaguch
754 1.101 yamaguch CLR(ifp->if_flags, IFF_RUNNING);
755 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
756 1.101 yamaguch netq = &sc->sc_netqs[i];
757 1.95 yamaguch
758 1.101 yamaguch mutex_enter(&netq->netq_lock);
759 1.101 yamaguch netq->netq_stopping = true;
760 1.101 yamaguch mutex_exit(&netq->netq_lock);
761 1.98 yamaguch }
762 1.46 yamaguch
763 1.101 yamaguch /* disable interrupts */
764 1.101 yamaguch vioif_net_intr_disable(sc, vsc);
765 1.101 yamaguch if (sc->sc_has_ctrl)
766 1.101 yamaguch virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
767 1.101 yamaguch
768 1.101 yamaguch /*
769 1.101 yamaguch * only way to stop interrupt, I/O and DMA is resetting...
770 1.101 yamaguch *
771 1.101 yamaguch * NOTE: Devices based on VirtIO draft specification can not
772 1.101 yamaguch * stop interrupt completely even if virtio_stop_vq_intr() is called.
773 1.101 yamaguch */
774 1.101 yamaguch virtio_reset(vsc);
775 1.101 yamaguch
776 1.101 yamaguch vioif_intr_barrier();
777 1.98 yamaguch
778 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
779 1.101 yamaguch netq = &sc->sc_netqs[i];
780 1.101 yamaguch vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work);
781 1.101 yamaguch }
782 1.42 yamaguch
783 1.101 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
784 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
785 1.101 yamaguch vioif_rx_queue_clear(sc, vsc, netq);
786 1.98 yamaguch
787 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
788 1.101 yamaguch vioif_tx_queue_clear(sc, vsc, netq);
789 1.98 yamaguch }
790 1.98 yamaguch
791 1.101 yamaguch /* all packet processing is stopped */
792 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
793 1.101 yamaguch netq = &sc->sc_netqs[i];
794 1.1 hannken
795 1.101 yamaguch mutex_enter(&netq->netq_lock);
796 1.101 yamaguch netq->netq_stopping = false;
797 1.101 yamaguch mutex_exit(&netq->netq_lock);
798 1.101 yamaguch }
799 1.101 yamaguch }
800 1.95 yamaguch
801 1.101 yamaguch static void
802 1.101 yamaguch vioif_start(struct ifnet *ifp)
803 1.101 yamaguch {
804 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
805 1.101 yamaguch struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
806 1.58 yamaguch
807 1.101 yamaguch #ifdef VIOIF_MPSAFE
808 1.101 yamaguch KASSERT(if_is_mpsafe(ifp));
809 1.101 yamaguch #endif
810 1.58 yamaguch
811 1.101 yamaguch mutex_enter(&txq0->netq_lock);
812 1.101 yamaguch vioif_start_locked(ifp, txq0);
813 1.101 yamaguch mutex_exit(&txq0->netq_lock);
814 1.101 yamaguch }
815 1.1 hannken
816 1.101 yamaguch static inline int
817 1.101 yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
818 1.101 yamaguch {
819 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
820 1.101 yamaguch u_int cpuid = cpu_index(curcpu());
821 1.58 yamaguch
822 1.101 yamaguch return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs);
823 1.101 yamaguch }
824 1.1 hannken
825 1.101 yamaguch static int
826 1.101 yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
827 1.101 yamaguch {
828 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
829 1.101 yamaguch struct vioif_netqueue *netq;
830 1.101 yamaguch struct vioif_tx_context *txc;
831 1.101 yamaguch int qid;
832 1.1 hannken
833 1.101 yamaguch qid = vioif_select_txqueue(ifp, m);
834 1.101 yamaguch netq = &sc->sc_netqs[qid];
835 1.101 yamaguch txc = netq->netq_ctx;
836 1.46 yamaguch
837 1.101 yamaguch if (__predict_false(!pcq_put(txc->txc_intrq, m))) {
838 1.101 yamaguch m_freem(m);
839 1.101 yamaguch return ENOBUFS;
840 1.101 yamaguch }
841 1.1 hannken
842 1.101 yamaguch net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
843 1.101 yamaguch if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
844 1.101 yamaguch if (m->m_flags & M_MCAST)
845 1.101 yamaguch if_statinc_ref(nsr, if_omcasts);
846 1.101 yamaguch IF_STAT_PUTREF(ifp);
847 1.74 yamaguch
848 1.101 yamaguch if (mutex_tryenter(&netq->netq_lock)) {
849 1.101 yamaguch vioif_transmit_locked(ifp, netq);
850 1.101 yamaguch mutex_exit(&netq->netq_lock);
851 1.1 hannken }
852 1.1 hannken
853 1.1 hannken return 0;
854 1.101 yamaguch }
855 1.1 hannken
856 1.101 yamaguch void
857 1.101 yamaguch vioif_watchdog(struct ifnet *ifp)
858 1.101 yamaguch {
859 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
860 1.101 yamaguch struct vioif_netqueue *netq;
861 1.101 yamaguch int i;
862 1.101 yamaguch
863 1.101 yamaguch if (ifp->if_flags & IFF_RUNNING) {
864 1.101 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
865 1.101 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
866 1.46 yamaguch
867 1.101 yamaguch mutex_enter(&netq->netq_lock);
868 1.101 yamaguch if (!netq->netq_running_handle) {
869 1.101 yamaguch netq->netq_running_handle = true;
870 1.101 yamaguch vioif_net_sched_handle(sc, netq);
871 1.101 yamaguch }
872 1.101 yamaguch mutex_exit(&netq->netq_lock);
873 1.46 yamaguch }
874 1.1 hannken }
875 1.101 yamaguch }
876 1.101 yamaguch
877 1.101 yamaguch static int
878 1.101 yamaguch vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
879 1.101 yamaguch {
880 1.101 yamaguch int s, r;
881 1.101 yamaguch
882 1.101 yamaguch s = splnet();
883 1.101 yamaguch
884 1.101 yamaguch r = ether_ioctl(ifp, cmd, data);
885 1.101 yamaguch if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) {
886 1.101 yamaguch if (ifp->if_flags & IFF_RUNNING) {
887 1.101 yamaguch r = vioif_rx_filter(ifp->if_softc);
888 1.101 yamaguch } else {
889 1.101 yamaguch r = 0;
890 1.101 yamaguch }
891 1.1 hannken }
892 1.101 yamaguch
893 1.101 yamaguch splx(s);
894 1.101 yamaguch
895 1.101 yamaguch return r;
896 1.1 hannken }
897 1.1 hannken
898 1.101 yamaguch static int
899 1.101 yamaguch vioif_ifflags(struct vioif_softc *sc)
900 1.1 hannken {
901 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
902 1.101 yamaguch bool onoff;
903 1.101 yamaguch int r;
904 1.1 hannken
905 1.101 yamaguch if (!sc->sc_has_ctrl) {
906 1.101 yamaguch /* no ctrl vq; always promisc and allmulti */
907 1.101 yamaguch ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI);
908 1.101 yamaguch return 0;
909 1.1 hannken }
910 1.1 hannken
911 1.101 yamaguch onoff = ifp->if_flags & IFF_ALLMULTI ? true : false;
912 1.101 yamaguch r = vioif_set_allmulti(sc, onoff);
913 1.101 yamaguch if (r != 0) {
914 1.101 yamaguch log(LOG_WARNING,
915 1.101 yamaguch "%s: couldn't %sable ALLMULTI\n",
916 1.101 yamaguch ifp->if_xname, onoff ? "en" : "dis");
917 1.101 yamaguch if (onoff == false) {
918 1.101 yamaguch ifp->if_flags |= IFF_ALLMULTI;
919 1.101 yamaguch }
920 1.101 yamaguch }
921 1.1 hannken
922 1.101 yamaguch onoff = ifp->if_flags & IFF_PROMISC ? true : false;
923 1.101 yamaguch r = vioif_set_promisc(sc, onoff);
924 1.101 yamaguch if (r != 0) {
925 1.101 yamaguch log(LOG_WARNING,
926 1.101 yamaguch "%s: couldn't %sable PROMISC\n",
927 1.101 yamaguch ifp->if_xname, onoff ? "en" : "dis");
928 1.101 yamaguch if (onoff == false) {
929 1.101 yamaguch ifp->if_flags |= IFF_PROMISC;
930 1.101 yamaguch }
931 1.101 yamaguch }
932 1.55 yamaguch
933 1.101 yamaguch return 0;
934 1.101 yamaguch }
935 1.62 yamaguch
936 1.101 yamaguch static int
937 1.101 yamaguch vioif_ifflags_cb(struct ethercom *ec)
938 1.101 yamaguch {
939 1.101 yamaguch struct ifnet *ifp = &ec->ec_if;
940 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
941 1.46 yamaguch
942 1.101 yamaguch return vioif_ifflags(sc);
943 1.101 yamaguch }
944 1.1 hannken
945 1.101 yamaguch static int
946 1.101 yamaguch vioif_setup_sysctl(struct vioif_softc *sc)
947 1.101 yamaguch {
948 1.101 yamaguch const char *devname;
949 1.101 yamaguch struct sysctllog **log;
950 1.101 yamaguch const struct sysctlnode *rnode, *rxnode, *txnode;
951 1.101 yamaguch int error;
952 1.32 jdolecek
953 1.101 yamaguch log = &sc->sc_sysctllog;
954 1.101 yamaguch devname = device_xname(sc->sc_dev);
955 1.32 jdolecek
956 1.101 yamaguch error = sysctl_createv(log, 0, NULL, &rnode,
957 1.101 yamaguch 0, CTLTYPE_NODE, devname,
958 1.101 yamaguch SYSCTL_DESCR("virtio-net information and settings"),
959 1.101 yamaguch NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
960 1.101 yamaguch if (error)
961 1.101 yamaguch goto out;
962 1.7 ozaki
963 1.101 yamaguch error = sysctl_createv(log, 0, &rnode, NULL,
964 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
965 1.101 yamaguch SYSCTL_DESCR("Use workqueue for packet processing"),
966 1.101 yamaguch NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
967 1.101 yamaguch if (error)
968 1.101 yamaguch goto out;
969 1.32 jdolecek
970 1.101 yamaguch error = sysctl_createv(log, 0, &rnode, &rxnode,
971 1.101 yamaguch 0, CTLTYPE_NODE, "rx",
972 1.101 yamaguch SYSCTL_DESCR("virtio-net information and settings for Rx"),
973 1.101 yamaguch NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
974 1.101 yamaguch if (error)
975 1.101 yamaguch goto out;
976 1.1 hannken
977 1.101 yamaguch error = sysctl_createv(log, 0, &rxnode, NULL,
978 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
979 1.101 yamaguch SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
980 1.101 yamaguch NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
981 1.101 yamaguch if (error)
982 1.101 yamaguch goto out;
983 1.66 reinoud
984 1.101 yamaguch error = sysctl_createv(log, 0, &rxnode, NULL,
985 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
986 1.101 yamaguch SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
987 1.101 yamaguch NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
988 1.101 yamaguch if (error)
989 1.101 yamaguch goto out;
990 1.46 yamaguch
991 1.101 yamaguch error = sysctl_createv(log, 0, &rnode, &txnode,
992 1.101 yamaguch 0, CTLTYPE_NODE, "tx",
993 1.101 yamaguch SYSCTL_DESCR("virtio-net information and settings for Tx"),
994 1.101 yamaguch NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
995 1.101 yamaguch if (error)
996 1.101 yamaguch goto out;
997 1.46 yamaguch
998 1.101 yamaguch error = sysctl_createv(log, 0, &txnode, NULL,
999 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
1000 1.101 yamaguch SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
1001 1.101 yamaguch NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
1002 1.101 yamaguch if (error)
1003 1.101 yamaguch goto out;
1004 1.46 yamaguch
1005 1.101 yamaguch error = sysctl_createv(log, 0, &txnode, NULL,
1006 1.101 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
1007 1.101 yamaguch SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
1008 1.101 yamaguch NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
1009 1.46 yamaguch
1010 1.101 yamaguch out:
1011 1.101 yamaguch if (error)
1012 1.101 yamaguch sysctl_teardown(log);
1013 1.46 yamaguch
1014 1.101 yamaguch return error;
1015 1.101 yamaguch }
1016 1.46 yamaguch
1017 1.101 yamaguch static void
1018 1.101 yamaguch vioif_setup_stats(struct vioif_softc *sc)
1019 1.101 yamaguch {
1020 1.101 yamaguch struct vioif_netqueue *netq;
1021 1.101 yamaguch struct vioif_tx_context *txc;
1022 1.101 yamaguch struct vioif_rx_context *rxc;
1023 1.101 yamaguch size_t i, netq_num;
1024 1.7 ozaki
1025 1.98 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
1026 1.98 yamaguch for (i = 0; i < netq_num; i++) {
1027 1.101 yamaguch netq = &sc->sc_netqs[i];
1028 1.101 yamaguch evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC,
1029 1.101 yamaguch NULL, netq->netq_evgroup, "failed to load mbuf to DMA");
1030 1.101 yamaguch evcnt_attach_dynamic(&netq->netq_enqueue_failed,
1031 1.101 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
1032 1.101 yamaguch "virtqueue enqueue failed failed");
1033 1.17 ozaki
1034 1.101 yamaguch switch (VIOIF_NETQ_DIR(i)) {
1035 1.101 yamaguch case VIOIF_NETQ_RX:
1036 1.101 yamaguch rxc = netq->netq_ctx;
1037 1.101 yamaguch evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs,
1038 1.101 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
1039 1.101 yamaguch "no receive buffer");
1040 1.101 yamaguch break;
1041 1.101 yamaguch case VIOIF_NETQ_TX:
1042 1.101 yamaguch txc = netq->netq_ctx;
1043 1.101 yamaguch evcnt_attach_dynamic(&txc->txc_defrag_failed,
1044 1.101 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
1045 1.101 yamaguch "m_defrag() failed");
1046 1.101 yamaguch break;
1047 1.1 hannken }
1048 1.1 hannken }
1049 1.34 ozaki
1050 1.101 yamaguch evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC,
1051 1.101 yamaguch NULL, device_xname(sc->sc_dev), "control command dmamap load failed");
1052 1.101 yamaguch evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC,
1053 1.101 yamaguch NULL, device_xname(sc->sc_dev), "control command failed");
1054 1.101 yamaguch }
1055 1.1 hannken
1056 1.101 yamaguch /*
1057 1.101 yamaguch * allocate memory
1058 1.101 yamaguch */
1059 1.101 yamaguch static int
1060 1.101 yamaguch vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
1061 1.101 yamaguch bus_size_t size, int nsegs, const char *usage)
1062 1.101 yamaguch {
1063 1.101 yamaguch int r;
1064 1.1 hannken
1065 1.101 yamaguch r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
1066 1.101 yamaguch nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
1067 1.32 jdolecek
1068 1.101 yamaguch if (r != 0) {
1069 1.101 yamaguch aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
1070 1.101 yamaguch "error code %d\n", usage, r);
1071 1.55 yamaguch }
1072 1.55 yamaguch
1073 1.101 yamaguch return r;
1074 1.101 yamaguch }
1075 1.63 yamaguch
1076 1.101 yamaguch static void
1077 1.101 yamaguch vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
1078 1.101 yamaguch {
1079 1.1 hannken
1080 1.101 yamaguch if (*map) {
1081 1.101 yamaguch bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
1082 1.101 yamaguch *map = NULL;
1083 1.101 yamaguch }
1084 1.101 yamaguch }
1085 1.11 ozaki
1086 1.101 yamaguch static int
1087 1.101 yamaguch vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
1088 1.101 yamaguch void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
1089 1.101 yamaguch {
1090 1.101 yamaguch int r;
1091 1.1 hannken
1092 1.101 yamaguch r = vioif_dmamap_create(sc, map, size, nsegs, usage);
1093 1.101 yamaguch if (r != 0)
1094 1.101 yamaguch return 1;
1095 1.1 hannken
1096 1.101 yamaguch r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
1097 1.101 yamaguch size, NULL, rw | BUS_DMA_NOWAIT);
1098 1.101 yamaguch if (r != 0) {
1099 1.101 yamaguch vioif_dmamap_destroy(sc, map);
1100 1.101 yamaguch aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
1101 1.101 yamaguch "error code %d\n", usage, r);
1102 1.43 yamaguch }
1103 1.7 ozaki
1104 1.101 yamaguch return r;
1105 1.1 hannken }
1106 1.1 hannken
1107 1.101 yamaguch static void *
1108 1.101 yamaguch vioif_assign_mem(intptr_t *p, size_t size)
1109 1.55 yamaguch {
1110 1.101 yamaguch intptr_t rv;
1111 1.55 yamaguch
1112 1.101 yamaguch rv = *p;
1113 1.101 yamaguch *p += size;
1114 1.55 yamaguch
1115 1.101 yamaguch return (void *)rv;
1116 1.55 yamaguch }
1117 1.55 yamaguch
1118 1.1 hannken /*
1119 1.101 yamaguch * dma memory is used for:
1120 1.101 yamaguch * netq_maps_kva: metadata array for received frames (READ) and
1121 1.101 yamaguch * sent frames (WRITE)
1122 1.101 yamaguch * ctrlq_cmd: command to be sent via ctrl vq (WRITE)
1123 1.101 yamaguch * ctrlq_status: return value for a command via ctrl vq (READ)
1124 1.101 yamaguch * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command
1125 1.101 yamaguch * (WRITE)
1126 1.101 yamaguch * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1127 1.101 yamaguch * class command (WRITE)
1128 1.101 yamaguch * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
1129 1.101 yamaguch * class command (WRITE)
1130 1.101 yamaguch * ctrlq_* structures are allocated only one each; they are protected by
1131 1.101 yamaguch * ctrlq_inuse variable and ctrlq_wait condvar.
1132 1.1 hannken */
1133 1.1 hannken static int
1134 1.101 yamaguch vioif_alloc_mems(struct vioif_softc *sc)
1135 1.1 hannken {
1136 1.33 ozaki struct virtio_softc *vsc = sc->sc_virtio;
1137 1.98 yamaguch struct vioif_netqueue *netq;
1138 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1139 1.101 yamaguch struct vioif_net_map *maps;
1140 1.101 yamaguch unsigned int vq_num;
1141 1.101 yamaguch int r, rsegs;
1142 1.101 yamaguch bus_size_t dmamemsize;
1143 1.101 yamaguch size_t qid, i, netq_num, kmemsize;
1144 1.101 yamaguch void *vaddr;
1145 1.101 yamaguch intptr_t p;
1146 1.101 yamaguch
1147 1.101 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
1148 1.1 hannken
1149 1.101 yamaguch /* allocate DMA memory */
1150 1.101 yamaguch dmamemsize = 0;
1151 1.7 ozaki
1152 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1153 1.101 yamaguch maps = sc->sc_netqs[qid].netq_maps;
1154 1.101 yamaguch vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
1155 1.101 yamaguch dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num;
1156 1.71 yamaguch }
1157 1.71 yamaguch
1158 1.101 yamaguch if (sc->sc_has_ctrl) {
1159 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_cmd);
1160 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_status);
1161 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_rx);
1162 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
1163 1.101 yamaguch + ETHER_ADDR_LEN;
1164 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
1165 1.101 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
1166 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr);
1167 1.101 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mq);
1168 1.101 yamaguch }
1169 1.46 yamaguch
1170 1.101 yamaguch r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0,
1171 1.101 yamaguch &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
1172 1.101 yamaguch if (r != 0) {
1173 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1174 1.101 yamaguch "DMA memory allocation failed, size %zu, "
1175 1.101 yamaguch "error code %d\n", dmamemsize, r);
1176 1.101 yamaguch goto err_none;
1177 1.101 yamaguch }
1178 1.101 yamaguch r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1,
1179 1.101 yamaguch dmamemsize, &vaddr, BUS_DMA_NOWAIT);
1180 1.101 yamaguch if (r != 0) {
1181 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1182 1.101 yamaguch "DMA memory map failed, error code %d\n", r);
1183 1.101 yamaguch goto err_dmamem_alloc;
1184 1.46 yamaguch }
1185 1.46 yamaguch
1186 1.101 yamaguch /* assign DMA memory */
1187 1.101 yamaguch memset(vaddr, 0, dmamemsize);
1188 1.101 yamaguch sc->sc_dmamem = vaddr;
1189 1.101 yamaguch p = (intptr_t) vaddr;
1190 1.47 yamaguch
1191 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1192 1.101 yamaguch netq = &sc->sc_netqs[qid];
1193 1.101 yamaguch maps = netq->netq_maps;
1194 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1195 1.46 yamaguch
1196 1.101 yamaguch netq->netq_maps_kva = vioif_assign_mem(&p,
1197 1.101 yamaguch sizeof(*maps[0].vnm_hdr) * vq_num);
1198 1.101 yamaguch }
1199 1.46 yamaguch
1200 1.101 yamaguch if (sc->sc_has_ctrl) {
1201 1.101 yamaguch ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
1202 1.101 yamaguch sizeof(*ctrlq->ctrlq_cmd));
1203 1.101 yamaguch ctrlq->ctrlq_status = vioif_assign_mem(&p,
1204 1.101 yamaguch sizeof(*ctrlq->ctrlq_status));
1205 1.101 yamaguch ctrlq->ctrlq_rx = vioif_assign_mem(&p,
1206 1.101 yamaguch sizeof(*ctrlq->ctrlq_rx));
1207 1.101 yamaguch ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
1208 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_uc)
1209 1.101 yamaguch + ETHER_ADDR_LEN);
1210 1.101 yamaguch ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
1211 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_mc)
1212 1.101 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
1213 1.101 yamaguch ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p,
1214 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_addr));
1215 1.101 yamaguch ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
1216 1.76 yamaguch }
1217 1.33 ozaki
1218 1.101 yamaguch /* allocate kmem */
1219 1.101 yamaguch kmemsize = 0;
1220 1.85 yamaguch
1221 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1222 1.101 yamaguch netq = &sc->sc_netqs[qid];
1223 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1224 1.55 yamaguch
1225 1.101 yamaguch kmemsize += sizeof(netq->netq_maps[0]) * vq_num;
1226 1.85 yamaguch }
1227 1.85 yamaguch
1228 1.101 yamaguch vaddr = kmem_zalloc(kmemsize, KM_SLEEP);
1229 1.101 yamaguch sc->sc_kmem = vaddr;
1230 1.98 yamaguch
1231 1.101 yamaguch /* assign allocated kmem */
1232 1.101 yamaguch p = (intptr_t) vaddr;
1233 1.46 yamaguch
1234 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1235 1.101 yamaguch netq = &sc->sc_netqs[qid];
1236 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1237 1.85 yamaguch
1238 1.101 yamaguch netq->netq_maps = vioif_assign_mem(&p,
1239 1.101 yamaguch sizeof(netq->netq_maps[0]) * vq_num);
1240 1.85 yamaguch }
1241 1.1 hannken
1242 1.101 yamaguch /* prepare dmamaps */
1243 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1244 1.101 yamaguch static const struct {
1245 1.101 yamaguch const char *msg_hdr;
1246 1.101 yamaguch const char *msg_payload;
1247 1.101 yamaguch int dma_flag;
1248 1.101 yamaguch bus_size_t dma_size;
1249 1.101 yamaguch int dma_nsegs;
1250 1.101 yamaguch } dmaparams[VIOIF_NETQ_IDX] = {
1251 1.101 yamaguch [VIOIF_NETQ_RX] = {
1252 1.101 yamaguch .msg_hdr = "rx header",
1253 1.101 yamaguch .msg_payload = "rx payload",
1254 1.101 yamaguch .dma_flag = BUS_DMA_READ,
1255 1.101 yamaguch .dma_size = MCLBYTES - ETHER_ALIGN,
1256 1.101 yamaguch .dma_nsegs = 1,
1257 1.101 yamaguch },
1258 1.101 yamaguch [VIOIF_NETQ_TX] = {
1259 1.101 yamaguch .msg_hdr = "tx header",
1260 1.101 yamaguch .msg_payload = "tx payload",
1261 1.101 yamaguch .dma_flag = BUS_DMA_WRITE,
1262 1.101 yamaguch .dma_size = ETHER_MAX_LEN,
1263 1.101 yamaguch .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS,
1264 1.101 yamaguch }
1265 1.101 yamaguch };
1266 1.1 hannken
1267 1.101 yamaguch struct virtio_net_hdr *hdrs;
1268 1.101 yamaguch int dir;
1269 1.46 yamaguch
1270 1.101 yamaguch dir = VIOIF_NETQ_DIR(qid);
1271 1.101 yamaguch netq = &sc->sc_netqs[qid];
1272 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1273 1.101 yamaguch maps = netq->netq_maps;
1274 1.101 yamaguch hdrs = netq->netq_maps_kva;
1275 1.7 ozaki
1276 1.101 yamaguch for (i = 0; i < vq_num; i++) {
1277 1.101 yamaguch maps[i].vnm_hdr = &hdrs[i];
1278 1.101 yamaguch
1279 1.101 yamaguch r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map,
1280 1.101 yamaguch maps[i].vnm_hdr, sc->sc_hdr_size, 1,
1281 1.101 yamaguch dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr);
1282 1.101 yamaguch if (r != 0)
1283 1.101 yamaguch goto err_reqs;
1284 1.98 yamaguch
1285 1.101 yamaguch r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map,
1286 1.101 yamaguch dmaparams[dir].dma_size, dmaparams[dir].dma_nsegs,
1287 1.101 yamaguch dmaparams[dir].msg_payload);
1288 1.101 yamaguch if (r != 0)
1289 1.101 yamaguch goto err_reqs;
1290 1.101 yamaguch }
1291 1.101 yamaguch }
1292 1.7 ozaki
1293 1.101 yamaguch if (sc->sc_has_ctrl) {
1294 1.101 yamaguch /* control vq class & command */
1295 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
1296 1.101 yamaguch ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
1297 1.101 yamaguch BUS_DMA_WRITE, "control command");
1298 1.101 yamaguch if (r != 0)
1299 1.101 yamaguch goto err_reqs;
1300 1.1 hannken
1301 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
1302 1.101 yamaguch ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
1303 1.101 yamaguch BUS_DMA_READ, "control status");
1304 1.101 yamaguch if (r != 0)
1305 1.101 yamaguch goto err_reqs;
1306 1.1 hannken
1307 1.101 yamaguch /* control vq rx mode command parameter */
1308 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
1309 1.101 yamaguch ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
1310 1.101 yamaguch BUS_DMA_WRITE, "rx mode control command");
1311 1.101 yamaguch if (r != 0)
1312 1.101 yamaguch goto err_reqs;
1313 1.46 yamaguch
1314 1.101 yamaguch /* multiqueue set command */
1315 1.101 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
1316 1.101 yamaguch ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
1317 1.101 yamaguch BUS_DMA_WRITE, "multiqueue set command");
1318 1.101 yamaguch if (r != 0)
1319 1.101 yamaguch goto err_reqs;
1320 1.36 jdolecek
1321 1.101 yamaguch /* control vq MAC filter table for unicast */
1322 1.101 yamaguch /* do not load now since its length is variable */
1323 1.101 yamaguch r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
1324 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_uc)
1325 1.101 yamaguch + ETHER_ADDR_LEN, 1,
1326 1.101 yamaguch "unicast MAC address filter command");
1327 1.101 yamaguch if (r != 0)
1328 1.101 yamaguch goto err_reqs;
1329 1.95 yamaguch
1330 1.101 yamaguch /* control vq MAC filter table for multicast */
1331 1.101 yamaguch r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
1332 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_mc)
1333 1.101 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
1334 1.101 yamaguch "multicast MAC address filter command");
1335 1.101 yamaguch if (r != 0)
1336 1.101 yamaguch goto err_reqs;
1337 1.36 jdolecek
1338 1.101 yamaguch /* control vq MAC address set command */
1339 1.101 yamaguch r = vioif_dmamap_create_load(sc,
1340 1.101 yamaguch &ctrlq->ctrlq_mac_addr_dmamap,
1341 1.101 yamaguch ctrlq->ctrlq_mac_addr,
1342 1.101 yamaguch sizeof(*ctrlq->ctrlq_mac_addr), 1,
1343 1.101 yamaguch BUS_DMA_WRITE, "mac addr set command");
1344 1.101 yamaguch if (r != 0)
1345 1.101 yamaguch goto err_reqs;
1346 1.101 yamaguch }
1347 1.36 jdolecek
1348 1.101 yamaguch return 0;
1349 1.36 jdolecek
1350 1.101 yamaguch err_reqs:
1351 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
1352 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
1353 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
1354 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
1355 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
1356 1.101 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap);
1357 1.101 yamaguch for (qid = 0; qid < netq_num; qid++) {
1358 1.101 yamaguch vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
1359 1.101 yamaguch maps = sc->sc_netqs[qid].netq_maps;
1360 1.99 yamaguch
1361 1.101 yamaguch for (i = 0; i < vq_num; i++) {
1362 1.101 yamaguch vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map);
1363 1.101 yamaguch vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map);
1364 1.1 hannken }
1365 1.1 hannken }
1366 1.101 yamaguch if (sc->sc_kmem) {
1367 1.101 yamaguch kmem_free(sc->sc_kmem, kmemsize);
1368 1.101 yamaguch sc->sc_kmem = NULL;
1369 1.1 hannken }
1370 1.101 yamaguch bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize);
1371 1.101 yamaguch err_dmamem_alloc:
1372 1.101 yamaguch bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1);
1373 1.101 yamaguch err_none:
1374 1.101 yamaguch return -1;
1375 1.46 yamaguch }
1376 1.46 yamaguch
1377 1.46 yamaguch static void
1378 1.101 yamaguch vioif_alloc_queues(struct vioif_softc *sc)
1379 1.46 yamaguch {
1380 1.101 yamaguch int nvq_pairs = sc->sc_max_nvq_pairs;
1381 1.101 yamaguch size_t nvqs, netq_num;
1382 1.101 yamaguch
1383 1.101 yamaguch KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
1384 1.46 yamaguch
1385 1.101 yamaguch nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
1386 1.101 yamaguch if (sc->sc_has_ctrl)
1387 1.101 yamaguch nvqs++;
1388 1.46 yamaguch
1389 1.101 yamaguch sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
1390 1.101 yamaguch sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * netq_num,
1391 1.101 yamaguch KM_SLEEP);
1392 1.46 yamaguch }
1393 1.46 yamaguch
1394 1.46 yamaguch static void
1395 1.101 yamaguch vioif_free_queues(struct vioif_softc *sc)
1396 1.46 yamaguch {
1397 1.101 yamaguch size_t nvqs, netq_num;
1398 1.46 yamaguch
1399 1.101 yamaguch nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
1400 1.101 yamaguch if (sc->sc_ctrlq.ctrlq_vq)
1401 1.101 yamaguch nvqs++;
1402 1.46 yamaguch
1403 1.101 yamaguch kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num);
1404 1.101 yamaguch kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
1405 1.101 yamaguch sc->sc_netqs = NULL;
1406 1.101 yamaguch sc->sc_vqs = NULL;
1407 1.46 yamaguch }
1408 1.46 yamaguch
1409 1.101 yamaguch /*
1410 1.101 yamaguch * Network queues
1411 1.101 yamaguch */
1412 1.101 yamaguch static int
1413 1.101 yamaguch vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc,
1414 1.101 yamaguch size_t qid, u_int softint_flags)
1415 1.46 yamaguch {
1416 1.101 yamaguch static const struct {
1417 1.101 yamaguch const char *dirname;
1418 1.101 yamaguch int segsize;
1419 1.101 yamaguch int nsegs;
1420 1.101 yamaguch int (*intrhand)(void *);
1421 1.101 yamaguch void (*sihand)(void *);
1422 1.101 yamaguch } params[VIOIF_NETQ_IDX] = {
1423 1.101 yamaguch [VIOIF_NETQ_RX] = {
1424 1.101 yamaguch .dirname = "rx",
1425 1.101 yamaguch .segsize = MCLBYTES,
1426 1.101 yamaguch .nsegs = 2,
1427 1.101 yamaguch .intrhand = vioif_rx_intr,
1428 1.101 yamaguch .sihand = vioif_rx_handle,
1429 1.101 yamaguch },
1430 1.101 yamaguch [VIOIF_NETQ_TX] = {
1431 1.101 yamaguch .dirname = "tx",
1432 1.101 yamaguch .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN,
1433 1.101 yamaguch .nsegs = 2,
1434 1.101 yamaguch .intrhand = vioif_tx_intr,
1435 1.101 yamaguch .sihand = vioif_tx_handle,
1436 1.101 yamaguch }
1437 1.101 yamaguch };
1438 1.46 yamaguch
1439 1.101 yamaguch struct virtqueue *vq;
1440 1.101 yamaguch struct vioif_netqueue *netq;
1441 1.101 yamaguch struct vioif_tx_context *txc;
1442 1.101 yamaguch struct vioif_rx_context *rxc;
1443 1.101 yamaguch char qname[32];
1444 1.101 yamaguch int r, dir;
1445 1.46 yamaguch
1446 1.101 yamaguch txc = NULL;
1447 1.101 yamaguch rxc = NULL;
1448 1.101 yamaguch netq = &sc->sc_netqs[qid];
1449 1.101 yamaguch vq = &sc->sc_vqs[qid];
1450 1.101 yamaguch dir = VIOIF_NETQ_DIR(qid);
1451 1.46 yamaguch
1452 1.101 yamaguch netq->netq_vq = &sc->sc_vqs[qid];
1453 1.101 yamaguch netq->netq_stopping = false;
1454 1.101 yamaguch netq->netq_running_handle = false;
1455 1.46 yamaguch
1456 1.101 yamaguch snprintf(qname, sizeof(qname), "%s%zu",
1457 1.101 yamaguch params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid));
1458 1.101 yamaguch snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup),
1459 1.101 yamaguch "%s-%s", device_xname(sc->sc_dev), qname);
1460 1.46 yamaguch
1461 1.101 yamaguch mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET);
1462 1.101 yamaguch r = virtio_alloc_vq(vsc, vq, qid,
1463 1.101 yamaguch params[dir].segsize + sc->sc_hdr_size,
1464 1.101 yamaguch params[dir].nsegs, qname);
1465 1.101 yamaguch if (r != 0)
1466 1.101 yamaguch goto err;
1467 1.101 yamaguch netq->netq_vq = vq;
1468 1.46 yamaguch
1469 1.101 yamaguch netq->netq_vq->vq_intrhand = params[dir].intrhand;
1470 1.101 yamaguch netq->netq_vq->vq_intrhand_arg = netq;
1471 1.101 yamaguch netq->netq_softint = softint_establish(softint_flags,
1472 1.101 yamaguch params[dir].sihand, netq);
1473 1.101 yamaguch if (netq->netq_softint == NULL) {
1474 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1475 1.101 yamaguch "couldn't establish %s softint\n",
1476 1.101 yamaguch params[dir].dirname);
1477 1.101 yamaguch goto err;
1478 1.46 yamaguch }
1479 1.101 yamaguch vioif_work_set(&netq->netq_work, params[dir].sihand, netq);
1480 1.46 yamaguch
1481 1.101 yamaguch switch (dir) {
1482 1.101 yamaguch case VIOIF_NETQ_RX:
1483 1.101 yamaguch rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP);
1484 1.101 yamaguch netq->netq_ctx = rxc;
1485 1.101 yamaguch /* nothing to do */
1486 1.101 yamaguch break;
1487 1.101 yamaguch case VIOIF_NETQ_TX:
1488 1.101 yamaguch txc = kmem_zalloc(sizeof(*txc), KM_SLEEP);
1489 1.101 yamaguch netq->netq_ctx = (void *)txc;
1490 1.101 yamaguch txc->txc_deferred_transmit = softint_establish(softint_flags,
1491 1.101 yamaguch vioif_deferred_transmit, netq);
1492 1.101 yamaguch if (txc->txc_deferred_transmit == NULL) {
1493 1.101 yamaguch aprint_error_dev(sc->sc_dev,
1494 1.101 yamaguch "couldn't establish softint for "
1495 1.101 yamaguch "tx deferred transmit\n");
1496 1.101 yamaguch goto err;
1497 1.101 yamaguch }
1498 1.101 yamaguch txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc);
1499 1.101 yamaguch txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP);
1500 1.101 yamaguch break;
1501 1.46 yamaguch }
1502 1.46 yamaguch
1503 1.46 yamaguch return 0;
1504 1.46 yamaguch
1505 1.101 yamaguch err:
1506 1.101 yamaguch netq->netq_ctx = NULL;
1507 1.7 ozaki
1508 1.101 yamaguch if (rxc != NULL) {
1509 1.101 yamaguch kmem_free(rxc, sizeof(*rxc));
1510 1.101 yamaguch }
1511 1.1 hannken
1512 1.101 yamaguch if (txc != NULL) {
1513 1.101 yamaguch if (txc->txc_deferred_transmit != NULL)
1514 1.101 yamaguch softint_disestablish(txc->txc_deferred_transmit);
1515 1.101 yamaguch if (txc->txc_intrq != NULL)
1516 1.101 yamaguch pcq_destroy(txc->txc_intrq);
1517 1.101 yamaguch kmem_free(txc, sizeof(txc));
1518 1.101 yamaguch }
1519 1.1 hannken
1520 1.101 yamaguch vioif_work_set(&netq->netq_work, NULL, NULL);
1521 1.101 yamaguch if (netq->netq_softint != NULL) {
1522 1.101 yamaguch softint_disestablish(netq->netq_softint);
1523 1.101 yamaguch netq->netq_softint = NULL;
1524 1.1 hannken }
1525 1.101 yamaguch netq->netq_vq->vq_intrhand = NULL;
1526 1.101 yamaguch netq->netq_vq->vq_intrhand_arg = NULL;
1527 1.1 hannken
1528 1.101 yamaguch virtio_free_vq(vsc, vq);
1529 1.101 yamaguch mutex_destroy(&netq->netq_lock);
1530 1.101 yamaguch netq->netq_vq = NULL;
1531 1.1 hannken
1532 1.101 yamaguch return -1;
1533 1.1 hannken }
1534 1.1 hannken
1535 1.101 yamaguch static void
1536 1.101 yamaguch vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc,
1537 1.101 yamaguch size_t qid)
1538 1.1 hannken {
1539 1.98 yamaguch struct vioif_netqueue *netq;
1540 1.101 yamaguch struct vioif_rx_context *rxc;
1541 1.101 yamaguch struct vioif_tx_context *txc;
1542 1.101 yamaguch int dir;
1543 1.101 yamaguch
1544 1.101 yamaguch netq = &sc->sc_netqs[qid];
1545 1.101 yamaguch
1546 1.101 yamaguch if (netq->netq_vq == NULL)
1547 1.101 yamaguch return;
1548 1.101 yamaguch
1549 1.101 yamaguch netq = &sc->sc_netqs[qid];
1550 1.101 yamaguch dir = VIOIF_NETQ_DIR(qid);
1551 1.101 yamaguch switch (dir) {
1552 1.101 yamaguch case VIOIF_NETQ_RX:
1553 1.101 yamaguch rxc = netq->netq_ctx;
1554 1.101 yamaguch netq->netq_ctx = NULL;
1555 1.101 yamaguch kmem_free(rxc, sizeof(*rxc));
1556 1.101 yamaguch break;
1557 1.101 yamaguch case VIOIF_NETQ_TX:
1558 1.101 yamaguch txc = netq->netq_ctx;
1559 1.101 yamaguch netq->netq_ctx = NULL;
1560 1.101 yamaguch softint_disestablish(txc->txc_deferred_transmit);
1561 1.101 yamaguch pcq_destroy(txc->txc_intrq);
1562 1.101 yamaguch kmem_free(txc, sizeof(*txc));
1563 1.101 yamaguch break;
1564 1.101 yamaguch }
1565 1.1 hannken
1566 1.101 yamaguch softint_disestablish(netq->netq_softint);
1567 1.101 yamaguch virtio_free_vq(vsc, netq->netq_vq);
1568 1.101 yamaguch mutex_destroy(&netq->netq_lock);
1569 1.101 yamaguch netq->netq_vq = NULL;
1570 1.1 hannken }
1571 1.1 hannken
1572 1.98 yamaguch static void
1573 1.98 yamaguch vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq)
1574 1.98 yamaguch {
1575 1.98 yamaguch
1576 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1577 1.98 yamaguch KASSERT(!netq->netq_stopping);
1578 1.98 yamaguch
1579 1.98 yamaguch if (netq->netq_workqueue) {
1580 1.98 yamaguch vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work);
1581 1.98 yamaguch } else {
1582 1.98 yamaguch softint_schedule(netq->netq_softint);
1583 1.98 yamaguch }
1584 1.98 yamaguch }
1585 1.98 yamaguch
1586 1.99 yamaguch static int
1587 1.99 yamaguch vioif_net_load_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map,
1588 1.99 yamaguch struct mbuf *m, int dma_flags)
1589 1.99 yamaguch {
1590 1.99 yamaguch int r;
1591 1.99 yamaguch
1592 1.99 yamaguch KASSERT(map->vnm_mbuf == NULL);
1593 1.99 yamaguch
1594 1.99 yamaguch r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1595 1.99 yamaguch map->vnm_mbuf_map, m, dma_flags | BUS_DMA_NOWAIT);
1596 1.99 yamaguch if (r == 0) {
1597 1.99 yamaguch map->vnm_mbuf = m;
1598 1.99 yamaguch }
1599 1.99 yamaguch
1600 1.99 yamaguch return r;
1601 1.99 yamaguch }
1602 1.99 yamaguch
1603 1.99 yamaguch static void
1604 1.99 yamaguch vioif_net_unload_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map)
1605 1.99 yamaguch {
1606 1.99 yamaguch
1607 1.99 yamaguch KASSERT(map->vnm_mbuf != NULL);
1608 1.99 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1609 1.99 yamaguch map->vnm_mbuf = NULL;
1610 1.99 yamaguch }
1611 1.99 yamaguch
1612 1.99 yamaguch static int
1613 1.99 yamaguch vioif_net_enqueue(struct virtio_softc *vsc, struct virtqueue *vq,
1614 1.99 yamaguch int slot, struct vioif_net_map *map, int dma_ops, bool is_write)
1615 1.99 yamaguch {
1616 1.99 yamaguch int r;
1617 1.99 yamaguch
1618 1.99 yamaguch KASSERT(map->vnm_mbuf != NULL);
1619 1.99 yamaguch
1620 1.99 yamaguch /* This should actually never fail */
1621 1.99 yamaguch r = virtio_enqueue_reserve(vsc, vq, slot,
1622 1.99 yamaguch map->vnm_mbuf_map->dm_nsegs + 1);
1623 1.99 yamaguch if (r != 0) {
1624 1.99 yamaguch /* slot already freed by virtio_enqueue_reserve */
1625 1.99 yamaguch return r;
1626 1.99 yamaguch }
1627 1.99 yamaguch
1628 1.99 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1629 1.99 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, dma_ops);
1630 1.99 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1631 1.99 yamaguch 0, map->vnm_hdr_map->dm_mapsize, dma_ops);
1632 1.99 yamaguch
1633 1.99 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, is_write);
1634 1.99 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, is_write);
1635 1.99 yamaguch virtio_enqueue_commit(vsc, vq, slot, false);
1636 1.99 yamaguch
1637 1.99 yamaguch return 0;
1638 1.99 yamaguch }
1639 1.99 yamaguch
1640 1.99 yamaguch static int
1641 1.99 yamaguch vioif_net_enqueue_tx(struct virtio_softc *vsc, struct virtqueue *vq,
1642 1.99 yamaguch int slot, struct vioif_net_map *map)
1643 1.99 yamaguch {
1644 1.99 yamaguch
1645 1.99 yamaguch return vioif_net_enqueue(vsc, vq, slot, map,
1646 1.99 yamaguch BUS_DMASYNC_PREWRITE, true);
1647 1.99 yamaguch }
1648 1.99 yamaguch
1649 1.99 yamaguch static int
1650 1.99 yamaguch vioif_net_enqueue_rx(struct virtio_softc *vsc, struct virtqueue *vq,
1651 1.99 yamaguch int slot, struct vioif_net_map *map)
1652 1.99 yamaguch {
1653 1.99 yamaguch
1654 1.99 yamaguch return vioif_net_enqueue(vsc, vq, slot, map,
1655 1.99 yamaguch BUS_DMASYNC_PREREAD, false);
1656 1.99 yamaguch }
1657 1.99 yamaguch
1658 1.101 yamaguch static struct mbuf *
1659 1.101 yamaguch vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq,
1660 1.101 yamaguch int slot, struct vioif_net_map *map, int dma_flags)
1661 1.101 yamaguch {
1662 1.101 yamaguch struct mbuf *m;
1663 1.101 yamaguch
1664 1.101 yamaguch m = map->vnm_mbuf;
1665 1.101 yamaguch KASSERT(m != NULL);
1666 1.101 yamaguch map->vnm_mbuf = NULL;
1667 1.101 yamaguch
1668 1.101 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1669 1.101 yamaguch 0, map->vnm_hdr_map->dm_mapsize, dma_flags);
1670 1.101 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1671 1.101 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, dma_flags);
1672 1.101 yamaguch
1673 1.101 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1674 1.101 yamaguch virtio_dequeue_commit(vsc, vq, slot);
1675 1.101 yamaguch
1676 1.101 yamaguch return m;
1677 1.101 yamaguch }
1678 1.101 yamaguch
1679 1.101 yamaguch static void
1680 1.101 yamaguch vioif_net_intr_enable(struct vioif_softc *sc, struct virtio_softc *vsc)
1681 1.101 yamaguch {
1682 1.101 yamaguch struct vioif_netqueue *netq;
1683 1.101 yamaguch size_t i, act_qnum;
1684 1.101 yamaguch int enqueued;
1685 1.101 yamaguch
1686 1.101 yamaguch act_qnum = sc->sc_act_nvq_pairs * 2;
1687 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
1688 1.101 yamaguch netq = &sc->sc_netqs[i];
1689 1.101 yamaguch
1690 1.101 yamaguch KASSERT(!netq->netq_stopping);
1691 1.101 yamaguch KASSERT(!netq->netq_running_handle);
1692 1.101 yamaguch
1693 1.101 yamaguch enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
1694 1.101 yamaguch if (enqueued != 0) {
1695 1.101 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1696 1.101 yamaguch
1697 1.101 yamaguch mutex_enter(&netq->netq_lock);
1698 1.101 yamaguch netq->netq_running_handle = true;
1699 1.101 yamaguch vioif_net_sched_handle(sc, netq);
1700 1.101 yamaguch mutex_exit(&netq->netq_lock);
1701 1.101 yamaguch }
1702 1.101 yamaguch }
1703 1.101 yamaguch }
1704 1.101 yamaguch
1705 1.101 yamaguch static void
1706 1.101 yamaguch vioif_net_intr_disable(struct vioif_softc *sc, struct virtio_softc *vsc)
1707 1.101 yamaguch {
1708 1.101 yamaguch struct vioif_netqueue *netq;
1709 1.101 yamaguch size_t i, act_qnum;
1710 1.101 yamaguch
1711 1.101 yamaguch act_qnum = sc->sc_act_nvq_pairs * 2;
1712 1.101 yamaguch for (i = 0; i < act_qnum; i++) {
1713 1.101 yamaguch netq = &sc->sc_netqs[i];
1714 1.101 yamaguch
1715 1.101 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1716 1.101 yamaguch }
1717 1.101 yamaguch }
1718 1.101 yamaguch
1719 1.101 yamaguch /*
1720 1.101 yamaguch * Receive implementation
1721 1.101 yamaguch */
1722 1.101 yamaguch /* enqueue mbufs to receive slots */
1723 1.101 yamaguch static void
1724 1.101 yamaguch vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq)
1725 1.101 yamaguch {
1726 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1727 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1728 1.101 yamaguch struct vioif_rx_context *rxc;
1729 1.101 yamaguch struct vioif_net_map *map;
1730 1.101 yamaguch struct mbuf *m;
1731 1.101 yamaguch int i, r, ndone = 0;
1732 1.101 yamaguch
1733 1.101 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1734 1.101 yamaguch
1735 1.101 yamaguch rxc = netq->netq_ctx;
1736 1.101 yamaguch
1737 1.101 yamaguch for (i = 0; i < vq->vq_num; i++) {
1738 1.101 yamaguch int slot;
1739 1.101 yamaguch r = virtio_enqueue_prep(vsc, vq, &slot);
1740 1.101 yamaguch if (r == EAGAIN)
1741 1.101 yamaguch break;
1742 1.101 yamaguch if (__predict_false(r != 0))
1743 1.101 yamaguch panic("enqueue_prep for rx buffers");
1744 1.101 yamaguch
1745 1.101 yamaguch MGETHDR(m, M_DONTWAIT, MT_DATA);
1746 1.101 yamaguch if (m == NULL) {
1747 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1748 1.101 yamaguch rxc->rxc_mbuf_enobufs.ev_count++;
1749 1.101 yamaguch break;
1750 1.101 yamaguch }
1751 1.101 yamaguch MCLGET(m, M_DONTWAIT);
1752 1.101 yamaguch if ((m->m_flags & M_EXT) == 0) {
1753 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1754 1.101 yamaguch m_freem(m);
1755 1.101 yamaguch rxc->rxc_mbuf_enobufs.ev_count++;
1756 1.101 yamaguch break;
1757 1.101 yamaguch }
1758 1.101 yamaguch
1759 1.101 yamaguch m->m_len = m->m_pkthdr.len = MCLBYTES;
1760 1.101 yamaguch m_adj(m, ETHER_ALIGN);
1761 1.101 yamaguch
1762 1.101 yamaguch map = &netq->netq_maps[slot];
1763 1.101 yamaguch r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_READ);
1764 1.101 yamaguch if (r != 0) {
1765 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1766 1.101 yamaguch m_freem(m);
1767 1.101 yamaguch netq->netq_mbuf_load_failed.ev_count++;
1768 1.101 yamaguch break;
1769 1.101 yamaguch }
1770 1.101 yamaguch
1771 1.101 yamaguch r = vioif_net_enqueue_rx(vsc, vq, slot, map);
1772 1.101 yamaguch if (r != 0) {
1773 1.101 yamaguch vioif_net_unload_mbuf(vsc, map);
1774 1.101 yamaguch netq->netq_enqueue_failed.ev_count++;
1775 1.101 yamaguch m_freem(m);
1776 1.101 yamaguch /* slot already freed by vioif_net_enqueue_rx */
1777 1.101 yamaguch break;
1778 1.101 yamaguch }
1779 1.101 yamaguch
1780 1.101 yamaguch ndone++;
1781 1.101 yamaguch }
1782 1.101 yamaguch
1783 1.101 yamaguch if (ndone > 0)
1784 1.101 yamaguch vioif_notify(vsc, vq);
1785 1.101 yamaguch }
1786 1.101 yamaguch
1787 1.101 yamaguch /* dequeue received packets */
1788 1.101 yamaguch static bool
1789 1.101 yamaguch vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1790 1.101 yamaguch struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
1791 1.101 yamaguch {
1792 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1793 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1794 1.101 yamaguch struct vioif_net_map *map;
1795 1.101 yamaguch struct mbuf *m;
1796 1.101 yamaguch int slot, len;
1797 1.101 yamaguch bool more;
1798 1.101 yamaguch size_t ndeq;
1799 1.101 yamaguch
1800 1.101 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1801 1.101 yamaguch
1802 1.101 yamaguch more = false;
1803 1.101 yamaguch ndeq = 0;
1804 1.101 yamaguch
1805 1.101 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
1806 1.101 yamaguch goto done;
1807 1.101 yamaguch
1808 1.101 yamaguch for (;;ndeq++) {
1809 1.101 yamaguch if (ndeq >= limit) {
1810 1.101 yamaguch more = true;
1811 1.101 yamaguch break;
1812 1.101 yamaguch }
1813 1.101 yamaguch
1814 1.101 yamaguch if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1815 1.101 yamaguch break;
1816 1.101 yamaguch
1817 1.101 yamaguch map = &netq->netq_maps[slot];
1818 1.101 yamaguch KASSERT(map->vnm_mbuf != NULL);
1819 1.101 yamaguch m = vioif_net_dequeue_commit(vsc, vq, slot,
1820 1.101 yamaguch map, BUS_DMASYNC_POSTREAD);
1821 1.101 yamaguch KASSERT(m != NULL);
1822 1.101 yamaguch
1823 1.101 yamaguch m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size;
1824 1.101 yamaguch m_set_rcvif(m, ifp);
1825 1.101 yamaguch if_percpuq_enqueue(ifp->if_percpuq, m);
1826 1.101 yamaguch }
1827 1.101 yamaguch
1828 1.101 yamaguch done:
1829 1.101 yamaguch if (ndeqp != NULL)
1830 1.101 yamaguch *ndeqp = ndeq;
1831 1.101 yamaguch
1832 1.101 yamaguch return more;
1833 1.101 yamaguch }
1834 1.101 yamaguch
1835 1.101 yamaguch static void
1836 1.101 yamaguch vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
1837 1.101 yamaguch struct vioif_netqueue *netq)
1838 1.101 yamaguch {
1839 1.101 yamaguch struct vioif_net_map *map;
1840 1.101 yamaguch struct mbuf *m;
1841 1.101 yamaguch unsigned int i, vq_num;
1842 1.101 yamaguch bool more;
1843 1.101 yamaguch
1844 1.101 yamaguch mutex_enter(&netq->netq_lock);
1845 1.101 yamaguch
1846 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
1847 1.101 yamaguch for (;;) {
1848 1.101 yamaguch more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL);
1849 1.101 yamaguch if (more == false)
1850 1.101 yamaguch break;
1851 1.101 yamaguch }
1852 1.101 yamaguch
1853 1.101 yamaguch for (i = 0; i < vq_num; i++) {
1854 1.101 yamaguch map = &netq->netq_maps[i];
1855 1.101 yamaguch
1856 1.101 yamaguch m = map->vnm_mbuf;
1857 1.101 yamaguch if (m == NULL)
1858 1.101 yamaguch continue;
1859 1.101 yamaguch
1860 1.101 yamaguch vioif_net_unload_mbuf(vsc, map);
1861 1.101 yamaguch m_freem(m);
1862 1.101 yamaguch }
1863 1.101 yamaguch mutex_exit(&netq->netq_lock);
1864 1.101 yamaguch }
1865 1.101 yamaguch
1866 1.101 yamaguch static void
1867 1.101 yamaguch vioif_rx_handle_locked(void *xnetq, u_int limit)
1868 1.101 yamaguch {
1869 1.101 yamaguch struct vioif_netqueue *netq = xnetq;
1870 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1871 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1872 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1873 1.101 yamaguch bool more;
1874 1.101 yamaguch int enqueued;
1875 1.101 yamaguch size_t ndeq;
1876 1.101 yamaguch
1877 1.101 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1878 1.101 yamaguch KASSERT(!netq->netq_stopping);
1879 1.101 yamaguch
1880 1.101 yamaguch more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq);
1881 1.101 yamaguch if (ndeq > 0)
1882 1.101 yamaguch vioif_populate_rx_mbufs_locked(sc, netq);
1883 1.101 yamaguch
1884 1.101 yamaguch if (more) {
1885 1.101 yamaguch vioif_net_sched_handle(sc, netq);
1886 1.101 yamaguch return;
1887 1.101 yamaguch }
1888 1.101 yamaguch
1889 1.101 yamaguch enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
1890 1.101 yamaguch if (enqueued != 0) {
1891 1.101 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1892 1.101 yamaguch vioif_net_sched_handle(sc, netq);
1893 1.101 yamaguch return;
1894 1.101 yamaguch }
1895 1.101 yamaguch
1896 1.101 yamaguch netq->netq_running_handle = false;
1897 1.101 yamaguch }
1898 1.101 yamaguch
1899 1.101 yamaguch static int
1900 1.101 yamaguch vioif_rx_intr(void *arg)
1901 1.99 yamaguch {
1902 1.101 yamaguch struct vioif_netqueue *netq = arg;
1903 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1904 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1905 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1906 1.101 yamaguch u_int limit;
1907 1.99 yamaguch
1908 1.101 yamaguch mutex_enter(&netq->netq_lock);
1909 1.99 yamaguch
1910 1.101 yamaguch /* handler is already running in softint/workqueue */
1911 1.101 yamaguch if (netq->netq_running_handle)
1912 1.101 yamaguch goto done;
1913 1.99 yamaguch
1914 1.101 yamaguch netq->netq_running_handle = true;
1915 1.99 yamaguch
1916 1.101 yamaguch limit = sc->sc_rx_intr_process_limit;
1917 1.101 yamaguch virtio_stop_vq_intr(vsc, vq);
1918 1.101 yamaguch vioif_rx_handle_locked(netq, limit);
1919 1.99 yamaguch
1920 1.101 yamaguch done:
1921 1.101 yamaguch mutex_exit(&netq->netq_lock);
1922 1.101 yamaguch return 1;
1923 1.99 yamaguch }
1924 1.99 yamaguch
1925 1.99 yamaguch static void
1926 1.101 yamaguch vioif_rx_handle(void *xnetq)
1927 1.99 yamaguch {
1928 1.101 yamaguch struct vioif_netqueue *netq = xnetq;
1929 1.101 yamaguch struct virtqueue *vq = netq->netq_vq;
1930 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1931 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1932 1.101 yamaguch u_int limit;
1933 1.99 yamaguch
1934 1.101 yamaguch mutex_enter(&netq->netq_lock);
1935 1.99 yamaguch
1936 1.101 yamaguch KASSERT(netq->netq_running_handle);
1937 1.99 yamaguch
1938 1.101 yamaguch if (netq->netq_stopping) {
1939 1.101 yamaguch netq->netq_running_handle = false;
1940 1.101 yamaguch goto done;
1941 1.99 yamaguch }
1942 1.99 yamaguch
1943 1.101 yamaguch limit = sc->sc_rx_process_limit;
1944 1.101 yamaguch vioif_rx_handle_locked(netq, limit);
1945 1.99 yamaguch
1946 1.101 yamaguch done:
1947 1.101 yamaguch mutex_exit(&netq->netq_lock);
1948 1.99 yamaguch }
1949 1.99 yamaguch
1950 1.1 hannken /*
1951 1.101 yamaguch * Transmition implementation
1952 1.1 hannken */
1953 1.101 yamaguch /* enqueue mbufs to send */
1954 1.1 hannken static void
1955 1.101 yamaguch vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq,
1956 1.101 yamaguch bool is_transmit)
1957 1.12 ozaki {
1958 1.101 yamaguch struct vioif_softc *sc = ifp->if_softc;
1959 1.101 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1960 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1961 1.101 yamaguch struct vioif_tx_context *txc;
1962 1.95 yamaguch struct vioif_net_map *map;
1963 1.89 yamaguch struct mbuf *m;
1964 1.101 yamaguch int queued = 0;
1965 1.1 hannken
1966 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1967 1.98 yamaguch
1968 1.101 yamaguch if (netq->netq_stopping ||
1969 1.101 yamaguch !ISSET(ifp->if_flags, IFF_RUNNING))
1970 1.101 yamaguch return;
1971 1.101 yamaguch
1972 1.101 yamaguch txc = netq->netq_ctx;
1973 1.101 yamaguch
1974 1.101 yamaguch if (!txc->txc_link_active)
1975 1.101 yamaguch return;
1976 1.101 yamaguch
1977 1.101 yamaguch if (!is_transmit &&
1978 1.101 yamaguch ISSET(ifp->if_flags, IFF_OACTIVE))
1979 1.101 yamaguch return;
1980 1.7 ozaki
1981 1.101 yamaguch for (;;) {
1982 1.101 yamaguch int slot, r;
1983 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
1984 1.1 hannken if (r == EAGAIN)
1985 1.1 hannken break;
1986 1.97 yamaguch if (__predict_false(r != 0))
1987 1.101 yamaguch panic("enqueue_prep for tx buffers");
1988 1.101 yamaguch
1989 1.101 yamaguch if (is_transmit)
1990 1.101 yamaguch m = pcq_get(txc->txc_intrq);
1991 1.101 yamaguch else
1992 1.101 yamaguch IFQ_DEQUEUE(&ifp->if_snd, m);
1993 1.89 yamaguch
1994 1.89 yamaguch if (m == NULL) {
1995 1.91 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1996 1.91 yamaguch break;
1997 1.91 yamaguch }
1998 1.89 yamaguch
1999 1.101 yamaguch map = &netq->netq_maps[slot];
2000 1.101 yamaguch KASSERT(map->vnm_mbuf == NULL);
2001 1.89 yamaguch
2002 1.101 yamaguch r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE);
2003 1.91 yamaguch if (r != 0) {
2004 1.101 yamaguch /* maybe just too fragmented */
2005 1.101 yamaguch struct mbuf *newm;
2006 1.101 yamaguch
2007 1.101 yamaguch newm = m_defrag(m, M_NOWAIT);
2008 1.101 yamaguch if (newm != NULL) {
2009 1.101 yamaguch m = newm;
2010 1.101 yamaguch r = vioif_net_load_mbuf(vsc, map, m,
2011 1.101 yamaguch BUS_DMA_WRITE);
2012 1.101 yamaguch } else {
2013 1.101 yamaguch txc->txc_defrag_failed.ev_count++;
2014 1.101 yamaguch r = -1;
2015 1.101 yamaguch }
2016 1.101 yamaguch
2017 1.101 yamaguch if (r != 0) {
2018 1.101 yamaguch netq->netq_mbuf_load_failed.ev_count++;
2019 1.101 yamaguch m_freem(m);
2020 1.101 yamaguch if_statinc(ifp, if_oerrors);
2021 1.101 yamaguch virtio_enqueue_abort(vsc, vq, slot);
2022 1.101 yamaguch continue;
2023 1.101 yamaguch }
2024 1.1 hannken }
2025 1.89 yamaguch
2026 1.101 yamaguch memset(map->vnm_hdr, 0, sc->sc_hdr_size);
2027 1.101 yamaguch
2028 1.101 yamaguch r = vioif_net_enqueue_tx(vsc, vq, slot, map);
2029 1.1 hannken if (r != 0) {
2030 1.101 yamaguch netq->netq_enqueue_failed.ev_count++;
2031 1.99 yamaguch vioif_net_unload_mbuf(vsc, map);
2032 1.89 yamaguch m_freem(m);
2033 1.101 yamaguch /* slot already freed by vioif_net_enqueue_tx */
2034 1.101 yamaguch
2035 1.101 yamaguch if_statinc(ifp, if_oerrors);
2036 1.101 yamaguch continue;
2037 1.1 hannken }
2038 1.95 yamaguch
2039 1.101 yamaguch queued++;
2040 1.101 yamaguch bpf_mtap(ifp, m, BPF_D_OUT);
2041 1.1 hannken }
2042 1.99 yamaguch
2043 1.101 yamaguch if (queued > 0) {
2044 1.99 yamaguch vioif_notify(vsc, vq);
2045 1.101 yamaguch ifp->if_timer = 5;
2046 1.93 yamaguch }
2047 1.7 ozaki }
2048 1.7 ozaki
2049 1.101 yamaguch /* dequeue sent mbufs */
2050 1.55 yamaguch static bool
2051 1.101 yamaguch vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
2052 1.101 yamaguch struct vioif_netqueue *netq, u_int limit)
2053 1.7 ozaki {
2054 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2055 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2056 1.95 yamaguch struct vioif_net_map *map;
2057 1.1 hannken struct mbuf *m;
2058 1.1 hannken int slot, len;
2059 1.101 yamaguch bool more = false;
2060 1.1 hannken
2061 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
2062 1.7 ozaki
2063 1.54 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
2064 1.101 yamaguch return false;
2065 1.55 yamaguch
2066 1.101 yamaguch for (;;) {
2067 1.101 yamaguch if (limit-- == 0) {
2068 1.55 yamaguch more = true;
2069 1.55 yamaguch break;
2070 1.55 yamaguch }
2071 1.55 yamaguch
2072 1.55 yamaguch if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
2073 1.55 yamaguch break;
2074 1.55 yamaguch
2075 1.98 yamaguch map = &netq->netq_maps[slot];
2076 1.95 yamaguch KASSERT(map->vnm_mbuf != NULL);
2077 1.99 yamaguch m = vioif_net_dequeue_commit(vsc, vq, slot,
2078 1.101 yamaguch map, BUS_DMASYNC_POSTWRITE);
2079 1.99 yamaguch KASSERT(m != NULL);
2080 1.95 yamaguch
2081 1.101 yamaguch if_statinc(ifp, if_opackets);
2082 1.101 yamaguch m_freem(m);
2083 1.70 skrll }
2084 1.87 yamaguch
2085 1.101 yamaguch return more;
2086 1.66 reinoud }
2087 1.66 reinoud
2088 1.101 yamaguch static void
2089 1.101 yamaguch vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
2090 1.101 yamaguch struct vioif_netqueue *netq)
2091 1.1 hannken {
2092 1.101 yamaguch struct vioif_net_map *map;
2093 1.101 yamaguch struct mbuf *m;
2094 1.101 yamaguch unsigned int i, vq_num;
2095 1.101 yamaguch bool more;
2096 1.55 yamaguch
2097 1.98 yamaguch mutex_enter(&netq->netq_lock);
2098 1.7 ozaki
2099 1.101 yamaguch vq_num = netq->netq_vq->vq_num;
2100 1.101 yamaguch for (;;) {
2101 1.101 yamaguch more = vioif_tx_deq_locked(sc, vsc, netq, vq_num);
2102 1.101 yamaguch if (more == false)
2103 1.101 yamaguch break;
2104 1.101 yamaguch }
2105 1.55 yamaguch
2106 1.101 yamaguch for (i = 0; i < vq_num; i++) {
2107 1.101 yamaguch map = &netq->netq_maps[i];
2108 1.55 yamaguch
2109 1.101 yamaguch m = map->vnm_mbuf;
2110 1.101 yamaguch if (m == NULL)
2111 1.101 yamaguch continue;
2112 1.55 yamaguch
2113 1.101 yamaguch vioif_net_unload_mbuf(vsc, map);
2114 1.101 yamaguch m_freem(m);
2115 1.101 yamaguch }
2116 1.98 yamaguch mutex_exit(&netq->netq_lock);
2117 1.55 yamaguch }
2118 1.55 yamaguch
2119 1.55 yamaguch static void
2120 1.101 yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
2121 1.55 yamaguch {
2122 1.55 yamaguch
2123 1.101 yamaguch /*
2124 1.101 yamaguch * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
2125 1.101 yamaguch */
2126 1.101 yamaguch vioif_send_common_locked(ifp, netq, false);
2127 1.101 yamaguch
2128 1.101 yamaguch }
2129 1.85 yamaguch
2130 1.101 yamaguch static void
2131 1.101 yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
2132 1.101 yamaguch {
2133 1.55 yamaguch
2134 1.101 yamaguch vioif_send_common_locked(ifp, netq, true);
2135 1.101 yamaguch }
2136 1.1 hannken
2137 1.101 yamaguch static void
2138 1.101 yamaguch vioif_deferred_transmit(void *arg)
2139 1.101 yamaguch {
2140 1.101 yamaguch struct vioif_netqueue *netq = arg;
2141 1.101 yamaguch struct virtio_softc *vsc = netq->netq_vq->vq_owner;
2142 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
2143 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2144 1.1 hannken
2145 1.101 yamaguch mutex_enter(&netq->netq_lock);
2146 1.101 yamaguch vioif_send_common_locked(ifp, netq, true);
2147 1.98 yamaguch mutex_exit(&netq->netq_lock);
2148 1.1 hannken }
2149 1.1 hannken
2150 1.66 reinoud static void
2151 1.98 yamaguch vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit)
2152 1.66 reinoud {
2153 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2154 1.98 yamaguch struct vioif_tx_context *txc = netq->netq_ctx;
2155 1.66 reinoud struct virtio_softc *vsc = vq->vq_owner;
2156 1.66 reinoud struct vioif_softc *sc = device_private(virtio_child(vsc));
2157 1.66 reinoud struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2158 1.66 reinoud bool more;
2159 1.87 yamaguch int enqueued;
2160 1.66 reinoud
2161 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
2162 1.98 yamaguch KASSERT(!netq->netq_stopping);
2163 1.66 reinoud
2164 1.98 yamaguch more = vioif_tx_deq_locked(sc, vsc, netq, limit);
2165 1.66 reinoud if (more) {
2166 1.98 yamaguch vioif_net_sched_handle(sc, netq);
2167 1.66 reinoud return;
2168 1.66 reinoud }
2169 1.66 reinoud
2170 1.87 yamaguch enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ?
2171 1.87 yamaguch virtio_postpone_intr_smart(vsc, vq):
2172 1.87 yamaguch virtio_start_vq_intr(vsc, vq);
2173 1.87 yamaguch if (enqueued != 0) {
2174 1.87 yamaguch virtio_stop_vq_intr(vsc, vq);
2175 1.98 yamaguch vioif_net_sched_handle(sc, netq);
2176 1.66 reinoud return;
2177 1.66 reinoud }
2178 1.66 reinoud
2179 1.98 yamaguch netq->netq_running_handle = false;
2180 1.84 yamaguch
2181 1.66 reinoud /* for ALTQ */
2182 1.98 yamaguch if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]) {
2183 1.66 reinoud if_schedule_deferred_start(ifp);
2184 1.66 reinoud ifp->if_flags &= ~IFF_OACTIVE;
2185 1.66 reinoud }
2186 1.98 yamaguch softint_schedule(txc->txc_deferred_transmit);
2187 1.66 reinoud }
2188 1.66 reinoud
2189 1.1 hannken static int
2190 1.54 yamaguch vioif_tx_intr(void *arg)
2191 1.1 hannken {
2192 1.98 yamaguch struct vioif_netqueue *netq = arg;
2193 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2194 1.1 hannken struct virtio_softc *vsc = vq->vq_owner;
2195 1.32 jdolecek struct vioif_softc *sc = device_private(virtio_child(vsc));
2196 1.55 yamaguch u_int limit;
2197 1.55 yamaguch
2198 1.98 yamaguch mutex_enter(&netq->netq_lock);
2199 1.7 ozaki
2200 1.84 yamaguch /* tx handler is already running in softint/workqueue */
2201 1.98 yamaguch if (netq->netq_running_handle)
2202 1.84 yamaguch goto done;
2203 1.7 ozaki
2204 1.98 yamaguch if (netq->netq_stopping)
2205 1.84 yamaguch goto done;
2206 1.55 yamaguch
2207 1.98 yamaguch netq->netq_running_handle = true;
2208 1.55 yamaguch
2209 1.84 yamaguch virtio_stop_vq_intr(vsc, vq);
2210 1.98 yamaguch netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl;
2211 1.98 yamaguch limit = sc->sc_tx_intr_process_limit;
2212 1.98 yamaguch vioif_tx_handle_locked(netq, limit);
2213 1.7 ozaki
2214 1.84 yamaguch done:
2215 1.98 yamaguch mutex_exit(&netq->netq_lock);
2216 1.55 yamaguch return 1;
2217 1.55 yamaguch }
2218 1.55 yamaguch
2219 1.55 yamaguch static void
2220 1.98 yamaguch vioif_tx_handle(void *xnetq)
2221 1.55 yamaguch {
2222 1.98 yamaguch struct vioif_netqueue *netq = xnetq;
2223 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
2224 1.55 yamaguch struct virtio_softc *vsc = vq->vq_owner;
2225 1.55 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
2226 1.55 yamaguch u_int limit;
2227 1.55 yamaguch
2228 1.98 yamaguch mutex_enter(&netq->netq_lock);
2229 1.85 yamaguch
2230 1.98 yamaguch KASSERT(netq->netq_running_handle);
2231 1.85 yamaguch
2232 1.98 yamaguch if (netq->netq_stopping) {
2233 1.98 yamaguch netq->netq_running_handle = false;
2234 1.85 yamaguch goto done;
2235 1.85 yamaguch }
2236 1.85 yamaguch
2237 1.55 yamaguch limit = sc->sc_tx_process_limit;
2238 1.98 yamaguch vioif_tx_handle_locked(netq, limit);
2239 1.55 yamaguch
2240 1.85 yamaguch done:
2241 1.98 yamaguch mutex_exit(&netq->netq_lock);
2242 1.7 ozaki }
2243 1.7 ozaki
2244 1.1 hannken /*
2245 1.1 hannken * Control vq
2246 1.1 hannken */
2247 1.1 hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
2248 1.44 yamaguch static void
2249 1.44 yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
2250 1.1 hannken {
2251 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2252 1.1 hannken
2253 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2254 1.43 yamaguch while (ctrlq->ctrlq_inuse != FREE)
2255 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
2256 1.43 yamaguch ctrlq->ctrlq_inuse = INUSE;
2257 1.44 yamaguch ctrlq->ctrlq_owner = curlwp;
2258 1.44 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2259 1.44 yamaguch }
2260 1.44 yamaguch
2261 1.44 yamaguch static void
2262 1.44 yamaguch vioif_ctrl_release(struct vioif_softc *sc)
2263 1.44 yamaguch {
2264 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2265 1.44 yamaguch
2266 1.44 yamaguch KASSERT(ctrlq->ctrlq_inuse != FREE);
2267 1.44 yamaguch KASSERT(ctrlq->ctrlq_owner == curlwp);
2268 1.44 yamaguch
2269 1.44 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2270 1.44 yamaguch ctrlq->ctrlq_inuse = FREE;
2271 1.46 yamaguch ctrlq->ctrlq_owner = NULL;
2272 1.44 yamaguch cv_signal(&ctrlq->ctrlq_wait);
2273 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2274 1.44 yamaguch }
2275 1.44 yamaguch
2276 1.44 yamaguch static int
2277 1.44 yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
2278 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2279 1.44 yamaguch {
2280 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2281 1.44 yamaguch int i, r, loaded;
2282 1.44 yamaguch
2283 1.44 yamaguch loaded = 0;
2284 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2285 1.44 yamaguch r = bus_dmamap_load(virtio_dmat(vsc),
2286 1.44 yamaguch specs[i].dmamap, specs[i].buf, specs[i].bufsize,
2287 1.48 msaitoh NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
2288 1.44 yamaguch if (r) {
2289 1.63 yamaguch sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++;
2290 1.44 yamaguch goto err;
2291 1.44 yamaguch }
2292 1.44 yamaguch loaded++;
2293 1.44 yamaguch
2294 1.44 yamaguch }
2295 1.44 yamaguch
2296 1.44 yamaguch return r;
2297 1.44 yamaguch
2298 1.44 yamaguch err:
2299 1.44 yamaguch for (i = 0; i < loaded; i++) {
2300 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2301 1.44 yamaguch }
2302 1.44 yamaguch
2303 1.44 yamaguch return r;
2304 1.44 yamaguch }
2305 1.44 yamaguch
2306 1.44 yamaguch static void
2307 1.44 yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
2308 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2309 1.44 yamaguch {
2310 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2311 1.44 yamaguch int i;
2312 1.44 yamaguch
2313 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2314 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2315 1.44 yamaguch }
2316 1.44 yamaguch }
2317 1.44 yamaguch
2318 1.44 yamaguch static int
2319 1.44 yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
2320 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2321 1.44 yamaguch {
2322 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2323 1.44 yamaguch struct virtqueue *vq = ctrlq->ctrlq_vq;
2324 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2325 1.44 yamaguch int i, r, slot;
2326 1.43 yamaguch
2327 1.44 yamaguch ctrlq->ctrlq_cmd->class = class;
2328 1.43 yamaguch ctrlq->ctrlq_cmd->command = cmd;
2329 1.1 hannken
2330 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
2331 1.50 christos 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
2332 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2333 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
2334 1.50 christos 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
2335 1.44 yamaguch }
2336 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
2337 1.50 christos 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
2338 1.1 hannken
2339 1.66 reinoud /* we need to explicitly (re)start vq intr when using RING EVENT IDX */
2340 1.66 reinoud if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
2341 1.66 reinoud virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
2342 1.66 reinoud
2343 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
2344 1.1 hannken if (r != 0)
2345 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2346 1.44 yamaguch r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
2347 1.1 hannken if (r != 0)
2348 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2349 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
2350 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2351 1.44 yamaguch virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
2352 1.44 yamaguch }
2353 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
2354 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, true);
2355 1.1 hannken
2356 1.1 hannken /* wait for done */
2357 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2358 1.43 yamaguch while (ctrlq->ctrlq_inuse != DONE)
2359 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
2360 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2361 1.1 hannken /* already dequeueued */
2362 1.1 hannken
2363 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
2364 1.50 christos sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
2365 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2366 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
2367 1.50 christos specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
2368 1.44 yamaguch }
2369 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
2370 1.50 christos sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
2371 1.1 hannken
2372 1.43 yamaguch if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
2373 1.1 hannken r = 0;
2374 1.1 hannken else {
2375 1.63 yamaguch device_printf(sc->sc_dev, "failed setting rx mode\n");
2376 1.63 yamaguch sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++;
2377 1.1 hannken r = EIO;
2378 1.1 hannken }
2379 1.1 hannken
2380 1.44 yamaguch return r;
2381 1.44 yamaguch }
2382 1.44 yamaguch
2383 1.101 yamaguch /* ctrl vq interrupt; wake up the command issuer */
2384 1.101 yamaguch static int
2385 1.101 yamaguch vioif_ctrl_intr(void *arg)
2386 1.101 yamaguch {
2387 1.101 yamaguch struct vioif_ctrlqueue *ctrlq = arg;
2388 1.101 yamaguch struct virtqueue *vq = ctrlq->ctrlq_vq;
2389 1.101 yamaguch struct virtio_softc *vsc = vq->vq_owner;
2390 1.101 yamaguch int r, slot;
2391 1.101 yamaguch
2392 1.101 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
2393 1.101 yamaguch return 0;
2394 1.101 yamaguch
2395 1.101 yamaguch r = virtio_dequeue(vsc, vq, &slot, NULL);
2396 1.101 yamaguch if (r == ENOENT)
2397 1.101 yamaguch return 0;
2398 1.101 yamaguch virtio_dequeue_commit(vsc, vq, slot);
2399 1.101 yamaguch
2400 1.101 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2401 1.101 yamaguch ctrlq->ctrlq_inuse = DONE;
2402 1.101 yamaguch cv_signal(&ctrlq->ctrlq_wait);
2403 1.101 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2404 1.101 yamaguch
2405 1.101 yamaguch return 1;
2406 1.101 yamaguch }
2407 1.101 yamaguch
2408 1.44 yamaguch static int
2409 1.44 yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
2410 1.44 yamaguch {
2411 1.44 yamaguch struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
2412 1.44 yamaguch struct vioif_ctrl_cmdspec specs[1];
2413 1.44 yamaguch int r;
2414 1.44 yamaguch
2415 1.44 yamaguch if (!sc->sc_has_ctrl)
2416 1.44 yamaguch return ENOTSUP;
2417 1.44 yamaguch
2418 1.44 yamaguch vioif_ctrl_acquire(sc);
2419 1.44 yamaguch
2420 1.44 yamaguch rx->onoff = onoff;
2421 1.44 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
2422 1.44 yamaguch specs[0].buf = rx;
2423 1.44 yamaguch specs[0].bufsize = sizeof(*rx);
2424 1.44 yamaguch
2425 1.44 yamaguch r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
2426 1.44 yamaguch specs, __arraycount(specs));
2427 1.3 christos
2428 1.44 yamaguch vioif_ctrl_release(sc);
2429 1.1 hannken return r;
2430 1.1 hannken }
2431 1.1 hannken
2432 1.1 hannken static int
2433 1.1 hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
2434 1.1 hannken {
2435 1.50 christos return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
2436 1.1 hannken }
2437 1.1 hannken
2438 1.1 hannken static int
2439 1.1 hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
2440 1.1 hannken {
2441 1.50 christos return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
2442 1.1 hannken }
2443 1.1 hannken
2444 1.1 hannken static int
2445 1.101 yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
2446 1.1 hannken {
2447 1.101 yamaguch struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
2448 1.101 yamaguch struct vioif_ctrl_cmdspec specs[1];
2449 1.44 yamaguch int r;
2450 1.44 yamaguch
2451 1.32 jdolecek if (!sc->sc_has_ctrl)
2452 1.1 hannken return ENOTSUP;
2453 1.1 hannken
2454 1.101 yamaguch if (nvq_pairs <= 1)
2455 1.101 yamaguch return EINVAL;
2456 1.101 yamaguch
2457 1.44 yamaguch vioif_ctrl_acquire(sc);
2458 1.1 hannken
2459 1.101 yamaguch mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs);
2460 1.101 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
2461 1.101 yamaguch specs[0].buf = mq;
2462 1.101 yamaguch specs[0].bufsize = sizeof(*mq);
2463 1.1 hannken
2464 1.44 yamaguch r = vioif_ctrl_send_command(sc,
2465 1.101 yamaguch VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
2466 1.101 yamaguch specs, __arraycount(specs));
2467 1.1 hannken
2468 1.44 yamaguch vioif_ctrl_release(sc);
2469 1.3 christos
2470 1.1 hannken return r;
2471 1.1 hannken }
2472 1.1 hannken
2473 1.46 yamaguch static int
2474 1.74 yamaguch vioif_set_mac_addr(struct vioif_softc *sc)
2475 1.74 yamaguch {
2476 1.74 yamaguch struct virtio_net_ctrl_mac_addr *ma =
2477 1.74 yamaguch sc->sc_ctrlq.ctrlq_mac_addr;
2478 1.74 yamaguch struct vioif_ctrl_cmdspec specs[1];
2479 1.74 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2480 1.74 yamaguch int nspecs = __arraycount(specs);
2481 1.78 yamaguch uint64_t features;
2482 1.74 yamaguch int r;
2483 1.78 yamaguch size_t i;
2484 1.74 yamaguch
2485 1.74 yamaguch if (!sc->sc_has_ctrl)
2486 1.74 yamaguch return ENOTSUP;
2487 1.74 yamaguch
2488 1.78 yamaguch if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac,
2489 1.78 yamaguch ETHER_ADDR_LEN) == 0) {
2490 1.78 yamaguch return 0;
2491 1.78 yamaguch }
2492 1.74 yamaguch
2493 1.78 yamaguch memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2494 1.74 yamaguch
2495 1.78 yamaguch features = virtio_features(sc->sc_virtio);
2496 1.78 yamaguch if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) {
2497 1.78 yamaguch vioif_ctrl_acquire(sc);
2498 1.78 yamaguch
2499 1.78 yamaguch memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN);
2500 1.78 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap;
2501 1.78 yamaguch specs[0].buf = ma;
2502 1.78 yamaguch specs[0].bufsize = sizeof(*ma);
2503 1.78 yamaguch
2504 1.78 yamaguch r = vioif_ctrl_send_command(sc,
2505 1.78 yamaguch VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET,
2506 1.78 yamaguch specs, nspecs);
2507 1.74 yamaguch
2508 1.78 yamaguch vioif_ctrl_release(sc);
2509 1.78 yamaguch } else {
2510 1.78 yamaguch for (i = 0; i < __arraycount(sc->sc_mac); i++) {
2511 1.78 yamaguch virtio_write_device_config_1(sc->sc_virtio,
2512 1.78 yamaguch VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
2513 1.78 yamaguch }
2514 1.78 yamaguch r = 0;
2515 1.78 yamaguch }
2516 1.74 yamaguch
2517 1.74 yamaguch return r;
2518 1.74 yamaguch }
2519 1.74 yamaguch
2520 1.74 yamaguch static int
2521 1.101 yamaguch vioif_set_rx_filter(struct vioif_softc *sc)
2522 1.1 hannken {
2523 1.101 yamaguch /* filter already set in ctrlq->ctrlq_mac_tbl */
2524 1.101 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2525 1.101 yamaguch struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
2526 1.101 yamaguch struct vioif_ctrl_cmdspec specs[2];
2527 1.101 yamaguch int nspecs = __arraycount(specs);
2528 1.101 yamaguch int r;
2529 1.1 hannken
2530 1.101 yamaguch mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
2531 1.101 yamaguch mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
2532 1.54 yamaguch
2533 1.101 yamaguch if (!sc->sc_has_ctrl)
2534 1.101 yamaguch return ENOTSUP;
2535 1.1 hannken
2536 1.101 yamaguch vioif_ctrl_acquire(sc);
2537 1.1 hannken
2538 1.101 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
2539 1.101 yamaguch specs[0].buf = mac_tbl_uc;
2540 1.101 yamaguch specs[0].bufsize = sizeof(*mac_tbl_uc)
2541 1.101 yamaguch + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries));
2542 1.1 hannken
2543 1.101 yamaguch specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
2544 1.101 yamaguch specs[1].buf = mac_tbl_mc;
2545 1.101 yamaguch specs[1].bufsize = sizeof(*mac_tbl_mc)
2546 1.101 yamaguch + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries));
2547 1.75 yamaguch
2548 1.101 yamaguch r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
2549 1.101 yamaguch if (r != 0)
2550 1.101 yamaguch goto out;
2551 1.75 yamaguch
2552 1.101 yamaguch r = vioif_ctrl_send_command(sc,
2553 1.101 yamaguch VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
2554 1.101 yamaguch specs, nspecs);
2555 1.75 yamaguch
2556 1.101 yamaguch vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
2557 1.75 yamaguch
2558 1.101 yamaguch out:
2559 1.101 yamaguch vioif_ctrl_release(sc);
2560 1.75 yamaguch
2561 1.101 yamaguch return r;
2562 1.75 yamaguch }
2563 1.75 yamaguch
2564 1.1 hannken /*
2565 1.1 hannken * If multicast filter small enough (<=MAXENTRIES) set rx filter
2566 1.1 hannken * If large multicast filter exist use ALLMULTI
2567 1.1 hannken * If setting rx filter fails fall back to ALLMULTI
2568 1.1 hannken */
2569 1.1 hannken static int
2570 1.1 hannken vioif_rx_filter(struct vioif_softc *sc)
2571 1.1 hannken {
2572 1.66 reinoud struct virtio_softc *vsc = sc->sc_virtio;
2573 1.48 msaitoh struct ethercom *ec = &sc->sc_ethercom;
2574 1.48 msaitoh struct ifnet *ifp = &ec->ec_if;
2575 1.1 hannken struct ether_multi *enm;
2576 1.1 hannken struct ether_multistep step;
2577 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2578 1.1 hannken int nentries;
2579 1.75 yamaguch bool allmulti = 0;
2580 1.1 hannken int r;
2581 1.1 hannken
2582 1.75 yamaguch if (!sc->sc_has_ctrl) {
2583 1.75 yamaguch goto set_ifflags;
2584 1.1 hannken }
2585 1.1 hannken
2586 1.74 yamaguch memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0],
2587 1.74 yamaguch CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2588 1.74 yamaguch
2589 1.75 yamaguch nentries = 0;
2590 1.75 yamaguch allmulti = false;
2591 1.75 yamaguch
2592 1.48 msaitoh ETHER_LOCK(ec);
2593 1.75 yamaguch for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
2594 1.75 yamaguch ETHER_NEXT_MULTI(step, enm)) {
2595 1.1 hannken if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
2596 1.75 yamaguch allmulti = true;
2597 1.75 yamaguch break;
2598 1.1 hannken }
2599 1.50 christos if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
2600 1.75 yamaguch allmulti = true;
2601 1.75 yamaguch break;
2602 1.1 hannken }
2603 1.75 yamaguch
2604 1.43 yamaguch memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
2605 1.50 christos enm->enm_addrlo, ETHER_ADDR_LEN);
2606 1.75 yamaguch nentries++;
2607 1.1 hannken }
2608 1.48 msaitoh ETHER_UNLOCK(ec);
2609 1.30 ozaki
2610 1.74 yamaguch r = vioif_set_mac_addr(sc);
2611 1.74 yamaguch if (r != 0) {
2612 1.74 yamaguch log(LOG_WARNING, "%s: couldn't set MAC address\n",
2613 1.74 yamaguch ifp->if_xname);
2614 1.74 yamaguch }
2615 1.74 yamaguch
2616 1.75 yamaguch if (!allmulti) {
2617 1.74 yamaguch ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1);
2618 1.66 reinoud ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries);
2619 1.1 hannken r = vioif_set_rx_filter(sc);
2620 1.1 hannken if (r != 0) {
2621 1.75 yamaguch allmulti = true; /* fallback */
2622 1.1 hannken }
2623 1.75 yamaguch }
2624 1.75 yamaguch
2625 1.75 yamaguch if (allmulti) {
2626 1.66 reinoud ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0);
2627 1.66 reinoud ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0);
2628 1.1 hannken r = vioif_set_rx_filter(sc);
2629 1.1 hannken if (r != 0) {
2630 1.75 yamaguch log(LOG_DEBUG, "%s: couldn't clear RX filter\n",
2631 1.75 yamaguch ifp->if_xname);
2632 1.75 yamaguch /* what to do on failure? */
2633 1.1 hannken }
2634 1.75 yamaguch
2635 1.75 yamaguch ifp->if_flags |= IFF_ALLMULTI;
2636 1.1 hannken }
2637 1.75 yamaguch
2638 1.75 yamaguch set_ifflags:
2639 1.75 yamaguch r = vioif_ifflags(sc);
2640 1.1 hannken
2641 1.1 hannken return r;
2642 1.1 hannken }
2643 1.1 hannken
2644 1.101 yamaguch /*
2645 1.101 yamaguch * VM configuration changes
2646 1.101 yamaguch */
2647 1.101 yamaguch static int
2648 1.101 yamaguch vioif_config_change(struct virtio_softc *vsc)
2649 1.101 yamaguch {
2650 1.101 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
2651 1.101 yamaguch
2652 1.101 yamaguch softint_schedule(sc->sc_cfg_softint);
2653 1.101 yamaguch return 0;
2654 1.101 yamaguch }
2655 1.101 yamaguch
2656 1.101 yamaguch static void
2657 1.101 yamaguch vioif_cfg_softint(void *arg)
2658 1.101 yamaguch {
2659 1.101 yamaguch struct vioif_softc *sc = arg;
2660 1.101 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2661 1.101 yamaguch
2662 1.101 yamaguch vioif_update_link_status(sc);
2663 1.101 yamaguch vioif_start(ifp);
2664 1.101 yamaguch }
2665 1.101 yamaguch
2666 1.82 knakahar static int
2667 1.82 knakahar vioif_get_link_status(struct vioif_softc *sc)
2668 1.34 ozaki {
2669 1.34 ozaki struct virtio_softc *vsc = sc->sc_virtio;
2670 1.34 ozaki uint16_t status;
2671 1.34 ozaki
2672 1.34 ozaki if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
2673 1.34 ozaki status = virtio_read_device_config_2(vsc,
2674 1.34 ozaki VIRTIO_NET_CONFIG_STATUS);
2675 1.34 ozaki else
2676 1.34 ozaki status = VIRTIO_NET_S_LINK_UP;
2677 1.34 ozaki
2678 1.82 knakahar if ((status & VIRTIO_NET_S_LINK_UP) != 0)
2679 1.82 knakahar return LINK_STATE_UP;
2680 1.82 knakahar
2681 1.82 knakahar return LINK_STATE_DOWN;
2682 1.34 ozaki }
2683 1.34 ozaki
2684 1.34 ozaki static void
2685 1.34 ozaki vioif_update_link_status(struct vioif_softc *sc)
2686 1.34 ozaki {
2687 1.34 ozaki struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2688 1.98 yamaguch struct vioif_netqueue *netq;
2689 1.98 yamaguch struct vioif_tx_context *txc;
2690 1.82 knakahar bool active;
2691 1.46 yamaguch int link, i;
2692 1.34 ozaki
2693 1.62 yamaguch mutex_enter(&sc->sc_lock);
2694 1.62 yamaguch
2695 1.82 knakahar link = vioif_get_link_status(sc);
2696 1.34 ozaki
2697 1.82 knakahar if (link == sc->sc_link_state)
2698 1.82 knakahar goto done;
2699 1.34 ozaki
2700 1.82 knakahar sc->sc_link_state = link;
2701 1.82 knakahar
2702 1.82 knakahar active = VIOIF_IS_LINK_ACTIVE(sc);
2703 1.82 knakahar for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
2704 1.98 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
2705 1.34 ozaki
2706 1.98 yamaguch mutex_enter(&netq->netq_lock);
2707 1.98 yamaguch txc = netq->netq_ctx;
2708 1.98 yamaguch txc->txc_link_active = active;
2709 1.98 yamaguch mutex_exit(&netq->netq_lock);
2710 1.34 ozaki }
2711 1.34 ozaki
2712 1.82 knakahar if_link_state_change(ifp, sc->sc_link_state);
2713 1.62 yamaguch
2714 1.82 knakahar done:
2715 1.62 yamaguch mutex_exit(&sc->sc_lock);
2716 1.34 ozaki }
2717 1.34 ozaki
2718 1.101 yamaguch static void
2719 1.101 yamaguch vioif_workq_work(struct work *wk, void *context)
2720 1.34 ozaki {
2721 1.101 yamaguch struct vioif_work *work;
2722 1.34 ozaki
2723 1.101 yamaguch work = container_of(wk, struct vioif_work, cookie);
2724 1.34 ozaki
2725 1.101 yamaguch atomic_store_relaxed(&work->added, 0);
2726 1.101 yamaguch work->func(work->arg);
2727 1.34 ozaki }
2728 1.34 ozaki
2729 1.55 yamaguch static struct workqueue *
2730 1.55 yamaguch vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
2731 1.55 yamaguch {
2732 1.55 yamaguch struct workqueue *wq;
2733 1.55 yamaguch int error;
2734 1.55 yamaguch
2735 1.55 yamaguch error = workqueue_create(&wq, name, vioif_workq_work, NULL,
2736 1.55 yamaguch prio, ipl, flags);
2737 1.55 yamaguch
2738 1.55 yamaguch if (error)
2739 1.55 yamaguch return NULL;
2740 1.55 yamaguch
2741 1.55 yamaguch return wq;
2742 1.55 yamaguch }
2743 1.55 yamaguch
2744 1.55 yamaguch static void
2745 1.55 yamaguch vioif_workq_destroy(struct workqueue *wq)
2746 1.55 yamaguch {
2747 1.55 yamaguch
2748 1.55 yamaguch workqueue_destroy(wq);
2749 1.55 yamaguch }
2750 1.55 yamaguch
2751 1.55 yamaguch static void
2752 1.55 yamaguch vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
2753 1.55 yamaguch {
2754 1.55 yamaguch
2755 1.55 yamaguch memset(work, 0, sizeof(*work));
2756 1.55 yamaguch work->func = func;
2757 1.55 yamaguch work->arg = arg;
2758 1.55 yamaguch }
2759 1.55 yamaguch
2760 1.55 yamaguch static void
2761 1.55 yamaguch vioif_work_add(struct workqueue *wq, struct vioif_work *work)
2762 1.55 yamaguch {
2763 1.55 yamaguch
2764 1.55 yamaguch if (atomic_load_relaxed(&work->added) != 0)
2765 1.55 yamaguch return;
2766 1.55 yamaguch
2767 1.55 yamaguch atomic_store_relaxed(&work->added, 1);
2768 1.55 yamaguch kpreempt_disable();
2769 1.55 yamaguch workqueue_enqueue(wq, &work->cookie, NULL);
2770 1.55 yamaguch kpreempt_enable();
2771 1.55 yamaguch }
2772 1.55 yamaguch
2773 1.55 yamaguch static void
2774 1.55 yamaguch vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
2775 1.55 yamaguch {
2776 1.55 yamaguch
2777 1.55 yamaguch workqueue_wait(wq, &work->cookie);
2778 1.55 yamaguch }
2779 1.55 yamaguch
2780 1.26 pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
2781 1.48 msaitoh
2782 1.26 pgoyette #ifdef _MODULE
2783 1.26 pgoyette #include "ioconf.c"
2784 1.26 pgoyette #endif
2785 1.48 msaitoh
2786 1.48 msaitoh static int
2787 1.26 pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
2788 1.26 pgoyette {
2789 1.26 pgoyette int error = 0;
2790 1.48 msaitoh
2791 1.26 pgoyette #ifdef _MODULE
2792 1.26 pgoyette switch (cmd) {
2793 1.26 pgoyette case MODULE_CMD_INIT:
2794 1.48 msaitoh error = config_init_component(cfdriver_ioconf_if_vioif,
2795 1.48 msaitoh cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2796 1.26 pgoyette break;
2797 1.26 pgoyette case MODULE_CMD_FINI:
2798 1.26 pgoyette error = config_fini_component(cfdriver_ioconf_if_vioif,
2799 1.26 pgoyette cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2800 1.26 pgoyette break;
2801 1.26 pgoyette default:
2802 1.26 pgoyette error = ENOTTY;
2803 1.48 msaitoh break;
2804 1.26 pgoyette }
2805 1.26 pgoyette #endif
2806 1.48 msaitoh
2807 1.26 pgoyette return error;
2808 1.26 pgoyette }
2809