if_vioif.c revision 1.98 1 1.98 yamaguch /* $NetBSD: if_vioif.c,v 1.98 2023/03/23 02:42:49 yamaguchi Exp $ */
2 1.1 hannken
3 1.1 hannken /*
4 1.66 reinoud * Copyright (c) 2020 The NetBSD Foundation, Inc.
5 1.1 hannken * Copyright (c) 2010 Minoura Makoto.
6 1.1 hannken * All rights reserved.
7 1.1 hannken *
8 1.1 hannken * Redistribution and use in source and binary forms, with or without
9 1.1 hannken * modification, are permitted provided that the following conditions
10 1.1 hannken * are met:
11 1.1 hannken * 1. Redistributions of source code must retain the above copyright
12 1.1 hannken * notice, this list of conditions and the following disclaimer.
13 1.1 hannken * 2. Redistributions in binary form must reproduce the above copyright
14 1.1 hannken * notice, this list of conditions and the following disclaimer in the
15 1.1 hannken * documentation and/or other materials provided with the distribution.
16 1.1 hannken *
17 1.1 hannken * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 1.1 hannken * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 1.1 hannken * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 1.1 hannken * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 1.1 hannken * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 1.1 hannken * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 1.1 hannken * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 1.1 hannken * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 1.1 hannken * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 1.1 hannken * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 1.1 hannken */
28 1.1 hannken
29 1.1 hannken #include <sys/cdefs.h>
30 1.98 yamaguch __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.98 2023/03/23 02:42:49 yamaguchi Exp $");
31 1.15 ozaki
32 1.15 ozaki #ifdef _KERNEL_OPT
33 1.15 ozaki #include "opt_net_mpsafe.h"
34 1.15 ozaki #endif
35 1.1 hannken
36 1.1 hannken #include <sys/param.h>
37 1.1 hannken #include <sys/systm.h>
38 1.1 hannken #include <sys/kernel.h>
39 1.55 yamaguch #include <sys/atomic.h>
40 1.1 hannken #include <sys/bus.h>
41 1.1 hannken #include <sys/condvar.h>
42 1.1 hannken #include <sys/device.h>
43 1.63 yamaguch #include <sys/evcnt.h>
44 1.1 hannken #include <sys/intr.h>
45 1.1 hannken #include <sys/kmem.h>
46 1.1 hannken #include <sys/mbuf.h>
47 1.1 hannken #include <sys/mutex.h>
48 1.1 hannken #include <sys/sockio.h>
49 1.71 yamaguch #include <sys/syslog.h>
50 1.12 ozaki #include <sys/cpu.h>
51 1.26 pgoyette #include <sys/module.h>
52 1.46 yamaguch #include <sys/pcq.h>
53 1.55 yamaguch #include <sys/workqueue.h>
54 1.85 yamaguch #include <sys/xcall.h>
55 1.1 hannken
56 1.1 hannken #include <dev/pci/virtioreg.h>
57 1.1 hannken #include <dev/pci/virtiovar.h>
58 1.1 hannken
59 1.1 hannken #include <net/if.h>
60 1.74 yamaguch #include <net/if_dl.h>
61 1.1 hannken #include <net/if_media.h>
62 1.1 hannken #include <net/if_ether.h>
63 1.1 hannken
64 1.1 hannken #include <net/bpf.h>
65 1.1 hannken
66 1.26 pgoyette #include "ioconf.h"
67 1.1 hannken
68 1.7 ozaki #ifdef NET_MPSAFE
69 1.7 ozaki #define VIOIF_MPSAFE 1
70 1.46 yamaguch #define VIOIF_MULTIQ 1
71 1.7 ozaki #endif
72 1.7 ozaki
73 1.1 hannken /*
74 1.1 hannken * if_vioifreg.h:
75 1.1 hannken */
76 1.1 hannken /* Configuration registers */
77 1.66 reinoud #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
78 1.66 reinoud #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
79 1.66 reinoud #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
80 1.66 reinoud #define VIRTIO_NET_CONFIG_MTU 10 /* 16bit */
81 1.1 hannken
82 1.1 hannken /* Feature bits */
83 1.46 yamaguch #define VIRTIO_NET_F_CSUM __BIT(0)
84 1.46 yamaguch #define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
85 1.46 yamaguch #define VIRTIO_NET_F_MAC __BIT(5)
86 1.46 yamaguch #define VIRTIO_NET_F_GSO __BIT(6)
87 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
88 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
89 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ECN __BIT(9)
90 1.46 yamaguch #define VIRTIO_NET_F_GUEST_UFO __BIT(10)
91 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
92 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
93 1.46 yamaguch #define VIRTIO_NET_F_HOST_ECN __BIT(13)
94 1.46 yamaguch #define VIRTIO_NET_F_HOST_UFO __BIT(14)
95 1.46 yamaguch #define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
96 1.46 yamaguch #define VIRTIO_NET_F_STATUS __BIT(16)
97 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VQ __BIT(17)
98 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX __BIT(18)
99 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
100 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
101 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
102 1.46 yamaguch #define VIRTIO_NET_F_MQ __BIT(22)
103 1.78 yamaguch #define VIRTIO_NET_F_CTRL_MAC_ADDR __BIT(23)
104 1.1 hannken
105 1.79 uwe #define VIRTIO_NET_FLAG_BITS \
106 1.79 uwe VIRTIO_COMMON_FLAG_BITS \
107 1.79 uwe "b\x17" "CTRL_MAC\0" \
108 1.79 uwe "b\x16" "MQ\0" \
109 1.79 uwe "b\x15" "GUEST_ANNOUNCE\0" \
110 1.79 uwe "b\x14" "CTRL_RX_EXTRA\0" \
111 1.79 uwe "b\x13" "CTRL_VLAN\0" \
112 1.79 uwe "b\x12" "CTRL_RX\0" \
113 1.79 uwe "b\x11" "CTRL_VQ\0" \
114 1.79 uwe "b\x10" "STATUS\0" \
115 1.79 uwe "b\x0f" "MRG_RXBUF\0" \
116 1.79 uwe "b\x0e" "HOST_UFO\0" \
117 1.79 uwe "b\x0d" "HOST_ECN\0" \
118 1.79 uwe "b\x0c" "HOST_TSO6\0" \
119 1.79 uwe "b\x0b" "HOST_TSO4\0" \
120 1.79 uwe "b\x0a" "GUEST_UFO\0" \
121 1.79 uwe "b\x09" "GUEST_ECN\0" \
122 1.79 uwe "b\x08" "GUEST_TSO6\0" \
123 1.79 uwe "b\x07" "GUEST_TSO4\0" \
124 1.79 uwe "b\x06" "GSO\0" \
125 1.79 uwe "b\x05" "MAC\0" \
126 1.79 uwe "b\x01" "GUEST_CSUM\0" \
127 1.79 uwe "b\x00" "CSUM\0"
128 1.18 christos
129 1.1 hannken /* Status */
130 1.1 hannken #define VIRTIO_NET_S_LINK_UP 1
131 1.1 hannken
132 1.1 hannken /* Packet header structure */
133 1.1 hannken struct virtio_net_hdr {
134 1.1 hannken uint8_t flags;
135 1.1 hannken uint8_t gso_type;
136 1.1 hannken uint16_t hdr_len;
137 1.1 hannken uint16_t gso_size;
138 1.1 hannken uint16_t csum_start;
139 1.1 hannken uint16_t csum_offset;
140 1.66 reinoud
141 1.66 reinoud uint16_t num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */
142 1.1 hannken } __packed;
143 1.1 hannken
144 1.1 hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
145 1.1 hannken #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
146 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
147 1.1 hannken #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
148 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
149 1.1 hannken #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
150 1.1 hannken
151 1.1 hannken #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN)
152 1.1 hannken
153 1.1 hannken /* Control virtqueue */
154 1.1 hannken struct virtio_net_ctrl_cmd {
155 1.1 hannken uint8_t class;
156 1.1 hannken uint8_t command;
157 1.1 hannken } __packed;
158 1.1 hannken #define VIRTIO_NET_CTRL_RX 0
159 1.1 hannken # define VIRTIO_NET_CTRL_RX_PROMISC 0
160 1.1 hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI 1
161 1.1 hannken
162 1.1 hannken #define VIRTIO_NET_CTRL_MAC 1
163 1.1 hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
164 1.74 yamaguch # define VIRTIO_NET_CTRL_MAC_ADDR_SET 1
165 1.1 hannken
166 1.1 hannken #define VIRTIO_NET_CTRL_VLAN 2
167 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_ADD 0
168 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_DEL 1
169 1.1 hannken
170 1.46 yamaguch #define VIRTIO_NET_CTRL_MQ 4
171 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
172 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
173 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
174 1.46 yamaguch
175 1.1 hannken struct virtio_net_ctrl_status {
176 1.1 hannken uint8_t ack;
177 1.1 hannken } __packed;
178 1.1 hannken #define VIRTIO_NET_OK 0
179 1.1 hannken #define VIRTIO_NET_ERR 1
180 1.1 hannken
181 1.1 hannken struct virtio_net_ctrl_rx {
182 1.1 hannken uint8_t onoff;
183 1.1 hannken } __packed;
184 1.1 hannken
185 1.1 hannken struct virtio_net_ctrl_mac_tbl {
186 1.1 hannken uint32_t nentries;
187 1.1 hannken uint8_t macs[][ETHER_ADDR_LEN];
188 1.1 hannken } __packed;
189 1.1 hannken
190 1.74 yamaguch struct virtio_net_ctrl_mac_addr {
191 1.74 yamaguch uint8_t mac[ETHER_ADDR_LEN];
192 1.74 yamaguch } __packed;
193 1.74 yamaguch
194 1.1 hannken struct virtio_net_ctrl_vlan {
195 1.1 hannken uint16_t id;
196 1.1 hannken } __packed;
197 1.1 hannken
198 1.46 yamaguch struct virtio_net_ctrl_mq {
199 1.46 yamaguch uint16_t virtqueue_pairs;
200 1.46 yamaguch } __packed;
201 1.46 yamaguch
202 1.1 hannken /*
203 1.1 hannken * if_vioifvar.h:
204 1.1 hannken */
205 1.43 yamaguch
206 1.43 yamaguch /*
207 1.43 yamaguch * Locking notes:
208 1.98 yamaguch * + a field in vioif_netueue is protected by netq_lock (a spin mutex)
209 1.43 yamaguch * - more than one lock cannot be held at onece
210 1.98 yamaguch * + a field in vioif_tx_context and vioif_rx_context is also protected
211 1.98 yamaguch * by netq_lock.
212 1.43 yamaguch * + ctrlq_inuse is protected by ctrlq_wait_lock.
213 1.43 yamaguch * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
214 1.98 yamaguch * - netq_lock cannot be held along with ctrlq_wait_lock
215 1.62 yamaguch * + fields in vioif_softc except queues are protected by
216 1.62 yamaguch * sc->sc_lock(an adaptive mutex)
217 1.62 yamaguch * - the lock is held before acquisition of other locks
218 1.43 yamaguch */
219 1.43 yamaguch
220 1.66 reinoud struct vioif_ctrl_cmdspec {
221 1.66 reinoud bus_dmamap_t dmamap;
222 1.66 reinoud void *buf;
223 1.66 reinoud bus_size_t bufsize;
224 1.66 reinoud };
225 1.66 reinoud
226 1.55 yamaguch struct vioif_work {
227 1.55 yamaguch struct work cookie;
228 1.55 yamaguch void (*func)(void *);
229 1.55 yamaguch void *arg;
230 1.55 yamaguch unsigned int added;
231 1.55 yamaguch };
232 1.55 yamaguch
233 1.95 yamaguch struct vioif_net_map {
234 1.95 yamaguch struct virtio_net_hdr *vnm_hdr;
235 1.95 yamaguch bus_dmamap_t vnm_hdr_map;
236 1.95 yamaguch struct mbuf *vnm_mbuf;
237 1.95 yamaguch bus_dmamap_t vnm_mbuf_map;
238 1.95 yamaguch };
239 1.95 yamaguch
240 1.98 yamaguch #define VIOIF_NETQ_RX 0
241 1.98 yamaguch #define VIOIF_NETQ_TX 1
242 1.98 yamaguch #define VIOIF_NETQ_IDX 2
243 1.98 yamaguch #define VIOIF_NETQ_DIR(n) ((n) % VIOIF_NETQ_IDX)
244 1.98 yamaguch #define VIOIF_NETQ_PAIRIDX(n) ((n) / VIOIF_NETQ_IDX)
245 1.98 yamaguch #define VIOIF_NETQ_RXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX)
246 1.98 yamaguch #define VIOIF_NETQ_TXQID(n) ((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX)
247 1.98 yamaguch
248 1.98 yamaguch struct vioif_netqueue {
249 1.98 yamaguch kmutex_t netq_lock;
250 1.98 yamaguch struct virtqueue *netq_vq;
251 1.98 yamaguch bool netq_stopping;
252 1.98 yamaguch bool netq_running_handle;
253 1.98 yamaguch void *netq_maps_kva;
254 1.98 yamaguch struct vioif_net_map *netq_maps;
255 1.98 yamaguch
256 1.98 yamaguch void *netq_softint;
257 1.98 yamaguch struct vioif_work netq_work;
258 1.98 yamaguch bool netq_workqueue;
259 1.98 yamaguch
260 1.98 yamaguch char netq_evgroup[32];
261 1.98 yamaguch struct evcnt netq_mbuf_load_failed;
262 1.98 yamaguch struct evcnt netq_enqueue_reserve_failed;
263 1.43 yamaguch
264 1.98 yamaguch void *netq_ctx;
265 1.43 yamaguch };
266 1.43 yamaguch
267 1.98 yamaguch struct vioif_tx_context {
268 1.98 yamaguch bool txc_link_active;
269 1.98 yamaguch pcq_t *txc_intrq;
270 1.98 yamaguch void *txc_deferred_transmit;
271 1.43 yamaguch
272 1.98 yamaguch struct evcnt txc_defrag_failed;
273 1.98 yamaguch };
274 1.43 yamaguch
275 1.98 yamaguch struct vioif_rx_context {
276 1.98 yamaguch struct evcnt rxc_mbuf_enobufs;
277 1.43 yamaguch };
278 1.43 yamaguch struct vioif_ctrlqueue {
279 1.43 yamaguch struct virtqueue *ctrlq_vq;
280 1.43 yamaguch enum {
281 1.43 yamaguch FREE, INUSE, DONE
282 1.43 yamaguch } ctrlq_inuse;
283 1.43 yamaguch kcondvar_t ctrlq_wait;
284 1.43 yamaguch kmutex_t ctrlq_wait_lock;
285 1.44 yamaguch struct lwp *ctrlq_owner;
286 1.43 yamaguch
287 1.43 yamaguch struct virtio_net_ctrl_cmd *ctrlq_cmd;
288 1.43 yamaguch struct virtio_net_ctrl_status *ctrlq_status;
289 1.43 yamaguch struct virtio_net_ctrl_rx *ctrlq_rx;
290 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
291 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
292 1.74 yamaguch struct virtio_net_ctrl_mac_addr *ctrlq_mac_addr;
293 1.46 yamaguch struct virtio_net_ctrl_mq *ctrlq_mq;
294 1.43 yamaguch
295 1.43 yamaguch bus_dmamap_t ctrlq_cmd_dmamap;
296 1.43 yamaguch bus_dmamap_t ctrlq_status_dmamap;
297 1.43 yamaguch bus_dmamap_t ctrlq_rx_dmamap;
298 1.43 yamaguch bus_dmamap_t ctrlq_tbl_uc_dmamap;
299 1.43 yamaguch bus_dmamap_t ctrlq_tbl_mc_dmamap;
300 1.74 yamaguch bus_dmamap_t ctrlq_mac_addr_dmamap;
301 1.46 yamaguch bus_dmamap_t ctrlq_mq_dmamap;
302 1.63 yamaguch
303 1.63 yamaguch struct evcnt ctrlq_cmd_load_failed;
304 1.63 yamaguch struct evcnt ctrlq_cmd_failed;
305 1.43 yamaguch };
306 1.43 yamaguch
307 1.1 hannken struct vioif_softc {
308 1.1 hannken device_t sc_dev;
309 1.62 yamaguch kmutex_t sc_lock;
310 1.55 yamaguch struct sysctllog *sc_sysctllog;
311 1.1 hannken
312 1.1 hannken struct virtio_softc *sc_virtio;
313 1.46 yamaguch struct virtqueue *sc_vqs;
314 1.66 reinoud u_int sc_hdr_size;
315 1.46 yamaguch
316 1.46 yamaguch int sc_max_nvq_pairs;
317 1.46 yamaguch int sc_req_nvq_pairs;
318 1.46 yamaguch int sc_act_nvq_pairs;
319 1.1 hannken
320 1.1 hannken uint8_t sc_mac[ETHER_ADDR_LEN];
321 1.1 hannken struct ethercom sc_ethercom;
322 1.82 knakahar int sc_link_state;
323 1.1 hannken
324 1.98 yamaguch struct vioif_netqueue *sc_netqs;
325 1.43 yamaguch
326 1.43 yamaguch bool sc_has_ctrl;
327 1.43 yamaguch struct vioif_ctrlqueue sc_ctrlq;
328 1.43 yamaguch
329 1.1 hannken bus_dma_segment_t sc_hdr_segs[1];
330 1.43 yamaguch void *sc_dmamem;
331 1.43 yamaguch void *sc_kmem;
332 1.1 hannken
333 1.34 ozaki void *sc_ctl_softint;
334 1.55 yamaguch
335 1.55 yamaguch struct workqueue *sc_txrx_workqueue;
336 1.55 yamaguch bool sc_txrx_workqueue_sysctl;
337 1.55 yamaguch u_int sc_tx_intr_process_limit;
338 1.55 yamaguch u_int sc_tx_process_limit;
339 1.55 yamaguch u_int sc_rx_intr_process_limit;
340 1.55 yamaguch u_int sc_rx_process_limit;
341 1.1 hannken };
342 1.1 hannken #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
343 1.1 hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
344 1.1 hannken
345 1.55 yamaguch #define VIOIF_TX_INTR_PROCESS_LIMIT 256
346 1.55 yamaguch #define VIOIF_TX_PROCESS_LIMIT 256
347 1.55 yamaguch #define VIOIF_RX_INTR_PROCESS_LIMIT 0U
348 1.55 yamaguch #define VIOIF_RX_PROCESS_LIMIT 256
349 1.55 yamaguch
350 1.55 yamaguch #define VIOIF_WORKQUEUE_PRI PRI_SOFTNET
351 1.82 knakahar #define VIOIF_IS_LINK_ACTIVE(_sc) ((_sc)->sc_link_state == LINK_STATE_UP ? \
352 1.82 knakahar true : false)
353 1.55 yamaguch
354 1.1 hannken /* cfattach interface functions */
355 1.1 hannken static int vioif_match(device_t, cfdata_t, void *);
356 1.1 hannken static void vioif_attach(device_t, device_t, void *);
357 1.55 yamaguch static int vioif_finalize_teardown(device_t);
358 1.1 hannken
359 1.1 hannken /* ifnet interface functions */
360 1.1 hannken static int vioif_init(struct ifnet *);
361 1.1 hannken static void vioif_stop(struct ifnet *, int);
362 1.1 hannken static void vioif_start(struct ifnet *);
363 1.98 yamaguch static void vioif_start_locked(struct ifnet *, struct vioif_netqueue *);
364 1.46 yamaguch static int vioif_transmit(struct ifnet *, struct mbuf *);
365 1.98 yamaguch static void vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *);
366 1.1 hannken static int vioif_ioctl(struct ifnet *, u_long, void *);
367 1.1 hannken static void vioif_watchdog(struct ifnet *);
368 1.75 yamaguch static int vioif_ifflags_cb(struct ethercom *);
369 1.1 hannken
370 1.98 yamaguch /* tx & rx */
371 1.98 yamaguch static void vioif_net_sched_handle(struct vioif_softc *,
372 1.98 yamaguch struct vioif_netqueue *);
373 1.98 yamaguch
374 1.1 hannken /* rx */
375 1.66 reinoud static void vioif_populate_rx_mbufs_locked(struct vioif_softc *,
376 1.98 yamaguch struct vioif_netqueue *);
377 1.93 yamaguch static void vioif_rx_queue_clear(struct vioif_softc *, struct virtio_softc *,
378 1.98 yamaguch struct vioif_netqueue *);
379 1.55 yamaguch static bool vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *,
380 1.98 yamaguch struct vioif_netqueue *, u_int, size_t *);
381 1.54 yamaguch static int vioif_rx_intr(void *);
382 1.55 yamaguch static void vioif_rx_handle(void *);
383 1.1 hannken
384 1.1 hannken /* tx */
385 1.54 yamaguch static int vioif_tx_intr(void *);
386 1.55 yamaguch static void vioif_tx_handle(void *);
387 1.93 yamaguch static void vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *,
388 1.98 yamaguch struct vioif_netqueue *);
389 1.55 yamaguch static bool vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *,
390 1.98 yamaguch struct vioif_netqueue *, u_int);
391 1.46 yamaguch static void vioif_deferred_transmit(void *);
392 1.1 hannken
393 1.55 yamaguch /* workqueue */
394 1.55 yamaguch static struct workqueue*
395 1.55 yamaguch vioif_workq_create(const char *, pri_t, int, int);
396 1.55 yamaguch static void vioif_workq_destroy(struct workqueue *);
397 1.55 yamaguch static void vioif_workq_work(struct work *, void *);
398 1.55 yamaguch static void vioif_work_set(struct vioif_work *, void(*)(void *), void *);
399 1.55 yamaguch static void vioif_work_add(struct workqueue *, struct vioif_work *);
400 1.55 yamaguch static void vioif_work_wait(struct workqueue *, struct vioif_work *);
401 1.55 yamaguch
402 1.1 hannken /* other control */
403 1.82 knakahar static int vioif_get_link_status(struct vioif_softc *);
404 1.34 ozaki static void vioif_update_link_status(struct vioif_softc *);
405 1.1 hannken static int vioif_ctrl_rx(struct vioif_softc *, int, bool);
406 1.1 hannken static int vioif_set_promisc(struct vioif_softc *, bool);
407 1.1 hannken static int vioif_set_allmulti(struct vioif_softc *, bool);
408 1.1 hannken static int vioif_set_rx_filter(struct vioif_softc *);
409 1.1 hannken static int vioif_rx_filter(struct vioif_softc *);
410 1.74 yamaguch static int vioif_set_mac_addr(struct vioif_softc *);
411 1.54 yamaguch static int vioif_ctrl_intr(void *);
412 1.34 ozaki static int vioif_config_change(struct virtio_softc *);
413 1.34 ozaki static void vioif_ctl_softint(void *);
414 1.46 yamaguch static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
415 1.46 yamaguch static void vioif_enable_interrupt_vqpairs(struct vioif_softc *);
416 1.46 yamaguch static void vioif_disable_interrupt_vqpairs(struct vioif_softc *);
417 1.55 yamaguch static int vioif_setup_sysctl(struct vioif_softc *);
418 1.63 yamaguch static void vioif_setup_stats(struct vioif_softc *);
419 1.75 yamaguch static int vioif_ifflags(struct vioif_softc *);
420 1.85 yamaguch static void vioif_intr_barrier(void);
421 1.1 hannken
422 1.1 hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
423 1.1 hannken vioif_match, vioif_attach, NULL, NULL);
424 1.1 hannken
425 1.1 hannken static int
426 1.1 hannken vioif_match(device_t parent, cfdata_t match, void *aux)
427 1.1 hannken {
428 1.32 jdolecek struct virtio_attach_args *va = aux;
429 1.1 hannken
430 1.66 reinoud if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK)
431 1.1 hannken return 1;
432 1.1 hannken
433 1.1 hannken return 0;
434 1.1 hannken }
435 1.1 hannken
436 1.58 yamaguch static int
437 1.58 yamaguch vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
438 1.58 yamaguch bus_size_t size, int nsegs, const char *usage)
439 1.58 yamaguch {
440 1.58 yamaguch int r;
441 1.58 yamaguch
442 1.58 yamaguch r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
443 1.58 yamaguch nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
444 1.58 yamaguch
445 1.58 yamaguch if (r != 0) {
446 1.58 yamaguch aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
447 1.58 yamaguch "error code %d\n", usage, r);
448 1.58 yamaguch }
449 1.58 yamaguch
450 1.58 yamaguch return r;
451 1.58 yamaguch }
452 1.58 yamaguch
453 1.58 yamaguch static void
454 1.58 yamaguch vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
455 1.58 yamaguch {
456 1.58 yamaguch
457 1.58 yamaguch if (*map) {
458 1.58 yamaguch bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
459 1.58 yamaguch *map = NULL;
460 1.58 yamaguch }
461 1.58 yamaguch }
462 1.58 yamaguch
463 1.58 yamaguch static int
464 1.58 yamaguch vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
465 1.58 yamaguch void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
466 1.58 yamaguch {
467 1.58 yamaguch int r;
468 1.58 yamaguch
469 1.58 yamaguch r = vioif_dmamap_create(sc, map, size, nsegs, usage);
470 1.58 yamaguch if (r != 0)
471 1.58 yamaguch return 1;
472 1.58 yamaguch
473 1.58 yamaguch r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
474 1.58 yamaguch size, NULL, rw | BUS_DMA_NOWAIT);
475 1.58 yamaguch if (r != 0) {
476 1.58 yamaguch vioif_dmamap_destroy(sc, map);
477 1.58 yamaguch aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
478 1.58 yamaguch "error code %d\n", usage, r);
479 1.58 yamaguch }
480 1.58 yamaguch
481 1.58 yamaguch return r;
482 1.58 yamaguch }
483 1.58 yamaguch
484 1.58 yamaguch static void *
485 1.58 yamaguch vioif_assign_mem(intptr_t *p, size_t size)
486 1.58 yamaguch {
487 1.58 yamaguch intptr_t rv;
488 1.58 yamaguch
489 1.58 yamaguch rv = *p;
490 1.58 yamaguch *p += size;
491 1.58 yamaguch
492 1.58 yamaguch return (void *)rv;
493 1.58 yamaguch }
494 1.58 yamaguch
495 1.51 chs static void
496 1.46 yamaguch vioif_alloc_queues(struct vioif_softc *sc)
497 1.46 yamaguch {
498 1.46 yamaguch int nvq_pairs = sc->sc_max_nvq_pairs;
499 1.98 yamaguch size_t nvqs, netq_num;
500 1.46 yamaguch
501 1.46 yamaguch KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
502 1.46 yamaguch
503 1.98 yamaguch nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
504 1.46 yamaguch if (sc->sc_has_ctrl)
505 1.46 yamaguch nvqs++;
506 1.46 yamaguch
507 1.51 chs sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
508 1.98 yamaguch sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * netq_num,
509 1.98 yamaguch KM_SLEEP);
510 1.46 yamaguch }
511 1.46 yamaguch
512 1.46 yamaguch static void
513 1.46 yamaguch vioif_free_queues(struct vioif_softc *sc)
514 1.46 yamaguch {
515 1.98 yamaguch size_t nvqs, netq_num;
516 1.46 yamaguch
517 1.98 yamaguch nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
518 1.46 yamaguch if (sc->sc_ctrlq.ctrlq_vq)
519 1.46 yamaguch nvqs++;
520 1.46 yamaguch
521 1.98 yamaguch kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num);
522 1.98 yamaguch kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
523 1.98 yamaguch sc->sc_netqs = NULL;
524 1.98 yamaguch sc->sc_vqs = NULL;
525 1.98 yamaguch }
526 1.98 yamaguch
527 1.98 yamaguch static int
528 1.98 yamaguch vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc,
529 1.98 yamaguch size_t qid, u_int softint_flags)
530 1.98 yamaguch {
531 1.98 yamaguch static const struct {
532 1.98 yamaguch const char *dirname;
533 1.98 yamaguch int segsize;
534 1.98 yamaguch int nsegs;
535 1.98 yamaguch int (*intrhand)(void *);
536 1.98 yamaguch void (*sihand)(void *);
537 1.98 yamaguch } params[VIOIF_NETQ_IDX] = {
538 1.98 yamaguch [VIOIF_NETQ_RX] = {
539 1.98 yamaguch .dirname = "rx",
540 1.98 yamaguch .segsize = MCLBYTES,
541 1.98 yamaguch .nsegs = 2,
542 1.98 yamaguch .intrhand = vioif_rx_intr,
543 1.98 yamaguch .sihand = vioif_rx_handle,
544 1.98 yamaguch },
545 1.98 yamaguch [VIOIF_NETQ_TX] = {
546 1.98 yamaguch .dirname = "tx",
547 1.98 yamaguch .segsize = ETHER_MAX_LEN - ETHER_HDR_LEN,
548 1.98 yamaguch .nsegs = 2,
549 1.98 yamaguch .intrhand = vioif_tx_intr,
550 1.98 yamaguch .sihand = vioif_tx_handle,
551 1.98 yamaguch }
552 1.98 yamaguch };
553 1.98 yamaguch
554 1.98 yamaguch struct virtqueue *vq;
555 1.98 yamaguch struct vioif_netqueue *netq;
556 1.98 yamaguch struct vioif_tx_context *txc;
557 1.98 yamaguch struct vioif_rx_context *rxc;
558 1.98 yamaguch char qname[32];
559 1.98 yamaguch int r, dir;
560 1.98 yamaguch
561 1.98 yamaguch txc = NULL;
562 1.98 yamaguch rxc = NULL;
563 1.98 yamaguch netq = &sc->sc_netqs[qid];
564 1.98 yamaguch vq = &sc->sc_vqs[qid];
565 1.98 yamaguch dir = VIOIF_NETQ_DIR(qid);
566 1.98 yamaguch
567 1.98 yamaguch netq->netq_vq = &sc->sc_vqs[qid];
568 1.98 yamaguch netq->netq_stopping = false;
569 1.98 yamaguch netq->netq_running_handle = false;
570 1.98 yamaguch
571 1.98 yamaguch snprintf(qname, sizeof(qname), "%s%zu",
572 1.98 yamaguch params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid));
573 1.98 yamaguch snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup),
574 1.98 yamaguch "%s-%s", device_xname(sc->sc_dev), qname);
575 1.98 yamaguch
576 1.98 yamaguch mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET);
577 1.98 yamaguch r = virtio_alloc_vq(vsc, vq, qid,
578 1.98 yamaguch params[dir].segsize + sc->sc_hdr_size,
579 1.98 yamaguch params[dir].nsegs, qname);
580 1.98 yamaguch if (r != 0)
581 1.98 yamaguch goto err;
582 1.98 yamaguch netq->netq_vq = vq;
583 1.98 yamaguch
584 1.98 yamaguch netq->netq_vq->vq_intrhand = params[dir].intrhand;
585 1.98 yamaguch netq->netq_vq->vq_intrhand_arg = netq;
586 1.98 yamaguch netq->netq_softint = softint_establish(softint_flags,
587 1.98 yamaguch params[dir].sihand, netq);
588 1.98 yamaguch if (netq->netq_softint == NULL) {
589 1.98 yamaguch aprint_error_dev(sc->sc_dev,
590 1.98 yamaguch "couldn't establish %s softint\n",
591 1.98 yamaguch params[dir].dirname);
592 1.98 yamaguch goto err;
593 1.46 yamaguch }
594 1.98 yamaguch vioif_work_set(&netq->netq_work, params[dir].sihand, netq);
595 1.46 yamaguch
596 1.98 yamaguch switch (dir) {
597 1.98 yamaguch case VIOIF_NETQ_RX:
598 1.98 yamaguch rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP);
599 1.98 yamaguch netq->netq_ctx = rxc;
600 1.98 yamaguch /* nothing to do */
601 1.98 yamaguch break;
602 1.98 yamaguch case VIOIF_NETQ_TX:
603 1.98 yamaguch txc = kmem_zalloc(sizeof(*txc), KM_SLEEP);
604 1.98 yamaguch netq->netq_ctx = (void *)txc;
605 1.98 yamaguch txc->txc_deferred_transmit = softint_establish(softint_flags,
606 1.98 yamaguch vioif_deferred_transmit, netq);
607 1.98 yamaguch if (txc->txc_deferred_transmit == NULL) {
608 1.98 yamaguch aprint_error_dev(sc->sc_dev,
609 1.98 yamaguch "couldn't establish softint for "
610 1.98 yamaguch "tx deferred transmit\n");
611 1.98 yamaguch goto err;
612 1.98 yamaguch }
613 1.98 yamaguch txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc);
614 1.98 yamaguch txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP);
615 1.98 yamaguch break;
616 1.46 yamaguch }
617 1.46 yamaguch
618 1.98 yamaguch return 0;
619 1.98 yamaguch
620 1.98 yamaguch err:
621 1.98 yamaguch netq->netq_ctx = NULL;
622 1.98 yamaguch
623 1.98 yamaguch if (rxc != NULL) {
624 1.98 yamaguch kmem_free(rxc, sizeof(*rxc));
625 1.98 yamaguch }
626 1.98 yamaguch
627 1.98 yamaguch if (txc != NULL) {
628 1.98 yamaguch if (txc->txc_deferred_transmit != NULL)
629 1.98 yamaguch softint_disestablish(txc->txc_deferred_transmit);
630 1.98 yamaguch if (txc->txc_intrq != NULL)
631 1.98 yamaguch pcq_destroy(txc->txc_intrq);
632 1.98 yamaguch kmem_free(txc, sizeof(txc));
633 1.98 yamaguch }
634 1.98 yamaguch
635 1.98 yamaguch vioif_work_set(&netq->netq_work, NULL, NULL);
636 1.98 yamaguch if (netq->netq_softint != NULL) {
637 1.98 yamaguch softint_disestablish(netq->netq_softint);
638 1.98 yamaguch netq->netq_softint = NULL;
639 1.98 yamaguch }
640 1.98 yamaguch netq->netq_vq->vq_intrhand = NULL;
641 1.98 yamaguch netq->netq_vq->vq_intrhand_arg = NULL;
642 1.98 yamaguch
643 1.98 yamaguch virtio_free_vq(vsc, vq);
644 1.98 yamaguch mutex_destroy(&netq->netq_lock);
645 1.98 yamaguch netq->netq_vq = NULL;
646 1.98 yamaguch
647 1.98 yamaguch return -1;
648 1.98 yamaguch }
649 1.98 yamaguch
650 1.98 yamaguch static void
651 1.98 yamaguch vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc,
652 1.98 yamaguch size_t qid)
653 1.98 yamaguch {
654 1.98 yamaguch struct vioif_netqueue *netq;
655 1.98 yamaguch struct vioif_rx_context *rxc;
656 1.98 yamaguch struct vioif_tx_context *txc;
657 1.98 yamaguch int dir;
658 1.98 yamaguch
659 1.98 yamaguch netq = &sc->sc_netqs[qid];
660 1.98 yamaguch
661 1.98 yamaguch if (netq->netq_vq == NULL)
662 1.98 yamaguch return;
663 1.98 yamaguch
664 1.98 yamaguch netq = &sc->sc_netqs[qid];
665 1.98 yamaguch dir = VIOIF_NETQ_DIR(qid);
666 1.98 yamaguch switch (dir) {
667 1.98 yamaguch case VIOIF_NETQ_RX:
668 1.98 yamaguch rxc = netq->netq_ctx;
669 1.98 yamaguch netq->netq_ctx = NULL;
670 1.98 yamaguch kmem_free(rxc, sizeof(*rxc));
671 1.98 yamaguch break;
672 1.98 yamaguch case VIOIF_NETQ_TX:
673 1.98 yamaguch txc = netq->netq_ctx;
674 1.98 yamaguch netq->netq_ctx = NULL;
675 1.98 yamaguch softint_disestablish(txc->txc_deferred_transmit);
676 1.98 yamaguch pcq_destroy(txc->txc_intrq);
677 1.98 yamaguch kmem_free(txc, sizeof(*txc));
678 1.98 yamaguch break;
679 1.46 yamaguch }
680 1.98 yamaguch
681 1.98 yamaguch softint_disestablish(netq->netq_softint);
682 1.98 yamaguch virtio_free_vq(vsc, netq->netq_vq);
683 1.98 yamaguch mutex_destroy(&netq->netq_lock);
684 1.98 yamaguch netq->netq_vq = NULL;
685 1.46 yamaguch }
686 1.46 yamaguch
687 1.1 hannken /* allocate memory */
688 1.1 hannken /*
689 1.1 hannken * dma memory is used for:
690 1.98 yamaguch * netq_maps_kva: metadata array for received frames (READ) and
691 1.98 yamaguch * sent frames (WRITE)
692 1.43 yamaguch * ctrlq_cmd: command to be sent via ctrl vq (WRITE)
693 1.43 yamaguch * ctrlq_status: return value for a command via ctrl vq (READ)
694 1.43 yamaguch * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command
695 1.1 hannken * (WRITE)
696 1.43 yamaguch * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
697 1.1 hannken * class command (WRITE)
698 1.43 yamaguch * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
699 1.1 hannken * class command (WRITE)
700 1.43 yamaguch * ctrlq_* structures are allocated only one each; they are protected by
701 1.43 yamaguch * ctrlq_inuse variable and ctrlq_wait condvar.
702 1.1 hannken */
703 1.1 hannken static int
704 1.1 hannken vioif_alloc_mems(struct vioif_softc *sc)
705 1.1 hannken {
706 1.1 hannken struct virtio_softc *vsc = sc->sc_virtio;
707 1.98 yamaguch struct vioif_netqueue *netq;
708 1.98 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
709 1.95 yamaguch struct vioif_net_map *maps;
710 1.98 yamaguch unsigned int vq_num;
711 1.98 yamaguch int r, rsegs;
712 1.98 yamaguch bus_size_t dmamemsize;
713 1.98 yamaguch size_t qid, i, netq_num, kmemsize;
714 1.1 hannken void *vaddr;
715 1.1 hannken intptr_t p;
716 1.1 hannken
717 1.98 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
718 1.98 yamaguch
719 1.98 yamaguch /* allocate DMA memory */
720 1.98 yamaguch dmamemsize = 0;
721 1.46 yamaguch
722 1.98 yamaguch for (qid = 0; qid < netq_num; qid++) {
723 1.98 yamaguch maps = sc->sc_netqs[qid].netq_maps;
724 1.98 yamaguch vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
725 1.98 yamaguch dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num;
726 1.46 yamaguch }
727 1.98 yamaguch
728 1.32 jdolecek if (sc->sc_has_ctrl) {
729 1.98 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_cmd);
730 1.98 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_status);
731 1.98 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_rx);
732 1.98 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
733 1.74 yamaguch + ETHER_ADDR_LEN;
734 1.98 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
735 1.50 christos + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
736 1.98 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr);
737 1.98 yamaguch dmamemsize += sizeof(struct virtio_net_ctrl_mq);
738 1.1 hannken }
739 1.98 yamaguch
740 1.98 yamaguch r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0,
741 1.50 christos &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
742 1.1 hannken if (r != 0) {
743 1.1 hannken aprint_error_dev(sc->sc_dev,
744 1.98 yamaguch "DMA memory allocation failed, size %zu, "
745 1.98 yamaguch "error code %d\n", dmamemsize, r);
746 1.1 hannken goto err_none;
747 1.1 hannken }
748 1.98 yamaguch r = bus_dmamem_map(virtio_dmat(vsc),&sc->sc_hdr_segs[0], 1,
749 1.98 yamaguch dmamemsize, &vaddr, BUS_DMA_NOWAIT);
750 1.1 hannken if (r != 0) {
751 1.1 hannken aprint_error_dev(sc->sc_dev,
752 1.50 christos "DMA memory map failed, error code %d\n", r);
753 1.1 hannken goto err_dmamem_alloc;
754 1.1 hannken }
755 1.42 yamaguch
756 1.98 yamaguch /* assign DMA memory */
757 1.98 yamaguch memset(vaddr, 0, dmamemsize);
758 1.43 yamaguch sc->sc_dmamem = vaddr;
759 1.1 hannken p = (intptr_t) vaddr;
760 1.42 yamaguch
761 1.98 yamaguch for (qid = 0; qid < netq_num; qid++) {
762 1.98 yamaguch netq = &sc->sc_netqs[qid];
763 1.98 yamaguch maps = netq->netq_maps;
764 1.98 yamaguch vq_num = netq->netq_vq->vq_num;
765 1.98 yamaguch
766 1.98 yamaguch netq->netq_maps_kva = vioif_assign_mem(&p,
767 1.98 yamaguch sizeof(*maps[0].vnm_hdr) * vq_num);
768 1.46 yamaguch }
769 1.98 yamaguch
770 1.32 jdolecek if (sc->sc_has_ctrl) {
771 1.58 yamaguch ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
772 1.58 yamaguch sizeof(*ctrlq->ctrlq_cmd));
773 1.58 yamaguch ctrlq->ctrlq_status = vioif_assign_mem(&p,
774 1.58 yamaguch sizeof(*ctrlq->ctrlq_status));
775 1.58 yamaguch ctrlq->ctrlq_rx = vioif_assign_mem(&p,
776 1.58 yamaguch sizeof(*ctrlq->ctrlq_rx));
777 1.58 yamaguch ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
778 1.74 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_uc)
779 1.74 yamaguch + ETHER_ADDR_LEN);
780 1.58 yamaguch ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
781 1.58 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_mc)
782 1.50 christos + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
783 1.74 yamaguch ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p,
784 1.74 yamaguch sizeof(*ctrlq->ctrlq_mac_addr));
785 1.58 yamaguch ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
786 1.1 hannken }
787 1.1 hannken
788 1.98 yamaguch /* allocate kmem */
789 1.98 yamaguch kmemsize = 0;
790 1.46 yamaguch
791 1.98 yamaguch for (qid = 0; qid < netq_num; qid++) {
792 1.98 yamaguch netq = &sc->sc_netqs[qid];
793 1.98 yamaguch vq_num = netq->netq_vq->vq_num;
794 1.95 yamaguch
795 1.98 yamaguch kmemsize += sizeof(netq->netq_maps[0]) * vq_num;
796 1.98 yamaguch }
797 1.46 yamaguch
798 1.98 yamaguch vaddr = kmem_zalloc(kmemsize, KM_SLEEP);
799 1.43 yamaguch sc->sc_kmem = vaddr;
800 1.98 yamaguch
801 1.98 yamaguch /* assign allocated kmem */
802 1.42 yamaguch p = (intptr_t) vaddr;
803 1.42 yamaguch
804 1.98 yamaguch for (qid = 0; qid < netq_num; qid++) {
805 1.98 yamaguch netq = &sc->sc_netqs[qid];
806 1.98 yamaguch vq_num = netq->netq_vq->vq_num;
807 1.98 yamaguch
808 1.98 yamaguch netq->netq_maps = vioif_assign_mem(&p,
809 1.98 yamaguch sizeof(netq->netq_maps[0]) * vq_num);
810 1.98 yamaguch }
811 1.98 yamaguch
812 1.98 yamaguch /* prepare dmamaps */
813 1.98 yamaguch for (qid = 0; qid < netq_num; qid++) {
814 1.98 yamaguch static const struct {
815 1.98 yamaguch const char *msg_hdr;
816 1.98 yamaguch const char *msg_payload;
817 1.98 yamaguch int dma_flag;
818 1.98 yamaguch bus_size_t dma_size;
819 1.98 yamaguch int dma_nsegs;
820 1.98 yamaguch } dmaparams[VIOIF_NETQ_IDX] = {
821 1.98 yamaguch [VIOIF_NETQ_RX] = {
822 1.98 yamaguch .msg_hdr = "rx header",
823 1.98 yamaguch .msg_payload = "rx payload",
824 1.98 yamaguch .dma_flag = BUS_DMA_READ,
825 1.98 yamaguch .dma_size = MCLBYTES - ETHER_ALIGN,
826 1.98 yamaguch .dma_nsegs = 1,
827 1.98 yamaguch },
828 1.98 yamaguch [VIOIF_NETQ_TX] = {
829 1.98 yamaguch .msg_hdr = "tx header",
830 1.98 yamaguch .msg_payload = "tx payload",
831 1.98 yamaguch .dma_flag = BUS_DMA_WRITE,
832 1.98 yamaguch .dma_size = ETHER_MAX_LEN,
833 1.98 yamaguch .dma_nsegs = VIRTIO_NET_TX_MAXNSEGS,
834 1.98 yamaguch }
835 1.98 yamaguch };
836 1.1 hannken
837 1.95 yamaguch struct virtio_net_hdr *hdrs;
838 1.98 yamaguch int dir;
839 1.95 yamaguch
840 1.98 yamaguch dir = VIOIF_NETQ_DIR(qid);
841 1.98 yamaguch netq = &sc->sc_netqs[qid];
842 1.98 yamaguch vq_num = netq->netq_vq->vq_num;
843 1.98 yamaguch maps = netq->netq_maps;
844 1.98 yamaguch hdrs = netq->netq_maps_kva;
845 1.58 yamaguch
846 1.95 yamaguch for (i = 0; i < vq_num; i++) {
847 1.95 yamaguch maps[i].vnm_hdr = &hdrs[i];
848 1.98 yamaguch
849 1.95 yamaguch r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map,
850 1.98 yamaguch maps[i].vnm_hdr, sc->sc_hdr_size, 1,
851 1.98 yamaguch dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr);
852 1.58 yamaguch if (r != 0)
853 1.58 yamaguch goto err_reqs;
854 1.58 yamaguch
855 1.95 yamaguch r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map,
856 1.98 yamaguch dmaparams[dir].dma_size, dmaparams[dir].dma_nsegs,
857 1.98 yamaguch dmaparams[dir].msg_payload);
858 1.58 yamaguch if (r != 0)
859 1.58 yamaguch goto err_reqs;
860 1.46 yamaguch }
861 1.1 hannken }
862 1.1 hannken
863 1.32 jdolecek if (sc->sc_has_ctrl) {
864 1.1 hannken /* control vq class & command */
865 1.58 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
866 1.43 yamaguch ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
867 1.42 yamaguch BUS_DMA_WRITE, "control command");
868 1.58 yamaguch if (r != 0)
869 1.58 yamaguch goto err_reqs;
870 1.58 yamaguch
871 1.58 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
872 1.43 yamaguch ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
873 1.42 yamaguch BUS_DMA_READ, "control status");
874 1.58 yamaguch if (r != 0)
875 1.58 yamaguch goto err_reqs;
876 1.1 hannken
877 1.1 hannken /* control vq rx mode command parameter */
878 1.58 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
879 1.43 yamaguch ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
880 1.42 yamaguch BUS_DMA_WRITE, "rx mode control command");
881 1.58 yamaguch if (r != 0)
882 1.58 yamaguch goto err_reqs;
883 1.1 hannken
884 1.46 yamaguch /* multiqueue set command */
885 1.58 yamaguch r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
886 1.46 yamaguch ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
887 1.46 yamaguch BUS_DMA_WRITE, "multiqueue set command");
888 1.58 yamaguch if (r != 0)
889 1.58 yamaguch goto err_reqs;
890 1.46 yamaguch
891 1.1 hannken /* control vq MAC filter table for unicast */
892 1.1 hannken /* do not load now since its length is variable */
893 1.58 yamaguch r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
894 1.74 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_uc)
895 1.74 yamaguch + ETHER_ADDR_LEN, 1,
896 1.42 yamaguch "unicast MAC address filter command");
897 1.58 yamaguch if (r != 0)
898 1.58 yamaguch goto err_reqs;
899 1.1 hannken
900 1.1 hannken /* control vq MAC filter table for multicast */
901 1.58 yamaguch r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
902 1.43 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_mc)
903 1.42 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
904 1.42 yamaguch "multicast MAC address filter command");
905 1.73 yamaguch if (r != 0)
906 1.73 yamaguch goto err_reqs;
907 1.74 yamaguch
908 1.74 yamaguch /* control vq MAC address set command */
909 1.74 yamaguch r = vioif_dmamap_create_load(sc,
910 1.74 yamaguch &ctrlq->ctrlq_mac_addr_dmamap,
911 1.74 yamaguch ctrlq->ctrlq_mac_addr,
912 1.74 yamaguch sizeof(*ctrlq->ctrlq_mac_addr), 1,
913 1.74 yamaguch BUS_DMA_WRITE, "mac addr set command");
914 1.74 yamaguch if (r != 0)
915 1.74 yamaguch goto err_reqs;
916 1.1 hannken }
917 1.1 hannken
918 1.1 hannken return 0;
919 1.1 hannken
920 1.1 hannken err_reqs:
921 1.58 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
922 1.58 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
923 1.58 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
924 1.58 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
925 1.58 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
926 1.74 yamaguch vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap);
927 1.98 yamaguch for (qid = 0; qid < netq_num; qid++) {
928 1.98 yamaguch vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
929 1.98 yamaguch maps = sc->sc_netqs[qid].netq_maps;
930 1.46 yamaguch
931 1.95 yamaguch for (i = 0; i < vq_num; i++) {
932 1.95 yamaguch vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map);
933 1.95 yamaguch vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map);
934 1.46 yamaguch }
935 1.1 hannken }
936 1.43 yamaguch if (sc->sc_kmem) {
937 1.98 yamaguch kmem_free(sc->sc_kmem, kmemsize);
938 1.43 yamaguch sc->sc_kmem = NULL;
939 1.1 hannken }
940 1.98 yamaguch bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize);
941 1.1 hannken err_dmamem_alloc:
942 1.32 jdolecek bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
943 1.1 hannken err_none:
944 1.1 hannken return -1;
945 1.1 hannken }
946 1.1 hannken
947 1.1 hannken static void
948 1.1 hannken vioif_attach(device_t parent, device_t self, void *aux)
949 1.1 hannken {
950 1.1 hannken struct vioif_softc *sc = device_private(self);
951 1.1 hannken struct virtio_softc *vsc = device_private(parent);
952 1.98 yamaguch struct vioif_netqueue *txq0;
953 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
954 1.66 reinoud uint64_t features, req_features;
955 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
956 1.43 yamaguch u_int softint_flags;
957 1.98 yamaguch int r, i, req_flags;
958 1.55 yamaguch char xnamebuf[MAXCOMLEN];
959 1.98 yamaguch size_t netq_num;
960 1.1 hannken
961 1.32 jdolecek if (virtio_child(vsc) != NULL) {
962 1.1 hannken aprint_normal(": child already attached for %s; "
963 1.50 christos "something wrong...\n", device_xname(parent));
964 1.1 hannken return;
965 1.1 hannken }
966 1.1 hannken
967 1.1 hannken sc->sc_dev = self;
968 1.1 hannken sc->sc_virtio = vsc;
969 1.82 knakahar sc->sc_link_state = LINK_STATE_UNKNOWN;
970 1.1 hannken
971 1.46 yamaguch sc->sc_max_nvq_pairs = 1;
972 1.46 yamaguch sc->sc_req_nvq_pairs = 1;
973 1.46 yamaguch sc->sc_act_nvq_pairs = 1;
974 1.55 yamaguch sc->sc_txrx_workqueue_sysctl = true;
975 1.55 yamaguch sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
976 1.55 yamaguch sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
977 1.55 yamaguch sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
978 1.55 yamaguch sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
979 1.55 yamaguch
980 1.62 yamaguch mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE);
981 1.62 yamaguch
982 1.55 yamaguch snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
983 1.55 yamaguch sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
984 1.55 yamaguch IPL_NET, WQ_PERCPU | WQ_MPSAFE);
985 1.55 yamaguch if (sc->sc_txrx_workqueue == NULL)
986 1.55 yamaguch goto err;
987 1.46 yamaguch
988 1.32 jdolecek req_flags = 0;
989 1.1 hannken
990 1.7 ozaki #ifdef VIOIF_MPSAFE
991 1.66 reinoud req_flags |= VIRTIO_F_INTR_MPSAFE;
992 1.7 ozaki #endif
993 1.66 reinoud req_flags |= VIRTIO_F_INTR_MSIX;
994 1.32 jdolecek
995 1.46 yamaguch req_features =
996 1.46 yamaguch VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
997 1.46 yamaguch VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
998 1.66 reinoud req_features |= VIRTIO_F_RING_EVENT_IDX;
999 1.78 yamaguch req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR;
1000 1.46 yamaguch #ifdef VIOIF_MULTIQ
1001 1.46 yamaguch req_features |= VIRTIO_NET_F_MQ;
1002 1.46 yamaguch #endif
1003 1.46 yamaguch virtio_child_attach_start(vsc, self, IPL_NET, NULL,
1004 1.54 yamaguch vioif_config_change, virtio_vq_intrhand, req_flags,
1005 1.46 yamaguch req_features, VIRTIO_NET_FLAG_BITS);
1006 1.32 jdolecek
1007 1.32 jdolecek features = virtio_features(vsc);
1008 1.66 reinoud if (features == 0)
1009 1.66 reinoud goto err;
1010 1.7 ozaki
1011 1.1 hannken if (features & VIRTIO_NET_F_MAC) {
1012 1.50 christos for (i = 0; i < __arraycount(sc->sc_mac); i++) {
1013 1.50 christos sc->sc_mac[i] = virtio_read_device_config_1(vsc,
1014 1.50 christos VIRTIO_NET_CONFIG_MAC + i);
1015 1.50 christos }
1016 1.1 hannken } else {
1017 1.1 hannken /* code stolen from sys/net/if_tap.c */
1018 1.1 hannken struct timeval tv;
1019 1.1 hannken uint32_t ui;
1020 1.1 hannken getmicrouptime(&tv);
1021 1.1 hannken ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
1022 1.1 hannken memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
1023 1.50 christos for (i = 0; i < __arraycount(sc->sc_mac); i++) {
1024 1.50 christos virtio_write_device_config_1(vsc,
1025 1.50 christos VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
1026 1.50 christos }
1027 1.1 hannken }
1028 1.32 jdolecek
1029 1.66 reinoud /* 'Ethernet' with capital follows other ethernet driver attachment */
1030 1.50 christos aprint_normal_dev(self, "Ethernet address %s\n",
1031 1.50 christos ether_sprintf(sc->sc_mac));
1032 1.1 hannken
1033 1.66 reinoud if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) {
1034 1.66 reinoud sc->sc_hdr_size = sizeof(struct virtio_net_hdr);
1035 1.66 reinoud } else {
1036 1.66 reinoud sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers);
1037 1.66 reinoud }
1038 1.66 reinoud
1039 1.46 yamaguch if ((features & VIRTIO_NET_F_CTRL_VQ) &&
1040 1.46 yamaguch (features & VIRTIO_NET_F_CTRL_RX)) {
1041 1.46 yamaguch sc->sc_has_ctrl = true;
1042 1.46 yamaguch
1043 1.46 yamaguch cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
1044 1.46 yamaguch mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
1045 1.46 yamaguch ctrlq->ctrlq_inuse = FREE;
1046 1.46 yamaguch } else {
1047 1.46 yamaguch sc->sc_has_ctrl = false;
1048 1.46 yamaguch }
1049 1.46 yamaguch
1050 1.46 yamaguch if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
1051 1.46 yamaguch sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
1052 1.46 yamaguch VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
1053 1.46 yamaguch
1054 1.46 yamaguch if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
1055 1.46 yamaguch goto err;
1056 1.46 yamaguch
1057 1.46 yamaguch /* Limit the number of queue pairs to use */
1058 1.46 yamaguch sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
1059 1.46 yamaguch }
1060 1.46 yamaguch
1061 1.51 chs vioif_alloc_queues(sc);
1062 1.46 yamaguch virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
1063 1.46 yamaguch
1064 1.43 yamaguch #ifdef VIOIF_MPSAFE
1065 1.43 yamaguch softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
1066 1.43 yamaguch #else
1067 1.43 yamaguch softint_flags = SOFTINT_NET;
1068 1.43 yamaguch #endif
1069 1.7 ozaki
1070 1.17 ozaki /*
1071 1.98 yamaguch * Initialize network queues
1072 1.17 ozaki */
1073 1.98 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
1074 1.98 yamaguch for (i = 0; i < netq_num; i++) {
1075 1.98 yamaguch r = vioif_netqueue_init(sc, vsc, i, softint_flags);
1076 1.46 yamaguch if (r != 0)
1077 1.46 yamaguch goto err;
1078 1.43 yamaguch }
1079 1.17 ozaki
1080 1.46 yamaguch if (sc->sc_has_ctrl) {
1081 1.98 yamaguch int ctrlq_idx = sc->sc_max_nvq_pairs * 2;
1082 1.17 ozaki /*
1083 1.17 ozaki * Allocating a virtqueue for control channel
1084 1.17 ozaki */
1085 1.98 yamaguch sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx];
1086 1.98 yamaguch r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx,
1087 1.17 ozaki NBPG, 1, "control");
1088 1.17 ozaki if (r != 0) {
1089 1.17 ozaki aprint_error_dev(self, "failed to allocate "
1090 1.50 christos "a virtqueue for control channel, error code %d\n",
1091 1.50 christos r);
1092 1.46 yamaguch
1093 1.46 yamaguch sc->sc_has_ctrl = false;
1094 1.46 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
1095 1.46 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
1096 1.46 yamaguch } else {
1097 1.54 yamaguch ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
1098 1.54 yamaguch ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
1099 1.1 hannken }
1100 1.1 hannken }
1101 1.34 ozaki
1102 1.48 msaitoh sc->sc_ctl_softint = softint_establish(softint_flags,
1103 1.48 msaitoh vioif_ctl_softint, sc);
1104 1.34 ozaki if (sc->sc_ctl_softint == NULL) {
1105 1.34 ozaki aprint_error_dev(self, "cannot establish ctl softint\n");
1106 1.1 hannken goto err;
1107 1.1 hannken }
1108 1.1 hannken
1109 1.1 hannken if (vioif_alloc_mems(sc) < 0)
1110 1.1 hannken goto err;
1111 1.1 hannken
1112 1.32 jdolecek if (virtio_child_attach_finish(vsc) != 0)
1113 1.32 jdolecek goto err;
1114 1.32 jdolecek
1115 1.55 yamaguch if (vioif_setup_sysctl(sc) != 0) {
1116 1.55 yamaguch aprint_error_dev(self, "unable to create sysctl node\n");
1117 1.55 yamaguch /* continue */
1118 1.55 yamaguch }
1119 1.55 yamaguch
1120 1.63 yamaguch vioif_setup_stats(sc);
1121 1.63 yamaguch
1122 1.1 hannken strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
1123 1.1 hannken ifp->if_softc = sc;
1124 1.1 hannken ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1125 1.45 yamaguch #ifdef VIOIF_MPSAFE
1126 1.45 yamaguch ifp->if_extflags = IFEF_MPSAFE;
1127 1.45 yamaguch #endif
1128 1.1 hannken ifp->if_start = vioif_start;
1129 1.46 yamaguch if (sc->sc_req_nvq_pairs > 1)
1130 1.46 yamaguch ifp->if_transmit = vioif_transmit;
1131 1.1 hannken ifp->if_ioctl = vioif_ioctl;
1132 1.1 hannken ifp->if_init = vioif_init;
1133 1.1 hannken ifp->if_stop = vioif_stop;
1134 1.1 hannken ifp->if_capabilities = 0;
1135 1.1 hannken ifp->if_watchdog = vioif_watchdog;
1136 1.98 yamaguch txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
1137 1.98 yamaguch IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN));
1138 1.36 jdolecek IFQ_SET_READY(&ifp->if_snd);
1139 1.1 hannken
1140 1.11 ozaki sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
1141 1.11 ozaki
1142 1.1 hannken if_attach(ifp);
1143 1.28 ozaki if_deferred_start_init(ifp, NULL);
1144 1.1 hannken ether_ifattach(ifp, sc->sc_mac);
1145 1.75 yamaguch ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb);
1146 1.1 hannken
1147 1.1 hannken return;
1148 1.1 hannken
1149 1.1 hannken err:
1150 1.98 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
1151 1.98 yamaguch for (i = 0; i < netq_num; i++) {
1152 1.98 yamaguch vioif_netqueue_teardown(sc, vsc, i);
1153 1.43 yamaguch }
1154 1.7 ozaki
1155 1.32 jdolecek if (sc->sc_has_ctrl) {
1156 1.43 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
1157 1.43 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
1158 1.98 yamaguch virtio_free_vq(vsc, ctrlq->ctrlq_vq);
1159 1.98 yamaguch ctrlq->ctrlq_vq = NULL;
1160 1.1 hannken }
1161 1.20 christos
1162 1.46 yamaguch vioif_free_queues(sc);
1163 1.62 yamaguch mutex_destroy(&sc->sc_lock);
1164 1.55 yamaguch virtio_child_attach_failed(vsc);
1165 1.55 yamaguch config_finalize_register(self, vioif_finalize_teardown);
1166 1.20 christos
1167 1.1 hannken return;
1168 1.1 hannken }
1169 1.1 hannken
1170 1.55 yamaguch static int
1171 1.55 yamaguch vioif_finalize_teardown(device_t self)
1172 1.55 yamaguch {
1173 1.55 yamaguch struct vioif_softc *sc = device_private(self);
1174 1.55 yamaguch
1175 1.55 yamaguch if (sc->sc_txrx_workqueue != NULL) {
1176 1.55 yamaguch vioif_workq_destroy(sc->sc_txrx_workqueue);
1177 1.55 yamaguch sc->sc_txrx_workqueue = NULL;
1178 1.55 yamaguch }
1179 1.55 yamaguch
1180 1.55 yamaguch return 0;
1181 1.55 yamaguch }
1182 1.55 yamaguch
1183 1.46 yamaguch static void
1184 1.46 yamaguch vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
1185 1.46 yamaguch {
1186 1.46 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1187 1.98 yamaguch struct vioif_netqueue *netq;
1188 1.98 yamaguch size_t i, netq_act_num;
1189 1.46 yamaguch
1190 1.98 yamaguch netq_act_num = sc->sc_act_nvq_pairs * 2;
1191 1.98 yamaguch for (i = 0; i < netq_act_num; i++) {
1192 1.98 yamaguch netq = &sc->sc_netqs[i];
1193 1.98 yamaguch virtio_start_vq_intr(vsc, netq->netq_vq);
1194 1.46 yamaguch }
1195 1.46 yamaguch }
1196 1.46 yamaguch
1197 1.46 yamaguch static void
1198 1.46 yamaguch vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
1199 1.46 yamaguch {
1200 1.46 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1201 1.98 yamaguch struct vioif_netqueue *netq;
1202 1.98 yamaguch size_t i, netq_act_num;
1203 1.46 yamaguch
1204 1.98 yamaguch netq_act_num = sc->sc_act_nvq_pairs * 2;
1205 1.98 yamaguch for (i = 0; i < netq_act_num; i++) {
1206 1.98 yamaguch netq = &sc->sc_netqs[i];
1207 1.98 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1208 1.46 yamaguch }
1209 1.46 yamaguch }
1210 1.46 yamaguch
1211 1.1 hannken /*
1212 1.1 hannken * Interface functions for ifnet
1213 1.1 hannken */
1214 1.1 hannken static int
1215 1.1 hannken vioif_init(struct ifnet *ifp)
1216 1.1 hannken {
1217 1.1 hannken struct vioif_softc *sc = ifp->if_softc;
1218 1.33 ozaki struct virtio_softc *vsc = sc->sc_virtio;
1219 1.98 yamaguch struct vioif_netqueue *netq;
1220 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1221 1.46 yamaguch int r, i;
1222 1.1 hannken
1223 1.1 hannken vioif_stop(ifp, 0);
1224 1.7 ozaki
1225 1.71 yamaguch r = virtio_reinit_start(vsc);
1226 1.71 yamaguch if (r != 0) {
1227 1.71 yamaguch log(LOG_ERR, "%s: reset failed\n", ifp->if_xname);
1228 1.71 yamaguch return EIO;
1229 1.71 yamaguch }
1230 1.71 yamaguch
1231 1.33 ozaki virtio_negotiate_features(vsc, virtio_features(vsc));
1232 1.46 yamaguch
1233 1.46 yamaguch for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
1234 1.98 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
1235 1.61 yamaguch
1236 1.98 yamaguch mutex_enter(&netq->netq_lock);
1237 1.98 yamaguch vioif_populate_rx_mbufs_locked(sc, netq);
1238 1.98 yamaguch mutex_exit(&netq->netq_lock);
1239 1.46 yamaguch }
1240 1.46 yamaguch
1241 1.46 yamaguch virtio_reinit_end(vsc);
1242 1.47 yamaguch
1243 1.47 yamaguch if (sc->sc_has_ctrl)
1244 1.43 yamaguch virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
1245 1.46 yamaguch
1246 1.46 yamaguch r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
1247 1.46 yamaguch if (r == 0)
1248 1.46 yamaguch sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
1249 1.46 yamaguch else
1250 1.46 yamaguch sc->sc_act_nvq_pairs = 1;
1251 1.46 yamaguch
1252 1.98 yamaguch SET(ifp->if_flags, IFF_RUNNING);
1253 1.98 yamaguch CLR(ifp->if_flags, IFF_OACTIVE);
1254 1.98 yamaguch
1255 1.46 yamaguch vioif_enable_interrupt_vqpairs(sc);
1256 1.33 ozaki
1257 1.34 ozaki vioif_update_link_status(sc);
1258 1.75 yamaguch r = vioif_rx_filter(sc);
1259 1.1 hannken
1260 1.75 yamaguch return r;
1261 1.1 hannken }
1262 1.1 hannken
1263 1.1 hannken static void
1264 1.77 yamaguch vioif_stop(struct ifnet *ifp, int disable)
1265 1.76 yamaguch {
1266 1.77 yamaguch struct vioif_softc *sc = ifp->if_softc;
1267 1.77 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1268 1.98 yamaguch struct vioif_netqueue *netq;
1269 1.77 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1270 1.98 yamaguch size_t i, netq_act_num;
1271 1.76 yamaguch
1272 1.98 yamaguch netq_act_num = sc->sc_act_nvq_pairs * 2;
1273 1.77 yamaguch
1274 1.98 yamaguch CLR(ifp->if_flags, IFF_RUNNING);
1275 1.98 yamaguch for (i = 0; i < netq_act_num; i++) {
1276 1.98 yamaguch netq = &sc->sc_netqs[i];
1277 1.76 yamaguch
1278 1.98 yamaguch mutex_enter(&netq->netq_lock);
1279 1.98 yamaguch netq->netq_stopping = true;
1280 1.98 yamaguch mutex_exit(&netq->netq_lock);
1281 1.76 yamaguch }
1282 1.33 ozaki
1283 1.85 yamaguch /* disable interrupts */
1284 1.85 yamaguch vioif_disable_interrupt_vqpairs(sc);
1285 1.85 yamaguch if (sc->sc_has_ctrl)
1286 1.85 yamaguch virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
1287 1.85 yamaguch
1288 1.85 yamaguch /*
1289 1.85 yamaguch * only way to stop interrupt, I/O and DMA is resetting...
1290 1.85 yamaguch *
1291 1.85 yamaguch * NOTE: Devices based on VirtIO draft specification can not
1292 1.85 yamaguch * stop interrupt completely even if virtio_stop_vq_intr() is called.
1293 1.85 yamaguch */
1294 1.1 hannken virtio_reset(vsc);
1295 1.55 yamaguch
1296 1.85 yamaguch vioif_intr_barrier();
1297 1.85 yamaguch
1298 1.98 yamaguch for (i = 0; i < netq_act_num; i++) {
1299 1.98 yamaguch netq = &sc->sc_netqs[i];
1300 1.98 yamaguch vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work);
1301 1.85 yamaguch }
1302 1.85 yamaguch
1303 1.55 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1304 1.98 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
1305 1.98 yamaguch vioif_rx_queue_clear(sc, vsc, netq);
1306 1.98 yamaguch
1307 1.98 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
1308 1.98 yamaguch vioif_tx_queue_clear(sc, vsc, netq);
1309 1.55 yamaguch }
1310 1.46 yamaguch
1311 1.85 yamaguch /* all packet processing is stopped */
1312 1.98 yamaguch for (i = 0; i < netq_act_num; i++) {
1313 1.98 yamaguch netq = &sc->sc_netqs[i];
1314 1.85 yamaguch
1315 1.98 yamaguch mutex_enter(&netq->netq_lock);
1316 1.98 yamaguch netq->netq_stopping = false;
1317 1.98 yamaguch mutex_exit(&netq->netq_lock);
1318 1.85 yamaguch }
1319 1.1 hannken }
1320 1.1 hannken
1321 1.1 hannken static void
1322 1.98 yamaguch vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq,
1323 1.48 msaitoh bool is_transmit)
1324 1.1 hannken {
1325 1.1 hannken struct vioif_softc *sc = ifp->if_softc;
1326 1.1 hannken struct virtio_softc *vsc = sc->sc_virtio;
1327 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1328 1.98 yamaguch struct vioif_tx_context *txc;
1329 1.95 yamaguch struct vioif_net_map *map;
1330 1.66 reinoud struct virtio_net_hdr *hdr;
1331 1.1 hannken struct mbuf *m;
1332 1.36 jdolecek int queued = 0;
1333 1.1 hannken
1334 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1335 1.46 yamaguch
1336 1.98 yamaguch if (netq->netq_stopping ||
1337 1.98 yamaguch !ISSET(ifp->if_flags, IFF_RUNNING))
1338 1.46 yamaguch return;
1339 1.7 ozaki
1340 1.98 yamaguch txc = netq->netq_ctx;
1341 1.98 yamaguch
1342 1.98 yamaguch if (!txc->txc_link_active)
1343 1.46 yamaguch return;
1344 1.7 ozaki
1345 1.98 yamaguch if (!is_transmit &&
1346 1.98 yamaguch ISSET(ifp->if_flags, IFF_OACTIVE))
1347 1.46 yamaguch return;
1348 1.1 hannken
1349 1.2 jmcneill for (;;) {
1350 1.1 hannken int slot, r;
1351 1.96 yamaguch r = virtio_enqueue_prep(vsc, vq, &slot);
1352 1.96 yamaguch if (r == EAGAIN)
1353 1.96 yamaguch break;
1354 1.96 yamaguch if (__predict_false(r != 0))
1355 1.96 yamaguch panic("enqueue_prep for tx buffers");
1356 1.1 hannken
1357 1.46 yamaguch if (is_transmit)
1358 1.98 yamaguch m = pcq_get(txc->txc_intrq);
1359 1.46 yamaguch else
1360 1.46 yamaguch IFQ_DEQUEUE(&ifp->if_snd, m);
1361 1.46 yamaguch
1362 1.96 yamaguch if (m == NULL) {
1363 1.96 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1364 1.36 jdolecek break;
1365 1.1 hannken }
1366 1.36 jdolecek
1367 1.98 yamaguch map = &netq->netq_maps[slot];
1368 1.95 yamaguch KASSERT(map->vnm_mbuf == NULL);
1369 1.95 yamaguch
1370 1.32 jdolecek r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1371 1.95 yamaguch map->vnm_mbuf_map, m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1372 1.1 hannken if (r != 0) {
1373 1.36 jdolecek /* maybe just too fragmented */
1374 1.36 jdolecek struct mbuf *newm;
1375 1.36 jdolecek
1376 1.36 jdolecek newm = m_defrag(m, M_NOWAIT);
1377 1.36 jdolecek if (newm == NULL) {
1378 1.98 yamaguch txc->txc_defrag_failed.ev_count++;
1379 1.36 jdolecek goto skip;
1380 1.36 jdolecek }
1381 1.36 jdolecek
1382 1.37 jdolecek m = newm;
1383 1.36 jdolecek r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1384 1.95 yamaguch map->vnm_mbuf_map, m,
1385 1.50 christos BUS_DMA_WRITE | BUS_DMA_NOWAIT);
1386 1.36 jdolecek if (r != 0) {
1387 1.98 yamaguch netq->netq_mbuf_load_failed.ev_count++;
1388 1.36 jdolecek skip:
1389 1.37 jdolecek m_freem(m);
1390 1.94 yamaguch if_statinc(ifp, if_oerrors);
1391 1.36 jdolecek virtio_enqueue_abort(vsc, vq, slot);
1392 1.36 jdolecek continue;
1393 1.36 jdolecek }
1394 1.1 hannken }
1395 1.36 jdolecek
1396 1.36 jdolecek /* This should actually never fail */
1397 1.1 hannken r = virtio_enqueue_reserve(vsc, vq, slot,
1398 1.95 yamaguch map->vnm_mbuf_map->dm_nsegs + 1);
1399 1.1 hannken if (r != 0) {
1400 1.98 yamaguch netq->netq_enqueue_reserve_failed.ev_count++;
1401 1.32 jdolecek bus_dmamap_unload(virtio_dmat(vsc),
1402 1.95 yamaguch map->vnm_mbuf_map);
1403 1.36 jdolecek /* slot already freed by virtio_enqueue_reserve */
1404 1.37 jdolecek m_freem(m);
1405 1.94 yamaguch if_statinc(ifp, if_oerrors);
1406 1.36 jdolecek continue;
1407 1.1 hannken }
1408 1.7 ozaki
1409 1.95 yamaguch map->vnm_mbuf = m;
1410 1.95 yamaguch hdr = map->vnm_hdr;
1411 1.66 reinoud memset(hdr, 0, sc->sc_hdr_size);
1412 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1413 1.95 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1414 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1415 1.95 yamaguch 0, map->vnm_hdr_map->dm_mapsize, BUS_DMASYNC_PREWRITE);
1416 1.95 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, true);
1417 1.95 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, true);
1418 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, false);
1419 1.36 jdolecek
1420 1.1 hannken queued++;
1421 1.41 msaitoh bpf_mtap(ifp, m, BPF_D_OUT);
1422 1.1 hannken }
1423 1.1 hannken
1424 1.1 hannken if (queued > 0) {
1425 1.1 hannken virtio_enqueue_commit(vsc, vq, -1, true);
1426 1.1 hannken ifp->if_timer = 5;
1427 1.1 hannken }
1428 1.46 yamaguch }
1429 1.46 yamaguch
1430 1.46 yamaguch static void
1431 1.98 yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
1432 1.46 yamaguch {
1433 1.46 yamaguch
1434 1.46 yamaguch /*
1435 1.46 yamaguch * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
1436 1.46 yamaguch */
1437 1.98 yamaguch vioif_send_common_locked(ifp, netq, false);
1438 1.46 yamaguch
1439 1.46 yamaguch }
1440 1.46 yamaguch
1441 1.46 yamaguch static void
1442 1.46 yamaguch vioif_start(struct ifnet *ifp)
1443 1.46 yamaguch {
1444 1.46 yamaguch struct vioif_softc *sc = ifp->if_softc;
1445 1.98 yamaguch struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
1446 1.46 yamaguch
1447 1.46 yamaguch #ifdef VIOIF_MPSAFE
1448 1.46 yamaguch KASSERT(if_is_mpsafe(ifp));
1449 1.46 yamaguch #endif
1450 1.46 yamaguch
1451 1.98 yamaguch mutex_enter(&txq0->netq_lock);
1452 1.98 yamaguch vioif_start_locked(ifp, txq0);
1453 1.98 yamaguch mutex_exit(&txq0->netq_lock);
1454 1.46 yamaguch }
1455 1.46 yamaguch
1456 1.46 yamaguch static inline int
1457 1.46 yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
1458 1.46 yamaguch {
1459 1.46 yamaguch struct vioif_softc *sc = ifp->if_softc;
1460 1.46 yamaguch u_int cpuid = cpu_index(curcpu());
1461 1.46 yamaguch
1462 1.98 yamaguch return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs);
1463 1.46 yamaguch }
1464 1.46 yamaguch
1465 1.46 yamaguch static void
1466 1.98 yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
1467 1.46 yamaguch {
1468 1.46 yamaguch
1469 1.98 yamaguch vioif_send_common_locked(ifp, netq, true);
1470 1.46 yamaguch }
1471 1.46 yamaguch
1472 1.46 yamaguch static int
1473 1.46 yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
1474 1.46 yamaguch {
1475 1.46 yamaguch struct vioif_softc *sc = ifp->if_softc;
1476 1.98 yamaguch struct vioif_netqueue *netq;
1477 1.98 yamaguch struct vioif_tx_context *txc;
1478 1.46 yamaguch int qid;
1479 1.46 yamaguch
1480 1.46 yamaguch qid = vioif_select_txqueue(ifp, m);
1481 1.98 yamaguch netq = &sc->sc_netqs[qid];
1482 1.98 yamaguch txc = netq->netq_ctx;
1483 1.46 yamaguch
1484 1.98 yamaguch if (__predict_false(!pcq_put(txc->txc_intrq, m))) {
1485 1.46 yamaguch m_freem(m);
1486 1.46 yamaguch return ENOBUFS;
1487 1.46 yamaguch }
1488 1.46 yamaguch
1489 1.52 thorpej net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
1490 1.52 thorpej if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
1491 1.46 yamaguch if (m->m_flags & M_MCAST)
1492 1.52 thorpej if_statinc_ref(nsr, if_omcasts);
1493 1.52 thorpej IF_STAT_PUTREF(ifp);
1494 1.46 yamaguch
1495 1.98 yamaguch if (mutex_tryenter(&netq->netq_lock)) {
1496 1.98 yamaguch vioif_transmit_locked(ifp, netq);
1497 1.98 yamaguch mutex_exit(&netq->netq_lock);
1498 1.46 yamaguch }
1499 1.46 yamaguch
1500 1.46 yamaguch return 0;
1501 1.46 yamaguch }
1502 1.46 yamaguch
1503 1.46 yamaguch static void
1504 1.46 yamaguch vioif_deferred_transmit(void *arg)
1505 1.46 yamaguch {
1506 1.98 yamaguch struct vioif_netqueue *netq = arg;
1507 1.98 yamaguch struct virtio_softc *vsc = netq->netq_vq->vq_owner;
1508 1.46 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1509 1.46 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1510 1.7 ozaki
1511 1.98 yamaguch mutex_enter(&netq->netq_lock);
1512 1.98 yamaguch vioif_send_common_locked(ifp, netq, true);
1513 1.98 yamaguch mutex_exit(&netq->netq_lock);
1514 1.1 hannken }
1515 1.1 hannken
1516 1.1 hannken static int
1517 1.1 hannken vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
1518 1.1 hannken {
1519 1.1 hannken int s, r;
1520 1.1 hannken
1521 1.1 hannken s = splnet();
1522 1.1 hannken
1523 1.1 hannken r = ether_ioctl(ifp, cmd, data);
1524 1.75 yamaguch if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) {
1525 1.75 yamaguch if (ifp->if_flags & IFF_RUNNING) {
1526 1.1 hannken r = vioif_rx_filter(ifp->if_softc);
1527 1.75 yamaguch } else {
1528 1.1 hannken r = 0;
1529 1.75 yamaguch }
1530 1.1 hannken }
1531 1.1 hannken
1532 1.1 hannken splx(s);
1533 1.1 hannken
1534 1.1 hannken return r;
1535 1.1 hannken }
1536 1.1 hannken
1537 1.1 hannken void
1538 1.1 hannken vioif_watchdog(struct ifnet *ifp)
1539 1.1 hannken {
1540 1.1 hannken struct vioif_softc *sc = ifp->if_softc;
1541 1.98 yamaguch struct vioif_netqueue *netq;
1542 1.46 yamaguch int i;
1543 1.1 hannken
1544 1.46 yamaguch if (ifp->if_flags & IFF_RUNNING) {
1545 1.55 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1546 1.98 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
1547 1.93 yamaguch
1548 1.98 yamaguch mutex_enter(&netq->netq_lock);
1549 1.98 yamaguch if (!netq->netq_running_handle) {
1550 1.98 yamaguch netq->netq_running_handle = true;
1551 1.98 yamaguch vioif_net_sched_handle(sc, netq);
1552 1.93 yamaguch }
1553 1.98 yamaguch mutex_exit(&netq->netq_lock);
1554 1.55 yamaguch }
1555 1.46 yamaguch }
1556 1.1 hannken }
1557 1.1 hannken
1558 1.98 yamaguch static void
1559 1.98 yamaguch vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq)
1560 1.98 yamaguch {
1561 1.98 yamaguch
1562 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1563 1.98 yamaguch KASSERT(!netq->netq_stopping);
1564 1.98 yamaguch
1565 1.98 yamaguch if (netq->netq_workqueue) {
1566 1.98 yamaguch vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work);
1567 1.98 yamaguch } else {
1568 1.98 yamaguch softint_schedule(netq->netq_softint);
1569 1.98 yamaguch }
1570 1.98 yamaguch }
1571 1.98 yamaguch
1572 1.1 hannken /*
1573 1.39 dholland * Receive implementation
1574 1.1 hannken */
1575 1.39 dholland /* add mbufs for all the empty receive slots */
1576 1.1 hannken static void
1577 1.98 yamaguch vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq)
1578 1.12 ozaki {
1579 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1580 1.46 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1581 1.98 yamaguch struct vioif_rx_context *rxc;
1582 1.95 yamaguch struct vioif_net_map *map;
1583 1.89 yamaguch struct mbuf *m;
1584 1.1 hannken int i, r, ndone = 0;
1585 1.1 hannken
1586 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1587 1.98 yamaguch
1588 1.98 yamaguch rxc = netq->netq_ctx;
1589 1.7 ozaki
1590 1.1 hannken for (i = 0; i < vq->vq_num; i++) {
1591 1.1 hannken int slot;
1592 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
1593 1.1 hannken if (r == EAGAIN)
1594 1.1 hannken break;
1595 1.97 yamaguch if (__predict_false(r != 0))
1596 1.1 hannken panic("enqueue_prep for rx buffers");
1597 1.89 yamaguch
1598 1.98 yamaguch map = &netq->netq_maps[slot];
1599 1.95 yamaguch KASSERT(map->vnm_mbuf == NULL);
1600 1.95 yamaguch
1601 1.91 yamaguch MGETHDR(m, M_DONTWAIT, MT_DATA);
1602 1.89 yamaguch if (m == NULL) {
1603 1.91 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1604 1.98 yamaguch rxc->rxc_mbuf_enobufs.ev_count++;
1605 1.91 yamaguch break;
1606 1.91 yamaguch }
1607 1.91 yamaguch MCLGET(m, M_DONTWAIT);
1608 1.91 yamaguch if ((m->m_flags & M_EXT) == 0) {
1609 1.91 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1610 1.91 yamaguch m_freem(m);
1611 1.98 yamaguch rxc->rxc_mbuf_enobufs.ev_count++;
1612 1.91 yamaguch break;
1613 1.91 yamaguch }
1614 1.89 yamaguch
1615 1.91 yamaguch m->m_len = m->m_pkthdr.len = MCLBYTES;
1616 1.91 yamaguch m_adj(m, ETHER_ALIGN);
1617 1.89 yamaguch
1618 1.91 yamaguch r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1619 1.95 yamaguch map->vnm_mbuf_map, m, BUS_DMA_READ | BUS_DMA_NOWAIT);
1620 1.89 yamaguch
1621 1.91 yamaguch if (r != 0) {
1622 1.91 yamaguch virtio_enqueue_abort(vsc, vq, slot);
1623 1.91 yamaguch m_freem(m);
1624 1.98 yamaguch netq->netq_mbuf_load_failed.ev_count++;
1625 1.91 yamaguch break;
1626 1.1 hannken }
1627 1.89 yamaguch
1628 1.1 hannken r = virtio_enqueue_reserve(vsc, vq, slot,
1629 1.95 yamaguch map->vnm_mbuf_map->dm_nsegs + 1);
1630 1.1 hannken if (r != 0) {
1631 1.98 yamaguch netq->netq_enqueue_reserve_failed.ev_count++;
1632 1.95 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1633 1.89 yamaguch m_freem(m);
1634 1.89 yamaguch /* slot already freed by virtio_enqueue_reserve */
1635 1.1 hannken break;
1636 1.1 hannken }
1637 1.95 yamaguch
1638 1.95 yamaguch map->vnm_mbuf = m;
1639 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1640 1.66 reinoud 0, sc->sc_hdr_size, BUS_DMASYNC_PREREAD);
1641 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1642 1.95 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, BUS_DMASYNC_PREREAD);
1643 1.95 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, false);
1644 1.95 yamaguch virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, false);
1645 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, false);
1646 1.1 hannken ndone++;
1647 1.1 hannken }
1648 1.1 hannken if (ndone > 0)
1649 1.1 hannken virtio_enqueue_commit(vsc, vq, -1, true);
1650 1.1 hannken }
1651 1.1 hannken
1652 1.55 yamaguch static void
1653 1.93 yamaguch vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
1654 1.98 yamaguch struct vioif_netqueue *netq)
1655 1.1 hannken {
1656 1.95 yamaguch struct vioif_net_map *map;
1657 1.93 yamaguch unsigned int i, vq_num;
1658 1.55 yamaguch bool more;
1659 1.7 ozaki
1660 1.98 yamaguch mutex_enter(&netq->netq_lock);
1661 1.98 yamaguch vq_num = netq->netq_vq->vq_num;
1662 1.93 yamaguch
1663 1.55 yamaguch for (;;) {
1664 1.98 yamaguch more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL);
1665 1.55 yamaguch if (more == false)
1666 1.55 yamaguch break;
1667 1.55 yamaguch }
1668 1.93 yamaguch
1669 1.93 yamaguch for (i = 0; i < vq_num; i++) {
1670 1.98 yamaguch map = &netq->netq_maps[i];
1671 1.95 yamaguch
1672 1.95 yamaguch if (map->vnm_mbuf == NULL)
1673 1.93 yamaguch continue;
1674 1.93 yamaguch
1675 1.95 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1676 1.95 yamaguch m_freem(map->vnm_mbuf);
1677 1.95 yamaguch map->vnm_mbuf = NULL;
1678 1.93 yamaguch }
1679 1.98 yamaguch mutex_exit(&netq->netq_lock);
1680 1.7 ozaki }
1681 1.7 ozaki
1682 1.39 dholland /* dequeue received packets */
1683 1.55 yamaguch static bool
1684 1.55 yamaguch vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1685 1.98 yamaguch struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
1686 1.7 ozaki {
1687 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1688 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1689 1.95 yamaguch struct vioif_net_map *map;
1690 1.1 hannken struct mbuf *m;
1691 1.1 hannken int slot, len;
1692 1.92 yamaguch bool more;
1693 1.92 yamaguch size_t ndeq;
1694 1.1 hannken
1695 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1696 1.7 ozaki
1697 1.92 yamaguch more = false;
1698 1.92 yamaguch ndeq = 0;
1699 1.92 yamaguch
1700 1.54 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
1701 1.92 yamaguch goto done;
1702 1.55 yamaguch
1703 1.92 yamaguch for (;;ndeq++) {
1704 1.92 yamaguch if (ndeq >= limit) {
1705 1.55 yamaguch more = true;
1706 1.55 yamaguch break;
1707 1.55 yamaguch }
1708 1.55 yamaguch
1709 1.55 yamaguch if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1710 1.55 yamaguch break;
1711 1.55 yamaguch
1712 1.98 yamaguch map = &netq->netq_maps[slot];
1713 1.95 yamaguch KASSERT(map->vnm_mbuf != NULL);
1714 1.95 yamaguch
1715 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1716 1.66 reinoud 0, sc->sc_hdr_size, BUS_DMASYNC_POSTREAD);
1717 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1718 1.95 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, BUS_DMASYNC_POSTREAD);
1719 1.95 yamaguch
1720 1.95 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1721 1.95 yamaguch m = map->vnm_mbuf;
1722 1.95 yamaguch map->vnm_mbuf = NULL;
1723 1.1 hannken virtio_dequeue_commit(vsc, vq, slot);
1724 1.95 yamaguch
1725 1.95 yamaguch m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size;
1726 1.24 ozaki m_set_rcvif(m, ifp);
1727 1.22 ozaki if_percpuq_enqueue(ifp->if_percpuq, m);
1728 1.1 hannken }
1729 1.3 christos
1730 1.92 yamaguch done:
1731 1.92 yamaguch if (ndeqp != NULL)
1732 1.92 yamaguch *ndeqp = ndeq;
1733 1.55 yamaguch
1734 1.55 yamaguch return more;
1735 1.1 hannken }
1736 1.1 hannken
1737 1.1 hannken /* rx interrupt; call _dequeue above and schedule a softint */
1738 1.66 reinoud
1739 1.66 reinoud static void
1740 1.98 yamaguch vioif_rx_handle_locked(void *xnetq, u_int limit)
1741 1.66 reinoud {
1742 1.98 yamaguch struct vioif_netqueue *netq = xnetq;
1743 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1744 1.66 reinoud struct virtio_softc *vsc = vq->vq_owner;
1745 1.66 reinoud struct vioif_softc *sc = device_private(virtio_child(vsc));
1746 1.66 reinoud bool more;
1747 1.87 yamaguch int enqueued;
1748 1.92 yamaguch size_t ndeq;
1749 1.66 reinoud
1750 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1751 1.98 yamaguch KASSERT(!netq->netq_stopping);
1752 1.66 reinoud
1753 1.98 yamaguch more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq);
1754 1.92 yamaguch if (ndeq > 0)
1755 1.98 yamaguch vioif_populate_rx_mbufs_locked(sc, netq);
1756 1.92 yamaguch
1757 1.66 reinoud if (more) {
1758 1.98 yamaguch vioif_net_sched_handle(sc, netq);
1759 1.66 reinoud return;
1760 1.70 skrll }
1761 1.87 yamaguch
1762 1.98 yamaguch enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
1763 1.87 yamaguch if (enqueued != 0) {
1764 1.98 yamaguch virtio_stop_vq_intr(vsc, netq->netq_vq);
1765 1.98 yamaguch vioif_net_sched_handle(sc, netq);
1766 1.66 reinoud return;
1767 1.70 skrll }
1768 1.84 yamaguch
1769 1.98 yamaguch netq->netq_running_handle = false;
1770 1.66 reinoud }
1771 1.66 reinoud
1772 1.1 hannken static int
1773 1.54 yamaguch vioif_rx_intr(void *arg)
1774 1.1 hannken {
1775 1.98 yamaguch struct vioif_netqueue *netq = arg;
1776 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1777 1.55 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1778 1.55 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1779 1.55 yamaguch u_int limit;
1780 1.55 yamaguch
1781 1.98 yamaguch mutex_enter(&netq->netq_lock);
1782 1.7 ozaki
1783 1.98 yamaguch /* handler is already running in softint/workqueue */
1784 1.98 yamaguch if (netq->netq_running_handle)
1785 1.84 yamaguch goto done;
1786 1.55 yamaguch
1787 1.98 yamaguch netq->netq_running_handle = true;
1788 1.55 yamaguch
1789 1.84 yamaguch limit = sc->sc_rx_intr_process_limit;
1790 1.84 yamaguch virtio_stop_vq_intr(vsc, vq);
1791 1.98 yamaguch vioif_rx_handle_locked(netq, limit);
1792 1.55 yamaguch
1793 1.84 yamaguch done:
1794 1.98 yamaguch mutex_exit(&netq->netq_lock);
1795 1.55 yamaguch return 1;
1796 1.55 yamaguch }
1797 1.55 yamaguch
1798 1.55 yamaguch static void
1799 1.98 yamaguch vioif_rx_handle(void *xnetq)
1800 1.55 yamaguch {
1801 1.98 yamaguch struct vioif_netqueue *netq = xnetq;
1802 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1803 1.55 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1804 1.55 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1805 1.55 yamaguch u_int limit;
1806 1.55 yamaguch
1807 1.98 yamaguch mutex_enter(&netq->netq_lock);
1808 1.85 yamaguch
1809 1.98 yamaguch KASSERT(netq->netq_running_handle);
1810 1.55 yamaguch
1811 1.98 yamaguch if (netq->netq_stopping) {
1812 1.98 yamaguch netq->netq_running_handle = false;
1813 1.85 yamaguch goto done;
1814 1.85 yamaguch }
1815 1.1 hannken
1816 1.85 yamaguch limit = sc->sc_rx_process_limit;
1817 1.98 yamaguch vioif_rx_handle_locked(netq, limit);
1818 1.1 hannken
1819 1.85 yamaguch done:
1820 1.98 yamaguch mutex_exit(&netq->netq_lock);
1821 1.1 hannken }
1822 1.1 hannken
1823 1.1 hannken /*
1824 1.1 hannken * Transmition implementation
1825 1.1 hannken */
1826 1.1 hannken /* actual transmission is done in if_start */
1827 1.1 hannken /* tx interrupt; dequeue and free mbufs */
1828 1.1 hannken /*
1829 1.1 hannken * tx interrupt is actually disabled; this should be called upon
1830 1.1 hannken * tx vq full and watchdog
1831 1.1 hannken */
1832 1.55 yamaguch
1833 1.66 reinoud static void
1834 1.98 yamaguch vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit)
1835 1.66 reinoud {
1836 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1837 1.98 yamaguch struct vioif_tx_context *txc = netq->netq_ctx;
1838 1.66 reinoud struct virtio_softc *vsc = vq->vq_owner;
1839 1.66 reinoud struct vioif_softc *sc = device_private(virtio_child(vsc));
1840 1.66 reinoud struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1841 1.66 reinoud bool more;
1842 1.87 yamaguch int enqueued;
1843 1.66 reinoud
1844 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1845 1.98 yamaguch KASSERT(!netq->netq_stopping);
1846 1.66 reinoud
1847 1.98 yamaguch more = vioif_tx_deq_locked(sc, vsc, netq, limit);
1848 1.66 reinoud if (more) {
1849 1.98 yamaguch vioif_net_sched_handle(sc, netq);
1850 1.66 reinoud return;
1851 1.66 reinoud }
1852 1.66 reinoud
1853 1.87 yamaguch enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ?
1854 1.87 yamaguch virtio_postpone_intr_smart(vsc, vq):
1855 1.87 yamaguch virtio_start_vq_intr(vsc, vq);
1856 1.87 yamaguch if (enqueued != 0) {
1857 1.87 yamaguch virtio_stop_vq_intr(vsc, vq);
1858 1.98 yamaguch vioif_net_sched_handle(sc, netq);
1859 1.66 reinoud return;
1860 1.66 reinoud }
1861 1.66 reinoud
1862 1.98 yamaguch netq->netq_running_handle = false;
1863 1.84 yamaguch
1864 1.66 reinoud /* for ALTQ */
1865 1.98 yamaguch if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]) {
1866 1.66 reinoud if_schedule_deferred_start(ifp);
1867 1.66 reinoud ifp->if_flags &= ~IFF_OACTIVE;
1868 1.66 reinoud }
1869 1.98 yamaguch softint_schedule(txc->txc_deferred_transmit);
1870 1.66 reinoud }
1871 1.66 reinoud
1872 1.1 hannken static int
1873 1.54 yamaguch vioif_tx_intr(void *arg)
1874 1.1 hannken {
1875 1.98 yamaguch struct vioif_netqueue *netq = arg;
1876 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1877 1.1 hannken struct virtio_softc *vsc = vq->vq_owner;
1878 1.32 jdolecek struct vioif_softc *sc = device_private(virtio_child(vsc));
1879 1.55 yamaguch u_int limit;
1880 1.55 yamaguch
1881 1.98 yamaguch mutex_enter(&netq->netq_lock);
1882 1.7 ozaki
1883 1.84 yamaguch /* tx handler is already running in softint/workqueue */
1884 1.98 yamaguch if (netq->netq_running_handle)
1885 1.84 yamaguch goto done;
1886 1.7 ozaki
1887 1.98 yamaguch if (netq->netq_stopping)
1888 1.84 yamaguch goto done;
1889 1.55 yamaguch
1890 1.98 yamaguch netq->netq_running_handle = true;
1891 1.55 yamaguch
1892 1.84 yamaguch virtio_stop_vq_intr(vsc, vq);
1893 1.98 yamaguch netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl;
1894 1.98 yamaguch limit = sc->sc_tx_intr_process_limit;
1895 1.98 yamaguch vioif_tx_handle_locked(netq, limit);
1896 1.7 ozaki
1897 1.84 yamaguch done:
1898 1.98 yamaguch mutex_exit(&netq->netq_lock);
1899 1.55 yamaguch return 1;
1900 1.55 yamaguch }
1901 1.55 yamaguch
1902 1.55 yamaguch static void
1903 1.98 yamaguch vioif_tx_handle(void *xnetq)
1904 1.55 yamaguch {
1905 1.98 yamaguch struct vioif_netqueue *netq = xnetq;
1906 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1907 1.55 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1908 1.55 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1909 1.55 yamaguch u_int limit;
1910 1.55 yamaguch
1911 1.98 yamaguch mutex_enter(&netq->netq_lock);
1912 1.85 yamaguch
1913 1.98 yamaguch KASSERT(netq->netq_running_handle);
1914 1.85 yamaguch
1915 1.98 yamaguch if (netq->netq_stopping) {
1916 1.98 yamaguch netq->netq_running_handle = false;
1917 1.85 yamaguch goto done;
1918 1.85 yamaguch }
1919 1.85 yamaguch
1920 1.55 yamaguch limit = sc->sc_tx_process_limit;
1921 1.98 yamaguch vioif_tx_handle_locked(netq, limit);
1922 1.55 yamaguch
1923 1.85 yamaguch done:
1924 1.98 yamaguch mutex_exit(&netq->netq_lock);
1925 1.7 ozaki }
1926 1.7 ozaki
1927 1.55 yamaguch static void
1928 1.93 yamaguch vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
1929 1.98 yamaguch struct vioif_netqueue *netq)
1930 1.7 ozaki {
1931 1.95 yamaguch struct vioif_net_map *map;
1932 1.93 yamaguch unsigned int i, vq_num;
1933 1.55 yamaguch bool more;
1934 1.55 yamaguch
1935 1.98 yamaguch mutex_enter(&netq->netq_lock);
1936 1.93 yamaguch
1937 1.98 yamaguch vq_num = netq->netq_vq->vq_num;
1938 1.55 yamaguch for (;;) {
1939 1.98 yamaguch more = vioif_tx_deq_locked(sc, vsc, netq, vq_num);
1940 1.55 yamaguch if (more == false)
1941 1.55 yamaguch break;
1942 1.55 yamaguch }
1943 1.93 yamaguch
1944 1.93 yamaguch for (i = 0; i < vq_num; i++) {
1945 1.98 yamaguch map = &netq->netq_maps[i];
1946 1.95 yamaguch if (map->vnm_mbuf == NULL)
1947 1.93 yamaguch continue;
1948 1.93 yamaguch
1949 1.95 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1950 1.95 yamaguch m_freem(map->vnm_mbuf);
1951 1.95 yamaguch map->vnm_mbuf = NULL;
1952 1.93 yamaguch }
1953 1.98 yamaguch mutex_exit(&netq->netq_lock);
1954 1.55 yamaguch }
1955 1.55 yamaguch
1956 1.55 yamaguch static bool
1957 1.55 yamaguch vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
1958 1.98 yamaguch struct vioif_netqueue *netq, u_int limit)
1959 1.55 yamaguch {
1960 1.98 yamaguch struct virtqueue *vq = netq->netq_vq;
1961 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1962 1.95 yamaguch struct vioif_net_map *map;
1963 1.1 hannken struct mbuf *m;
1964 1.1 hannken int slot, len;
1965 1.55 yamaguch bool more = false;
1966 1.1 hannken
1967 1.98 yamaguch KASSERT(mutex_owned(&netq->netq_lock));
1968 1.7 ozaki
1969 1.54 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
1970 1.55 yamaguch return false;
1971 1.55 yamaguch
1972 1.55 yamaguch for (;;) {
1973 1.55 yamaguch if (limit-- == 0) {
1974 1.55 yamaguch more = true;
1975 1.55 yamaguch break;
1976 1.55 yamaguch }
1977 1.55 yamaguch
1978 1.55 yamaguch if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
1979 1.55 yamaguch break;
1980 1.54 yamaguch
1981 1.98 yamaguch map = &netq->netq_maps[slot];
1982 1.95 yamaguch KASSERT(map->vnm_mbuf != NULL);
1983 1.95 yamaguch
1984 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
1985 1.66 reinoud 0, sc->sc_hdr_size, BUS_DMASYNC_POSTWRITE);
1986 1.95 yamaguch bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
1987 1.95 yamaguch 0, map->vnm_mbuf_map->dm_mapsize, BUS_DMASYNC_POSTWRITE);
1988 1.95 yamaguch
1989 1.95 yamaguch bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
1990 1.95 yamaguch m = map->vnm_mbuf;
1991 1.95 yamaguch map->vnm_mbuf = NULL;
1992 1.1 hannken virtio_dequeue_commit(vsc, vq, slot);
1993 1.95 yamaguch
1994 1.52 thorpej if_statinc(ifp, if_opackets);
1995 1.1 hannken m_freem(m);
1996 1.1 hannken }
1997 1.1 hannken
1998 1.55 yamaguch return more;
1999 1.1 hannken }
2000 1.1 hannken
2001 1.1 hannken /*
2002 1.1 hannken * Control vq
2003 1.1 hannken */
2004 1.1 hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
2005 1.44 yamaguch static void
2006 1.44 yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
2007 1.1 hannken {
2008 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2009 1.1 hannken
2010 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2011 1.43 yamaguch while (ctrlq->ctrlq_inuse != FREE)
2012 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
2013 1.43 yamaguch ctrlq->ctrlq_inuse = INUSE;
2014 1.44 yamaguch ctrlq->ctrlq_owner = curlwp;
2015 1.44 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2016 1.44 yamaguch }
2017 1.44 yamaguch
2018 1.44 yamaguch static void
2019 1.44 yamaguch vioif_ctrl_release(struct vioif_softc *sc)
2020 1.44 yamaguch {
2021 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2022 1.44 yamaguch
2023 1.44 yamaguch KASSERT(ctrlq->ctrlq_inuse != FREE);
2024 1.44 yamaguch KASSERT(ctrlq->ctrlq_owner == curlwp);
2025 1.44 yamaguch
2026 1.44 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2027 1.44 yamaguch ctrlq->ctrlq_inuse = FREE;
2028 1.46 yamaguch ctrlq->ctrlq_owner = NULL;
2029 1.44 yamaguch cv_signal(&ctrlq->ctrlq_wait);
2030 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2031 1.44 yamaguch }
2032 1.44 yamaguch
2033 1.44 yamaguch static int
2034 1.44 yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
2035 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2036 1.44 yamaguch {
2037 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2038 1.44 yamaguch int i, r, loaded;
2039 1.44 yamaguch
2040 1.44 yamaguch loaded = 0;
2041 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2042 1.44 yamaguch r = bus_dmamap_load(virtio_dmat(vsc),
2043 1.44 yamaguch specs[i].dmamap, specs[i].buf, specs[i].bufsize,
2044 1.48 msaitoh NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
2045 1.44 yamaguch if (r) {
2046 1.63 yamaguch sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++;
2047 1.44 yamaguch goto err;
2048 1.44 yamaguch }
2049 1.44 yamaguch loaded++;
2050 1.44 yamaguch
2051 1.44 yamaguch }
2052 1.44 yamaguch
2053 1.44 yamaguch return r;
2054 1.44 yamaguch
2055 1.44 yamaguch err:
2056 1.44 yamaguch for (i = 0; i < loaded; i++) {
2057 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2058 1.44 yamaguch }
2059 1.44 yamaguch
2060 1.44 yamaguch return r;
2061 1.44 yamaguch }
2062 1.44 yamaguch
2063 1.44 yamaguch static void
2064 1.44 yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
2065 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2066 1.44 yamaguch {
2067 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2068 1.44 yamaguch int i;
2069 1.44 yamaguch
2070 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2071 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
2072 1.44 yamaguch }
2073 1.44 yamaguch }
2074 1.44 yamaguch
2075 1.44 yamaguch static int
2076 1.44 yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
2077 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
2078 1.44 yamaguch {
2079 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2080 1.44 yamaguch struct virtqueue *vq = ctrlq->ctrlq_vq;
2081 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
2082 1.44 yamaguch int i, r, slot;
2083 1.43 yamaguch
2084 1.44 yamaguch ctrlq->ctrlq_cmd->class = class;
2085 1.43 yamaguch ctrlq->ctrlq_cmd->command = cmd;
2086 1.1 hannken
2087 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
2088 1.50 christos 0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
2089 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2090 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
2091 1.50 christos 0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
2092 1.44 yamaguch }
2093 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
2094 1.50 christos 0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
2095 1.1 hannken
2096 1.66 reinoud /* we need to explicitly (re)start vq intr when using RING EVENT IDX */
2097 1.66 reinoud if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
2098 1.66 reinoud virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
2099 1.66 reinoud
2100 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
2101 1.1 hannken if (r != 0)
2102 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2103 1.44 yamaguch r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
2104 1.1 hannken if (r != 0)
2105 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
2106 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
2107 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2108 1.44 yamaguch virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
2109 1.44 yamaguch }
2110 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
2111 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, true);
2112 1.1 hannken
2113 1.1 hannken /* wait for done */
2114 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2115 1.43 yamaguch while (ctrlq->ctrlq_inuse != DONE)
2116 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
2117 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2118 1.1 hannken /* already dequeueued */
2119 1.1 hannken
2120 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
2121 1.50 christos sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
2122 1.44 yamaguch for (i = 0; i < nspecs; i++) {
2123 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
2124 1.50 christos specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
2125 1.44 yamaguch }
2126 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
2127 1.50 christos sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
2128 1.1 hannken
2129 1.43 yamaguch if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
2130 1.1 hannken r = 0;
2131 1.1 hannken else {
2132 1.63 yamaguch device_printf(sc->sc_dev, "failed setting rx mode\n");
2133 1.63 yamaguch sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++;
2134 1.1 hannken r = EIO;
2135 1.1 hannken }
2136 1.1 hannken
2137 1.44 yamaguch return r;
2138 1.44 yamaguch }
2139 1.44 yamaguch
2140 1.44 yamaguch static int
2141 1.44 yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
2142 1.44 yamaguch {
2143 1.44 yamaguch struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
2144 1.44 yamaguch struct vioif_ctrl_cmdspec specs[1];
2145 1.44 yamaguch int r;
2146 1.44 yamaguch
2147 1.44 yamaguch if (!sc->sc_has_ctrl)
2148 1.44 yamaguch return ENOTSUP;
2149 1.44 yamaguch
2150 1.44 yamaguch vioif_ctrl_acquire(sc);
2151 1.44 yamaguch
2152 1.44 yamaguch rx->onoff = onoff;
2153 1.44 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
2154 1.44 yamaguch specs[0].buf = rx;
2155 1.44 yamaguch specs[0].bufsize = sizeof(*rx);
2156 1.44 yamaguch
2157 1.44 yamaguch r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
2158 1.44 yamaguch specs, __arraycount(specs));
2159 1.3 christos
2160 1.44 yamaguch vioif_ctrl_release(sc);
2161 1.1 hannken return r;
2162 1.1 hannken }
2163 1.1 hannken
2164 1.1 hannken static int
2165 1.1 hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
2166 1.1 hannken {
2167 1.50 christos return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
2168 1.1 hannken }
2169 1.1 hannken
2170 1.1 hannken static int
2171 1.1 hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
2172 1.1 hannken {
2173 1.50 christos return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
2174 1.1 hannken }
2175 1.1 hannken
2176 1.1 hannken /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
2177 1.1 hannken static int
2178 1.1 hannken vioif_set_rx_filter(struct vioif_softc *sc)
2179 1.1 hannken {
2180 1.43 yamaguch /* filter already set in ctrlq->ctrlq_mac_tbl */
2181 1.66 reinoud struct virtio_softc *vsc = sc->sc_virtio;
2182 1.44 yamaguch struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
2183 1.44 yamaguch struct vioif_ctrl_cmdspec specs[2];
2184 1.44 yamaguch int nspecs = __arraycount(specs);
2185 1.44 yamaguch int r;
2186 1.44 yamaguch
2187 1.44 yamaguch mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
2188 1.44 yamaguch mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
2189 1.1 hannken
2190 1.32 jdolecek if (!sc->sc_has_ctrl)
2191 1.1 hannken return ENOTSUP;
2192 1.1 hannken
2193 1.44 yamaguch vioif_ctrl_acquire(sc);
2194 1.1 hannken
2195 1.44 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
2196 1.44 yamaguch specs[0].buf = mac_tbl_uc;
2197 1.44 yamaguch specs[0].bufsize = sizeof(*mac_tbl_uc)
2198 1.66 reinoud + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries));
2199 1.44 yamaguch
2200 1.44 yamaguch specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
2201 1.44 yamaguch specs[1].buf = mac_tbl_mc;
2202 1.44 yamaguch specs[1].bufsize = sizeof(*mac_tbl_mc)
2203 1.66 reinoud + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries));
2204 1.1 hannken
2205 1.44 yamaguch r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
2206 1.44 yamaguch if (r != 0)
2207 1.1 hannken goto out;
2208 1.1 hannken
2209 1.44 yamaguch r = vioif_ctrl_send_command(sc,
2210 1.44 yamaguch VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
2211 1.44 yamaguch specs, nspecs);
2212 1.1 hannken
2213 1.44 yamaguch vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
2214 1.1 hannken
2215 1.1 hannken out:
2216 1.44 yamaguch vioif_ctrl_release(sc);
2217 1.3 christos
2218 1.1 hannken return r;
2219 1.1 hannken }
2220 1.1 hannken
2221 1.46 yamaguch static int
2222 1.74 yamaguch vioif_set_mac_addr(struct vioif_softc *sc)
2223 1.74 yamaguch {
2224 1.74 yamaguch struct virtio_net_ctrl_mac_addr *ma =
2225 1.74 yamaguch sc->sc_ctrlq.ctrlq_mac_addr;
2226 1.74 yamaguch struct vioif_ctrl_cmdspec specs[1];
2227 1.74 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2228 1.74 yamaguch int nspecs = __arraycount(specs);
2229 1.78 yamaguch uint64_t features;
2230 1.74 yamaguch int r;
2231 1.78 yamaguch size_t i;
2232 1.74 yamaguch
2233 1.74 yamaguch if (!sc->sc_has_ctrl)
2234 1.74 yamaguch return ENOTSUP;
2235 1.74 yamaguch
2236 1.78 yamaguch if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac,
2237 1.78 yamaguch ETHER_ADDR_LEN) == 0) {
2238 1.78 yamaguch return 0;
2239 1.78 yamaguch }
2240 1.74 yamaguch
2241 1.78 yamaguch memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2242 1.74 yamaguch
2243 1.78 yamaguch features = virtio_features(sc->sc_virtio);
2244 1.78 yamaguch if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) {
2245 1.78 yamaguch vioif_ctrl_acquire(sc);
2246 1.78 yamaguch
2247 1.78 yamaguch memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN);
2248 1.78 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap;
2249 1.78 yamaguch specs[0].buf = ma;
2250 1.78 yamaguch specs[0].bufsize = sizeof(*ma);
2251 1.78 yamaguch
2252 1.78 yamaguch r = vioif_ctrl_send_command(sc,
2253 1.78 yamaguch VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET,
2254 1.78 yamaguch specs, nspecs);
2255 1.74 yamaguch
2256 1.78 yamaguch vioif_ctrl_release(sc);
2257 1.78 yamaguch } else {
2258 1.78 yamaguch for (i = 0; i < __arraycount(sc->sc_mac); i++) {
2259 1.78 yamaguch virtio_write_device_config_1(sc->sc_virtio,
2260 1.78 yamaguch VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
2261 1.78 yamaguch }
2262 1.78 yamaguch r = 0;
2263 1.78 yamaguch }
2264 1.74 yamaguch
2265 1.74 yamaguch return r;
2266 1.74 yamaguch }
2267 1.74 yamaguch
2268 1.74 yamaguch static int
2269 1.46 yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
2270 1.46 yamaguch {
2271 1.46 yamaguch struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
2272 1.46 yamaguch struct vioif_ctrl_cmdspec specs[1];
2273 1.46 yamaguch int r;
2274 1.46 yamaguch
2275 1.46 yamaguch if (!sc->sc_has_ctrl)
2276 1.46 yamaguch return ENOTSUP;
2277 1.46 yamaguch
2278 1.46 yamaguch if (nvq_pairs <= 1)
2279 1.46 yamaguch return EINVAL;
2280 1.46 yamaguch
2281 1.46 yamaguch vioif_ctrl_acquire(sc);
2282 1.46 yamaguch
2283 1.66 reinoud mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs);
2284 1.46 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
2285 1.46 yamaguch specs[0].buf = mq;
2286 1.46 yamaguch specs[0].bufsize = sizeof(*mq);
2287 1.46 yamaguch
2288 1.46 yamaguch r = vioif_ctrl_send_command(sc,
2289 1.46 yamaguch VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
2290 1.46 yamaguch specs, __arraycount(specs));
2291 1.46 yamaguch
2292 1.46 yamaguch vioif_ctrl_release(sc);
2293 1.46 yamaguch
2294 1.46 yamaguch return r;
2295 1.46 yamaguch }
2296 1.46 yamaguch
2297 1.1 hannken /* ctrl vq interrupt; wake up the command issuer */
2298 1.1 hannken static int
2299 1.54 yamaguch vioif_ctrl_intr(void *arg)
2300 1.1 hannken {
2301 1.54 yamaguch struct vioif_ctrlqueue *ctrlq = arg;
2302 1.54 yamaguch struct virtqueue *vq = ctrlq->ctrlq_vq;
2303 1.1 hannken struct virtio_softc *vsc = vq->vq_owner;
2304 1.1 hannken int r, slot;
2305 1.1 hannken
2306 1.54 yamaguch if (virtio_vq_is_enqueued(vsc, vq) == false)
2307 1.54 yamaguch return 0;
2308 1.54 yamaguch
2309 1.1 hannken r = virtio_dequeue(vsc, vq, &slot, NULL);
2310 1.1 hannken if (r == ENOENT)
2311 1.1 hannken return 0;
2312 1.1 hannken virtio_dequeue_commit(vsc, vq, slot);
2313 1.1 hannken
2314 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
2315 1.43 yamaguch ctrlq->ctrlq_inuse = DONE;
2316 1.43 yamaguch cv_signal(&ctrlq->ctrlq_wait);
2317 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
2318 1.1 hannken
2319 1.1 hannken return 1;
2320 1.1 hannken }
2321 1.1 hannken
2322 1.75 yamaguch static int
2323 1.75 yamaguch vioif_ifflags(struct vioif_softc *sc)
2324 1.75 yamaguch {
2325 1.75 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2326 1.75 yamaguch bool onoff;
2327 1.75 yamaguch int r;
2328 1.75 yamaguch
2329 1.75 yamaguch if (!sc->sc_has_ctrl) {
2330 1.75 yamaguch /* no ctrl vq; always promisc and allmulti */
2331 1.75 yamaguch ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI);
2332 1.75 yamaguch return 0;
2333 1.75 yamaguch }
2334 1.75 yamaguch
2335 1.75 yamaguch onoff = ifp->if_flags & IFF_ALLMULTI ? true : false;
2336 1.75 yamaguch r = vioif_set_allmulti(sc, onoff);
2337 1.75 yamaguch if (r != 0) {
2338 1.75 yamaguch log(LOG_WARNING,
2339 1.75 yamaguch "%s: couldn't %sable ALLMULTI\n",
2340 1.75 yamaguch ifp->if_xname, onoff ? "en" : "dis");
2341 1.75 yamaguch if (onoff == false) {
2342 1.75 yamaguch ifp->if_flags |= IFF_ALLMULTI;
2343 1.75 yamaguch }
2344 1.75 yamaguch }
2345 1.75 yamaguch
2346 1.75 yamaguch onoff = ifp->if_flags & IFF_PROMISC ? true : false;
2347 1.75 yamaguch r = vioif_set_promisc(sc, onoff);
2348 1.75 yamaguch if (r != 0) {
2349 1.75 yamaguch log(LOG_WARNING,
2350 1.75 yamaguch "%s: couldn't %sable PROMISC\n",
2351 1.75 yamaguch ifp->if_xname, onoff ? "en" : "dis");
2352 1.75 yamaguch if (onoff == false) {
2353 1.75 yamaguch ifp->if_flags |= IFF_PROMISC;
2354 1.75 yamaguch }
2355 1.75 yamaguch }
2356 1.75 yamaguch
2357 1.75 yamaguch return 0;
2358 1.75 yamaguch }
2359 1.75 yamaguch
2360 1.75 yamaguch static int
2361 1.75 yamaguch vioif_ifflags_cb(struct ethercom *ec)
2362 1.75 yamaguch {
2363 1.75 yamaguch struct ifnet *ifp = &ec->ec_if;
2364 1.75 yamaguch struct vioif_softc *sc = ifp->if_softc;
2365 1.75 yamaguch
2366 1.75 yamaguch return vioif_ifflags(sc);
2367 1.75 yamaguch }
2368 1.75 yamaguch
2369 1.1 hannken /*
2370 1.1 hannken * If multicast filter small enough (<=MAXENTRIES) set rx filter
2371 1.1 hannken * If large multicast filter exist use ALLMULTI
2372 1.1 hannken * If setting rx filter fails fall back to ALLMULTI
2373 1.1 hannken */
2374 1.1 hannken static int
2375 1.1 hannken vioif_rx_filter(struct vioif_softc *sc)
2376 1.1 hannken {
2377 1.66 reinoud struct virtio_softc *vsc = sc->sc_virtio;
2378 1.48 msaitoh struct ethercom *ec = &sc->sc_ethercom;
2379 1.48 msaitoh struct ifnet *ifp = &ec->ec_if;
2380 1.1 hannken struct ether_multi *enm;
2381 1.1 hannken struct ether_multistep step;
2382 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
2383 1.1 hannken int nentries;
2384 1.75 yamaguch bool allmulti = 0;
2385 1.1 hannken int r;
2386 1.1 hannken
2387 1.75 yamaguch if (!sc->sc_has_ctrl) {
2388 1.75 yamaguch goto set_ifflags;
2389 1.1 hannken }
2390 1.1 hannken
2391 1.74 yamaguch memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0],
2392 1.74 yamaguch CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
2393 1.74 yamaguch
2394 1.75 yamaguch nentries = 0;
2395 1.75 yamaguch allmulti = false;
2396 1.75 yamaguch
2397 1.48 msaitoh ETHER_LOCK(ec);
2398 1.75 yamaguch for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
2399 1.75 yamaguch ETHER_NEXT_MULTI(step, enm)) {
2400 1.1 hannken if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
2401 1.75 yamaguch allmulti = true;
2402 1.75 yamaguch break;
2403 1.1 hannken }
2404 1.50 christos if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
2405 1.75 yamaguch allmulti = true;
2406 1.75 yamaguch break;
2407 1.1 hannken }
2408 1.75 yamaguch
2409 1.43 yamaguch memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
2410 1.50 christos enm->enm_addrlo, ETHER_ADDR_LEN);
2411 1.75 yamaguch nentries++;
2412 1.1 hannken }
2413 1.48 msaitoh ETHER_UNLOCK(ec);
2414 1.30 ozaki
2415 1.74 yamaguch r = vioif_set_mac_addr(sc);
2416 1.74 yamaguch if (r != 0) {
2417 1.74 yamaguch log(LOG_WARNING, "%s: couldn't set MAC address\n",
2418 1.74 yamaguch ifp->if_xname);
2419 1.74 yamaguch }
2420 1.74 yamaguch
2421 1.75 yamaguch if (!allmulti) {
2422 1.74 yamaguch ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1);
2423 1.66 reinoud ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries);
2424 1.1 hannken r = vioif_set_rx_filter(sc);
2425 1.1 hannken if (r != 0) {
2426 1.75 yamaguch allmulti = true; /* fallback */
2427 1.1 hannken }
2428 1.75 yamaguch }
2429 1.75 yamaguch
2430 1.75 yamaguch if (allmulti) {
2431 1.66 reinoud ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0);
2432 1.66 reinoud ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0);
2433 1.1 hannken r = vioif_set_rx_filter(sc);
2434 1.1 hannken if (r != 0) {
2435 1.75 yamaguch log(LOG_DEBUG, "%s: couldn't clear RX filter\n",
2436 1.75 yamaguch ifp->if_xname);
2437 1.75 yamaguch /* what to do on failure? */
2438 1.1 hannken }
2439 1.75 yamaguch
2440 1.75 yamaguch ifp->if_flags |= IFF_ALLMULTI;
2441 1.1 hannken }
2442 1.75 yamaguch
2443 1.75 yamaguch set_ifflags:
2444 1.75 yamaguch r = vioif_ifflags(sc);
2445 1.1 hannken
2446 1.1 hannken return r;
2447 1.1 hannken }
2448 1.1 hannken
2449 1.82 knakahar static int
2450 1.82 knakahar vioif_get_link_status(struct vioif_softc *sc)
2451 1.34 ozaki {
2452 1.34 ozaki struct virtio_softc *vsc = sc->sc_virtio;
2453 1.34 ozaki uint16_t status;
2454 1.34 ozaki
2455 1.34 ozaki if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
2456 1.34 ozaki status = virtio_read_device_config_2(vsc,
2457 1.34 ozaki VIRTIO_NET_CONFIG_STATUS);
2458 1.34 ozaki else
2459 1.34 ozaki status = VIRTIO_NET_S_LINK_UP;
2460 1.34 ozaki
2461 1.82 knakahar if ((status & VIRTIO_NET_S_LINK_UP) != 0)
2462 1.82 knakahar return LINK_STATE_UP;
2463 1.82 knakahar
2464 1.82 knakahar return LINK_STATE_DOWN;
2465 1.34 ozaki }
2466 1.34 ozaki
2467 1.34 ozaki /* change link status */
2468 1.34 ozaki static void
2469 1.34 ozaki vioif_update_link_status(struct vioif_softc *sc)
2470 1.34 ozaki {
2471 1.34 ozaki struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2472 1.98 yamaguch struct vioif_netqueue *netq;
2473 1.98 yamaguch struct vioif_tx_context *txc;
2474 1.82 knakahar bool active;
2475 1.46 yamaguch int link, i;
2476 1.34 ozaki
2477 1.62 yamaguch mutex_enter(&sc->sc_lock);
2478 1.62 yamaguch
2479 1.82 knakahar link = vioif_get_link_status(sc);
2480 1.34 ozaki
2481 1.82 knakahar if (link == sc->sc_link_state)
2482 1.82 knakahar goto done;
2483 1.34 ozaki
2484 1.82 knakahar sc->sc_link_state = link;
2485 1.82 knakahar
2486 1.82 knakahar active = VIOIF_IS_LINK_ACTIVE(sc);
2487 1.82 knakahar for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
2488 1.98 yamaguch netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
2489 1.34 ozaki
2490 1.98 yamaguch mutex_enter(&netq->netq_lock);
2491 1.98 yamaguch txc = netq->netq_ctx;
2492 1.98 yamaguch txc->txc_link_active = active;
2493 1.98 yamaguch mutex_exit(&netq->netq_lock);
2494 1.34 ozaki }
2495 1.34 ozaki
2496 1.82 knakahar if_link_state_change(ifp, sc->sc_link_state);
2497 1.62 yamaguch
2498 1.82 knakahar done:
2499 1.62 yamaguch mutex_exit(&sc->sc_lock);
2500 1.34 ozaki }
2501 1.34 ozaki
2502 1.34 ozaki static int
2503 1.34 ozaki vioif_config_change(struct virtio_softc *vsc)
2504 1.34 ozaki {
2505 1.34 ozaki struct vioif_softc *sc = device_private(virtio_child(vsc));
2506 1.34 ozaki
2507 1.34 ozaki softint_schedule(sc->sc_ctl_softint);
2508 1.34 ozaki return 0;
2509 1.34 ozaki }
2510 1.34 ozaki
2511 1.34 ozaki static void
2512 1.34 ozaki vioif_ctl_softint(void *arg)
2513 1.34 ozaki {
2514 1.34 ozaki struct vioif_softc *sc = arg;
2515 1.34 ozaki struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2516 1.34 ozaki
2517 1.34 ozaki vioif_update_link_status(sc);
2518 1.34 ozaki vioif_start(ifp);
2519 1.34 ozaki }
2520 1.34 ozaki
2521 1.55 yamaguch static struct workqueue *
2522 1.55 yamaguch vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
2523 1.55 yamaguch {
2524 1.55 yamaguch struct workqueue *wq;
2525 1.55 yamaguch int error;
2526 1.55 yamaguch
2527 1.55 yamaguch error = workqueue_create(&wq, name, vioif_workq_work, NULL,
2528 1.55 yamaguch prio, ipl, flags);
2529 1.55 yamaguch
2530 1.55 yamaguch if (error)
2531 1.55 yamaguch return NULL;
2532 1.55 yamaguch
2533 1.55 yamaguch return wq;
2534 1.55 yamaguch }
2535 1.55 yamaguch
2536 1.55 yamaguch static void
2537 1.55 yamaguch vioif_workq_destroy(struct workqueue *wq)
2538 1.55 yamaguch {
2539 1.55 yamaguch
2540 1.55 yamaguch workqueue_destroy(wq);
2541 1.55 yamaguch }
2542 1.55 yamaguch
2543 1.55 yamaguch static void
2544 1.55 yamaguch vioif_workq_work(struct work *wk, void *context)
2545 1.55 yamaguch {
2546 1.55 yamaguch struct vioif_work *work;
2547 1.55 yamaguch
2548 1.55 yamaguch work = container_of(wk, struct vioif_work, cookie);
2549 1.55 yamaguch
2550 1.55 yamaguch atomic_store_relaxed(&work->added, 0);
2551 1.55 yamaguch work->func(work->arg);
2552 1.55 yamaguch }
2553 1.55 yamaguch
2554 1.55 yamaguch static void
2555 1.55 yamaguch vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
2556 1.55 yamaguch {
2557 1.55 yamaguch
2558 1.55 yamaguch memset(work, 0, sizeof(*work));
2559 1.55 yamaguch work->func = func;
2560 1.55 yamaguch work->arg = arg;
2561 1.55 yamaguch }
2562 1.55 yamaguch
2563 1.55 yamaguch static void
2564 1.55 yamaguch vioif_work_add(struct workqueue *wq, struct vioif_work *work)
2565 1.55 yamaguch {
2566 1.55 yamaguch
2567 1.55 yamaguch if (atomic_load_relaxed(&work->added) != 0)
2568 1.55 yamaguch return;
2569 1.55 yamaguch
2570 1.55 yamaguch atomic_store_relaxed(&work->added, 1);
2571 1.55 yamaguch kpreempt_disable();
2572 1.55 yamaguch workqueue_enqueue(wq, &work->cookie, NULL);
2573 1.55 yamaguch kpreempt_enable();
2574 1.55 yamaguch }
2575 1.55 yamaguch
2576 1.55 yamaguch static void
2577 1.55 yamaguch vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
2578 1.55 yamaguch {
2579 1.55 yamaguch
2580 1.55 yamaguch workqueue_wait(wq, &work->cookie);
2581 1.55 yamaguch }
2582 1.55 yamaguch
2583 1.55 yamaguch static int
2584 1.55 yamaguch vioif_setup_sysctl(struct vioif_softc *sc)
2585 1.55 yamaguch {
2586 1.55 yamaguch const char *devname;
2587 1.55 yamaguch struct sysctllog **log;
2588 1.55 yamaguch const struct sysctlnode *rnode, *rxnode, *txnode;
2589 1.55 yamaguch int error;
2590 1.55 yamaguch
2591 1.55 yamaguch log = &sc->sc_sysctllog;
2592 1.55 yamaguch devname = device_xname(sc->sc_dev);
2593 1.55 yamaguch
2594 1.55 yamaguch error = sysctl_createv(log, 0, NULL, &rnode,
2595 1.55 yamaguch 0, CTLTYPE_NODE, devname,
2596 1.55 yamaguch SYSCTL_DESCR("virtio-net information and settings"),
2597 1.55 yamaguch NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
2598 1.55 yamaguch if (error)
2599 1.55 yamaguch goto out;
2600 1.55 yamaguch
2601 1.55 yamaguch error = sysctl_createv(log, 0, &rnode, NULL,
2602 1.55 yamaguch CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
2603 1.55 yamaguch SYSCTL_DESCR("Use workqueue for packet processing"),
2604 1.55 yamaguch NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
2605 1.55 yamaguch if (error)
2606 1.55 yamaguch goto out;
2607 1.55 yamaguch
2608 1.55 yamaguch error = sysctl_createv(log, 0, &rnode, &rxnode,
2609 1.55 yamaguch 0, CTLTYPE_NODE, "rx",
2610 1.55 yamaguch SYSCTL_DESCR("virtio-net information and settings for Rx"),
2611 1.55 yamaguch NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2612 1.55 yamaguch if (error)
2613 1.55 yamaguch goto out;
2614 1.55 yamaguch
2615 1.55 yamaguch error = sysctl_createv(log, 0, &rxnode, NULL,
2616 1.55 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2617 1.55 yamaguch SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
2618 1.55 yamaguch NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2619 1.55 yamaguch if (error)
2620 1.55 yamaguch goto out;
2621 1.55 yamaguch
2622 1.55 yamaguch error = sysctl_createv(log, 0, &rxnode, NULL,
2623 1.55 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2624 1.55 yamaguch SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
2625 1.55 yamaguch NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
2626 1.55 yamaguch if (error)
2627 1.55 yamaguch goto out;
2628 1.55 yamaguch
2629 1.55 yamaguch error = sysctl_createv(log, 0, &rnode, &txnode,
2630 1.55 yamaguch 0, CTLTYPE_NODE, "tx",
2631 1.55 yamaguch SYSCTL_DESCR("virtio-net information and settings for Tx"),
2632 1.55 yamaguch NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
2633 1.55 yamaguch if (error)
2634 1.55 yamaguch goto out;
2635 1.55 yamaguch
2636 1.55 yamaguch error = sysctl_createv(log, 0, &txnode, NULL,
2637 1.55 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
2638 1.55 yamaguch SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
2639 1.55 yamaguch NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
2640 1.55 yamaguch if (error)
2641 1.55 yamaguch goto out;
2642 1.55 yamaguch
2643 1.55 yamaguch error = sysctl_createv(log, 0, &txnode, NULL,
2644 1.55 yamaguch CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
2645 1.55 yamaguch SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
2646 1.55 yamaguch NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
2647 1.55 yamaguch
2648 1.55 yamaguch out:
2649 1.55 yamaguch if (error)
2650 1.55 yamaguch sysctl_teardown(log);
2651 1.55 yamaguch
2652 1.55 yamaguch return error;
2653 1.55 yamaguch }
2654 1.55 yamaguch
2655 1.63 yamaguch static void
2656 1.63 yamaguch vioif_setup_stats(struct vioif_softc *sc)
2657 1.63 yamaguch {
2658 1.98 yamaguch struct vioif_netqueue *netq;
2659 1.98 yamaguch struct vioif_tx_context *txc;
2660 1.98 yamaguch struct vioif_rx_context *rxc;
2661 1.98 yamaguch size_t i, netq_num;
2662 1.98 yamaguch
2663 1.98 yamaguch netq_num = sc->sc_max_nvq_pairs * 2;
2664 1.98 yamaguch for (i = 0; i < netq_num; i++) {
2665 1.98 yamaguch netq = &sc->sc_netqs[i];
2666 1.98 yamaguch evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC,
2667 1.98 yamaguch NULL, netq->netq_evgroup, "failed to load mbuf to DMA");
2668 1.98 yamaguch evcnt_attach_dynamic(&netq->netq_enqueue_reserve_failed,
2669 1.98 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
2670 1.98 yamaguch "virtio_enqueue_reserve failed");
2671 1.98 yamaguch
2672 1.98 yamaguch switch (VIOIF_NETQ_DIR(i)) {
2673 1.98 yamaguch case VIOIF_NETQ_RX:
2674 1.98 yamaguch rxc = netq->netq_ctx;
2675 1.98 yamaguch evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs,
2676 1.98 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
2677 1.98 yamaguch "no receive buffer");
2678 1.98 yamaguch break;
2679 1.98 yamaguch case VIOIF_NETQ_TX:
2680 1.98 yamaguch txc = netq->netq_ctx;
2681 1.98 yamaguch evcnt_attach_dynamic(&txc->txc_defrag_failed,
2682 1.98 yamaguch EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
2683 1.98 yamaguch "m_defrag() failed");
2684 1.98 yamaguch break;
2685 1.98 yamaguch }
2686 1.63 yamaguch }
2687 1.63 yamaguch
2688 1.63 yamaguch evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC,
2689 1.65 riastrad NULL, device_xname(sc->sc_dev), "control command dmamap load failed");
2690 1.63 yamaguch evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC,
2691 1.65 riastrad NULL, device_xname(sc->sc_dev), "control command failed");
2692 1.63 yamaguch }
2693 1.63 yamaguch
2694 1.85 yamaguch static void
2695 1.85 yamaguch vioif_intr_barrier(void)
2696 1.85 yamaguch {
2697 1.85 yamaguch
2698 1.85 yamaguch /* wait for finish all interrupt handler */
2699 1.85 yamaguch xc_barrier(0);
2700 1.85 yamaguch }
2701 1.85 yamaguch
2702 1.26 pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
2703 1.48 msaitoh
2704 1.26 pgoyette #ifdef _MODULE
2705 1.26 pgoyette #include "ioconf.c"
2706 1.26 pgoyette #endif
2707 1.48 msaitoh
2708 1.48 msaitoh static int
2709 1.26 pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
2710 1.26 pgoyette {
2711 1.26 pgoyette int error = 0;
2712 1.48 msaitoh
2713 1.26 pgoyette #ifdef _MODULE
2714 1.26 pgoyette switch (cmd) {
2715 1.26 pgoyette case MODULE_CMD_INIT:
2716 1.48 msaitoh error = config_init_component(cfdriver_ioconf_if_vioif,
2717 1.48 msaitoh cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2718 1.26 pgoyette break;
2719 1.26 pgoyette case MODULE_CMD_FINI:
2720 1.26 pgoyette error = config_fini_component(cfdriver_ioconf_if_vioif,
2721 1.26 pgoyette cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2722 1.26 pgoyette break;
2723 1.26 pgoyette default:
2724 1.26 pgoyette error = ENOTTY;
2725 1.48 msaitoh break;
2726 1.26 pgoyette }
2727 1.26 pgoyette #endif
2728 1.48 msaitoh
2729 1.26 pgoyette return error;
2730 1.26 pgoyette }
2731