if_vioif.c revision 1.46 1 1.46 yamaguch /* $NetBSD: if_vioif.c,v 1.46 2019/01/14 15:00:27 yamaguchi Exp $ */
2 1.1 hannken
3 1.1 hannken /*
4 1.1 hannken * Copyright (c) 2010 Minoura Makoto.
5 1.1 hannken * All rights reserved.
6 1.1 hannken *
7 1.1 hannken * Redistribution and use in source and binary forms, with or without
8 1.1 hannken * modification, are permitted provided that the following conditions
9 1.1 hannken * are met:
10 1.1 hannken * 1. Redistributions of source code must retain the above copyright
11 1.1 hannken * notice, this list of conditions and the following disclaimer.
12 1.1 hannken * 2. Redistributions in binary form must reproduce the above copyright
13 1.1 hannken * notice, this list of conditions and the following disclaimer in the
14 1.1 hannken * documentation and/or other materials provided with the distribution.
15 1.1 hannken *
16 1.1 hannken * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 1.1 hannken * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 1.1 hannken * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 1.1 hannken * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 1.1 hannken * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 1.1 hannken * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 1.1 hannken * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 1.1 hannken * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 1.1 hannken * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 1.1 hannken * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 1.1 hannken */
27 1.1 hannken
28 1.1 hannken #include <sys/cdefs.h>
29 1.46 yamaguch __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.46 2019/01/14 15:00:27 yamaguchi Exp $");
30 1.15 ozaki
31 1.15 ozaki #ifdef _KERNEL_OPT
32 1.15 ozaki #include "opt_net_mpsafe.h"
33 1.15 ozaki #endif
34 1.1 hannken
35 1.1 hannken #include <sys/param.h>
36 1.1 hannken #include <sys/systm.h>
37 1.1 hannken #include <sys/kernel.h>
38 1.1 hannken #include <sys/bus.h>
39 1.1 hannken #include <sys/condvar.h>
40 1.1 hannken #include <sys/device.h>
41 1.1 hannken #include <sys/intr.h>
42 1.1 hannken #include <sys/kmem.h>
43 1.1 hannken #include <sys/mbuf.h>
44 1.1 hannken #include <sys/mutex.h>
45 1.1 hannken #include <sys/sockio.h>
46 1.12 ozaki #include <sys/cpu.h>
47 1.26 pgoyette #include <sys/module.h>
48 1.46 yamaguch #include <sys/pcq.h>
49 1.1 hannken
50 1.1 hannken #include <dev/pci/virtioreg.h>
51 1.1 hannken #include <dev/pci/virtiovar.h>
52 1.1 hannken
53 1.1 hannken #include <net/if.h>
54 1.1 hannken #include <net/if_media.h>
55 1.1 hannken #include <net/if_ether.h>
56 1.1 hannken
57 1.1 hannken #include <net/bpf.h>
58 1.1 hannken
59 1.26 pgoyette #include "ioconf.h"
60 1.1 hannken
61 1.7 ozaki #ifdef NET_MPSAFE
62 1.7 ozaki #define VIOIF_MPSAFE 1
63 1.46 yamaguch #define VIOIF_MULTIQ 1
64 1.7 ozaki #endif
65 1.7 ozaki
66 1.12 ozaki #ifdef SOFTINT_INTR
67 1.12 ozaki #define VIOIF_SOFTINT_INTR 1
68 1.12 ozaki #endif
69 1.12 ozaki
70 1.1 hannken /*
71 1.1 hannken * if_vioifreg.h:
72 1.1 hannken */
73 1.1 hannken /* Configuration registers */
74 1.1 hannken #define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
75 1.1 hannken #define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
76 1.46 yamaguch #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
77 1.1 hannken
78 1.1 hannken /* Feature bits */
79 1.46 yamaguch #define VIRTIO_NET_F_CSUM __BIT(0)
80 1.46 yamaguch #define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
81 1.46 yamaguch #define VIRTIO_NET_F_MAC __BIT(5)
82 1.46 yamaguch #define VIRTIO_NET_F_GSO __BIT(6)
83 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
84 1.46 yamaguch #define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
85 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ECN __BIT(9)
86 1.46 yamaguch #define VIRTIO_NET_F_GUEST_UFO __BIT(10)
87 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
88 1.46 yamaguch #define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
89 1.46 yamaguch #define VIRTIO_NET_F_HOST_ECN __BIT(13)
90 1.46 yamaguch #define VIRTIO_NET_F_HOST_UFO __BIT(14)
91 1.46 yamaguch #define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
92 1.46 yamaguch #define VIRTIO_NET_F_STATUS __BIT(16)
93 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VQ __BIT(17)
94 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX __BIT(18)
95 1.46 yamaguch #define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
96 1.46 yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
97 1.46 yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
98 1.46 yamaguch #define VIRTIO_NET_F_MQ __BIT(22)
99 1.1 hannken
100 1.18 christos #define VIRTIO_NET_FLAG_BITS \
101 1.18 christos VIRTIO_COMMON_FLAG_BITS \
102 1.46 yamaguch "\x17""MQ" \
103 1.46 yamaguch "\x16""GUEST_ANNOUNCE" \
104 1.46 yamaguch "\x15""CTRL_RX_EXTRA" \
105 1.18 christos "\x14""CTRL_VLAN" \
106 1.18 christos "\x13""CTRL_RX" \
107 1.18 christos "\x12""CTRL_VQ" \
108 1.18 christos "\x11""STATUS" \
109 1.18 christos "\x10""MRG_RXBUF" \
110 1.18 christos "\x0f""HOST_UFO" \
111 1.18 christos "\x0e""HOST_ECN" \
112 1.18 christos "\x0d""HOST_TSO6" \
113 1.18 christos "\x0c""HOST_TSO4" \
114 1.18 christos "\x0b""GUEST_UFO" \
115 1.18 christos "\x0a""GUEST_ECN" \
116 1.18 christos "\x09""GUEST_TSO6" \
117 1.18 christos "\x08""GUEST_TSO4" \
118 1.18 christos "\x07""GSO" \
119 1.18 christos "\x06""MAC" \
120 1.18 christos "\x02""GUEST_CSUM" \
121 1.18 christos "\x01""CSUM"
122 1.18 christos
123 1.1 hannken /* Status */
124 1.1 hannken #define VIRTIO_NET_S_LINK_UP 1
125 1.1 hannken
126 1.1 hannken /* Packet header structure */
127 1.1 hannken struct virtio_net_hdr {
128 1.1 hannken uint8_t flags;
129 1.1 hannken uint8_t gso_type;
130 1.1 hannken uint16_t hdr_len;
131 1.1 hannken uint16_t gso_size;
132 1.1 hannken uint16_t csum_start;
133 1.1 hannken uint16_t csum_offset;
134 1.1 hannken #if 0
135 1.1 hannken uint16_t num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */
136 1.1 hannken #endif
137 1.1 hannken } __packed;
138 1.1 hannken
139 1.1 hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM 1 /* flags */
140 1.1 hannken #define VIRTIO_NET_HDR_GSO_NONE 0 /* gso_type */
141 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV4 1 /* gso_type */
142 1.1 hannken #define VIRTIO_NET_HDR_GSO_UDP 3 /* gso_type */
143 1.1 hannken #define VIRTIO_NET_HDR_GSO_TCPV6 4 /* gso_type */
144 1.1 hannken #define VIRTIO_NET_HDR_GSO_ECN 0x80 /* gso_type, |'ed */
145 1.1 hannken
146 1.1 hannken #define VIRTIO_NET_MAX_GSO_LEN (65536+ETHER_HDR_LEN)
147 1.1 hannken
148 1.1 hannken /* Control virtqueue */
149 1.1 hannken struct virtio_net_ctrl_cmd {
150 1.1 hannken uint8_t class;
151 1.1 hannken uint8_t command;
152 1.1 hannken } __packed;
153 1.1 hannken #define VIRTIO_NET_CTRL_RX 0
154 1.1 hannken # define VIRTIO_NET_CTRL_RX_PROMISC 0
155 1.1 hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI 1
156 1.1 hannken
157 1.1 hannken #define VIRTIO_NET_CTRL_MAC 1
158 1.1 hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET 0
159 1.1 hannken
160 1.1 hannken #define VIRTIO_NET_CTRL_VLAN 2
161 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_ADD 0
162 1.1 hannken # define VIRTIO_NET_CTRL_VLAN_DEL 1
163 1.1 hannken
164 1.46 yamaguch #define VIRTIO_NET_CTRL_MQ 4
165 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
166 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
167 1.46 yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
168 1.46 yamaguch
169 1.1 hannken struct virtio_net_ctrl_status {
170 1.1 hannken uint8_t ack;
171 1.1 hannken } __packed;
172 1.1 hannken #define VIRTIO_NET_OK 0
173 1.1 hannken #define VIRTIO_NET_ERR 1
174 1.1 hannken
175 1.1 hannken struct virtio_net_ctrl_rx {
176 1.1 hannken uint8_t onoff;
177 1.1 hannken } __packed;
178 1.1 hannken
179 1.1 hannken struct virtio_net_ctrl_mac_tbl {
180 1.1 hannken uint32_t nentries;
181 1.1 hannken uint8_t macs[][ETHER_ADDR_LEN];
182 1.1 hannken } __packed;
183 1.1 hannken
184 1.1 hannken struct virtio_net_ctrl_vlan {
185 1.1 hannken uint16_t id;
186 1.1 hannken } __packed;
187 1.1 hannken
188 1.46 yamaguch struct virtio_net_ctrl_mq {
189 1.46 yamaguch uint16_t virtqueue_pairs;
190 1.46 yamaguch } __packed;
191 1.46 yamaguch
192 1.44 yamaguch struct vioif_ctrl_cmdspec {
193 1.44 yamaguch bus_dmamap_t dmamap;
194 1.44 yamaguch void *buf;
195 1.44 yamaguch bus_size_t bufsize;
196 1.44 yamaguch };
197 1.1 hannken
198 1.1 hannken /*
199 1.1 hannken * if_vioifvar.h:
200 1.1 hannken */
201 1.43 yamaguch
202 1.43 yamaguch /*
203 1.43 yamaguch * Locking notes:
204 1.43 yamaguch * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and
205 1.43 yamaguch * a filds in vioif_rxqueue is protected by rxq_lock (a spin mutex).
206 1.43 yamaguch * - more than one lock cannot be held at onece
207 1.43 yamaguch * + ctrlq_inuse is protected by ctrlq_wait_lock.
208 1.43 yamaguch * - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
209 1.43 yamaguch * - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock
210 1.43 yamaguch */
211 1.43 yamaguch
212 1.43 yamaguch struct vioif_txqueue {
213 1.43 yamaguch kmutex_t *txq_lock; /* lock for tx operations */
214 1.43 yamaguch
215 1.43 yamaguch struct virtqueue *txq_vq;
216 1.43 yamaguch bool txq_stopping;
217 1.43 yamaguch bool txq_link_active;
218 1.46 yamaguch pcq_t *txq_intrq;
219 1.43 yamaguch
220 1.43 yamaguch struct virtio_net_hdr *txq_hdrs;
221 1.43 yamaguch bus_dmamap_t *txq_hdr_dmamaps;
222 1.43 yamaguch
223 1.43 yamaguch struct mbuf **txq_mbufs;
224 1.43 yamaguch bus_dmamap_t *txq_dmamaps;
225 1.46 yamaguch
226 1.46 yamaguch void *txq_deferred_transmit;
227 1.43 yamaguch };
228 1.43 yamaguch
229 1.43 yamaguch struct vioif_rxqueue {
230 1.43 yamaguch kmutex_t *rxq_lock; /* lock for rx operations */
231 1.43 yamaguch
232 1.43 yamaguch struct virtqueue *rxq_vq;
233 1.43 yamaguch bool rxq_stopping;
234 1.43 yamaguch
235 1.43 yamaguch struct virtio_net_hdr *rxq_hdrs;
236 1.43 yamaguch bus_dmamap_t *rxq_hdr_dmamaps;
237 1.43 yamaguch
238 1.43 yamaguch struct mbuf **rxq_mbufs;
239 1.43 yamaguch bus_dmamap_t *rxq_dmamaps;
240 1.43 yamaguch
241 1.43 yamaguch void *rxq_softint;
242 1.43 yamaguch };
243 1.43 yamaguch
244 1.43 yamaguch struct vioif_ctrlqueue {
245 1.43 yamaguch struct virtqueue *ctrlq_vq;
246 1.43 yamaguch enum {
247 1.43 yamaguch FREE, INUSE, DONE
248 1.43 yamaguch } ctrlq_inuse;
249 1.43 yamaguch kcondvar_t ctrlq_wait;
250 1.43 yamaguch kmutex_t ctrlq_wait_lock;
251 1.44 yamaguch struct lwp *ctrlq_owner;
252 1.43 yamaguch
253 1.43 yamaguch struct virtio_net_ctrl_cmd *ctrlq_cmd;
254 1.43 yamaguch struct virtio_net_ctrl_status *ctrlq_status;
255 1.43 yamaguch struct virtio_net_ctrl_rx *ctrlq_rx;
256 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
257 1.43 yamaguch struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
258 1.46 yamaguch struct virtio_net_ctrl_mq *ctrlq_mq;
259 1.43 yamaguch
260 1.43 yamaguch bus_dmamap_t ctrlq_cmd_dmamap;
261 1.43 yamaguch bus_dmamap_t ctrlq_status_dmamap;
262 1.43 yamaguch bus_dmamap_t ctrlq_rx_dmamap;
263 1.43 yamaguch bus_dmamap_t ctrlq_tbl_uc_dmamap;
264 1.43 yamaguch bus_dmamap_t ctrlq_tbl_mc_dmamap;
265 1.46 yamaguch bus_dmamap_t ctrlq_mq_dmamap;
266 1.43 yamaguch };
267 1.43 yamaguch
268 1.1 hannken struct vioif_softc {
269 1.1 hannken device_t sc_dev;
270 1.1 hannken
271 1.1 hannken struct virtio_softc *sc_virtio;
272 1.46 yamaguch struct virtqueue *sc_vqs;
273 1.46 yamaguch
274 1.46 yamaguch int sc_max_nvq_pairs;
275 1.46 yamaguch int sc_req_nvq_pairs;
276 1.46 yamaguch int sc_act_nvq_pairs;
277 1.1 hannken
278 1.1 hannken uint8_t sc_mac[ETHER_ADDR_LEN];
279 1.1 hannken struct ethercom sc_ethercom;
280 1.8 pooka short sc_deferred_init_done;
281 1.34 ozaki bool sc_link_active;
282 1.1 hannken
283 1.46 yamaguch struct vioif_txqueue *sc_txq;
284 1.46 yamaguch struct vioif_rxqueue *sc_rxq;
285 1.43 yamaguch
286 1.43 yamaguch bool sc_has_ctrl;
287 1.43 yamaguch struct vioif_ctrlqueue sc_ctrlq;
288 1.43 yamaguch
289 1.1 hannken bus_dma_segment_t sc_hdr_segs[1];
290 1.43 yamaguch void *sc_dmamem;
291 1.43 yamaguch void *sc_kmem;
292 1.1 hannken
293 1.34 ozaki void *sc_ctl_softint;
294 1.1 hannken };
295 1.1 hannken #define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
296 1.1 hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
297 1.1 hannken
298 1.1 hannken /* cfattach interface functions */
299 1.1 hannken static int vioif_match(device_t, cfdata_t, void *);
300 1.1 hannken static void vioif_attach(device_t, device_t, void *);
301 1.1 hannken static void vioif_deferred_init(device_t);
302 1.1 hannken
303 1.1 hannken /* ifnet interface functions */
304 1.1 hannken static int vioif_init(struct ifnet *);
305 1.1 hannken static void vioif_stop(struct ifnet *, int);
306 1.1 hannken static void vioif_start(struct ifnet *);
307 1.46 yamaguch static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
308 1.46 yamaguch static int vioif_transmit(struct ifnet *, struct mbuf *);
309 1.46 yamaguch static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
310 1.1 hannken static int vioif_ioctl(struct ifnet *, u_long, void *);
311 1.1 hannken static void vioif_watchdog(struct ifnet *);
312 1.1 hannken
313 1.1 hannken /* rx */
314 1.46 yamaguch static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
315 1.46 yamaguch static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
316 1.46 yamaguch static void vioif_populate_rx_mbufs(struct vioif_rxqueue *);
317 1.46 yamaguch static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *);
318 1.46 yamaguch static int vioif_rx_deq(struct vioif_rxqueue *);
319 1.46 yamaguch static int vioif_rx_deq_locked(struct vioif_rxqueue *);
320 1.1 hannken static int vioif_rx_vq_done(struct virtqueue *);
321 1.1 hannken static void vioif_rx_softint(void *);
322 1.46 yamaguch static void vioif_rx_drain(struct vioif_rxqueue *);
323 1.1 hannken
324 1.1 hannken /* tx */
325 1.1 hannken static int vioif_tx_vq_done(struct virtqueue *);
326 1.7 ozaki static int vioif_tx_vq_done_locked(struct virtqueue *);
327 1.46 yamaguch static void vioif_tx_drain(struct vioif_txqueue *);
328 1.46 yamaguch static void vioif_deferred_transmit(void *);
329 1.1 hannken
330 1.1 hannken /* other control */
331 1.34 ozaki static bool vioif_is_link_up(struct vioif_softc *);
332 1.34 ozaki static void vioif_update_link_status(struct vioif_softc *);
333 1.1 hannken static int vioif_ctrl_rx(struct vioif_softc *, int, bool);
334 1.1 hannken static int vioif_set_promisc(struct vioif_softc *, bool);
335 1.1 hannken static int vioif_set_allmulti(struct vioif_softc *, bool);
336 1.1 hannken static int vioif_set_rx_filter(struct vioif_softc *);
337 1.1 hannken static int vioif_rx_filter(struct vioif_softc *);
338 1.1 hannken static int vioif_ctrl_vq_done(struct virtqueue *);
339 1.34 ozaki static int vioif_config_change(struct virtio_softc *);
340 1.34 ozaki static void vioif_ctl_softint(void *);
341 1.46 yamaguch static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
342 1.46 yamaguch static void vioif_enable_interrupt_vqpairs(struct vioif_softc *);
343 1.46 yamaguch static void vioif_disable_interrupt_vqpairs(struct vioif_softc *);
344 1.1 hannken
345 1.1 hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
346 1.1 hannken vioif_match, vioif_attach, NULL, NULL);
347 1.1 hannken
348 1.1 hannken static int
349 1.1 hannken vioif_match(device_t parent, cfdata_t match, void *aux)
350 1.1 hannken {
351 1.32 jdolecek struct virtio_attach_args *va = aux;
352 1.1 hannken
353 1.1 hannken if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK)
354 1.1 hannken return 1;
355 1.1 hannken
356 1.1 hannken return 0;
357 1.1 hannken }
358 1.1 hannken
359 1.46 yamaguch static int
360 1.46 yamaguch vioif_alloc_queues(struct vioif_softc *sc)
361 1.46 yamaguch {
362 1.46 yamaguch int nvq_pairs = sc->sc_max_nvq_pairs;
363 1.46 yamaguch int nvqs = nvq_pairs * 2;
364 1.46 yamaguch int i;
365 1.46 yamaguch
366 1.46 yamaguch KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
367 1.46 yamaguch
368 1.46 yamaguch sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
369 1.46 yamaguch KM_NOSLEEP);
370 1.46 yamaguch if (sc->sc_rxq == NULL)
371 1.46 yamaguch return -1;
372 1.46 yamaguch
373 1.46 yamaguch sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
374 1.46 yamaguch KM_NOSLEEP);
375 1.46 yamaguch if (sc->sc_txq == NULL)
376 1.46 yamaguch return -1;
377 1.46 yamaguch
378 1.46 yamaguch if (sc->sc_has_ctrl)
379 1.46 yamaguch nvqs++;
380 1.46 yamaguch
381 1.46 yamaguch sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_NOSLEEP);
382 1.46 yamaguch if (sc->sc_vqs == NULL)
383 1.46 yamaguch return -1;
384 1.46 yamaguch
385 1.46 yamaguch nvqs = 0;
386 1.46 yamaguch for (i = 0; i < nvq_pairs; i++) {
387 1.46 yamaguch sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
388 1.46 yamaguch sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
389 1.46 yamaguch }
390 1.46 yamaguch
391 1.46 yamaguch if (sc->sc_has_ctrl)
392 1.46 yamaguch sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
393 1.46 yamaguch
394 1.46 yamaguch return 0;
395 1.46 yamaguch }
396 1.46 yamaguch
397 1.46 yamaguch static void
398 1.46 yamaguch vioif_free_queues(struct vioif_softc *sc)
399 1.46 yamaguch {
400 1.46 yamaguch int nvq_pairs = sc->sc_max_nvq_pairs;
401 1.46 yamaguch int nvqs = nvq_pairs * 2;
402 1.46 yamaguch
403 1.46 yamaguch if (sc->sc_ctrlq.ctrlq_vq)
404 1.46 yamaguch nvqs++;
405 1.46 yamaguch
406 1.46 yamaguch if (sc->sc_txq) {
407 1.46 yamaguch kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
408 1.46 yamaguch sc->sc_txq = NULL;
409 1.46 yamaguch }
410 1.46 yamaguch
411 1.46 yamaguch if (sc->sc_rxq) {
412 1.46 yamaguch kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
413 1.46 yamaguch sc->sc_rxq = NULL;
414 1.46 yamaguch }
415 1.46 yamaguch
416 1.46 yamaguch if (sc->sc_vqs) {
417 1.46 yamaguch kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
418 1.46 yamaguch sc->sc_vqs = NULL;
419 1.46 yamaguch }
420 1.46 yamaguch }
421 1.46 yamaguch
422 1.1 hannken /* allocate memory */
423 1.1 hannken /*
424 1.1 hannken * dma memory is used for:
425 1.43 yamaguch * rxq_hdrs[slot]: metadata array for received frames (READ)
426 1.43 yamaguch * txq_hdrs[slot]: metadata array for frames to be sent (WRITE)
427 1.43 yamaguch * ctrlq_cmd: command to be sent via ctrl vq (WRITE)
428 1.43 yamaguch * ctrlq_status: return value for a command via ctrl vq (READ)
429 1.43 yamaguch * ctrlq_rx: parameter for a VIRTIO_NET_CTRL_RX class command
430 1.1 hannken * (WRITE)
431 1.43 yamaguch * ctrlq_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
432 1.1 hannken * class command (WRITE)
433 1.43 yamaguch * ctrlq_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
434 1.1 hannken * class command (WRITE)
435 1.43 yamaguch * ctrlq_* structures are allocated only one each; they are protected by
436 1.43 yamaguch * ctrlq_inuse variable and ctrlq_wait condvar.
437 1.1 hannken */
438 1.1 hannken /*
439 1.1 hannken * dynamically allocated memory is used for:
440 1.43 yamaguch * rxq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_rx_hdrs[slot]
441 1.43 yamaguch * txq_hdr_dmamaps[slot]: bus_dmamap_t array for sc_tx_hdrs[slot]
442 1.43 yamaguch * rxq_dmamaps[slot]: bus_dmamap_t array for received payload
443 1.43 yamaguch * txq_dmamaps[slot]: bus_dmamap_t array for sent payload
444 1.43 yamaguch * rxq_mbufs[slot]: mbuf pointer array for received frames
445 1.43 yamaguch * txq_mbufs[slot]: mbuf pointer array for sent frames
446 1.1 hannken */
447 1.1 hannken static int
448 1.1 hannken vioif_alloc_mems(struct vioif_softc *sc)
449 1.1 hannken {
450 1.1 hannken struct virtio_softc *vsc = sc->sc_virtio;
451 1.46 yamaguch struct vioif_txqueue *txq;
452 1.46 yamaguch struct vioif_rxqueue *rxq;
453 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
454 1.46 yamaguch int allocsize, allocsize2, r, rsegs, i, qid;
455 1.1 hannken void *vaddr;
456 1.1 hannken intptr_t p;
457 1.1 hannken
458 1.46 yamaguch allocsize = 0;
459 1.46 yamaguch for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
460 1.46 yamaguch rxq = &sc->sc_rxq[qid];
461 1.46 yamaguch txq = &sc->sc_txq[qid];
462 1.46 yamaguch
463 1.46 yamaguch allocsize +=
464 1.46 yamaguch sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num;
465 1.46 yamaguch allocsize +=
466 1.46 yamaguch sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num;
467 1.46 yamaguch }
468 1.32 jdolecek if (sc->sc_has_ctrl) {
469 1.1 hannken allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1;
470 1.1 hannken allocsize += sizeof(struct virtio_net_ctrl_status) * 1;
471 1.1 hannken allocsize += sizeof(struct virtio_net_ctrl_rx) * 1;
472 1.1 hannken allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
473 1.1 hannken + sizeof(struct virtio_net_ctrl_mac_tbl)
474 1.1 hannken + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
475 1.46 yamaguch allocsize += sizeof(struct virtio_net_ctrl_mq) * 1;
476 1.1 hannken }
477 1.32 jdolecek r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0,
478 1.1 hannken &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
479 1.1 hannken if (r != 0) {
480 1.1 hannken aprint_error_dev(sc->sc_dev,
481 1.1 hannken "DMA memory allocation failed, size %d, "
482 1.1 hannken "error code %d\n", allocsize, r);
483 1.1 hannken goto err_none;
484 1.1 hannken }
485 1.32 jdolecek r = bus_dmamem_map(virtio_dmat(vsc),
486 1.1 hannken &sc->sc_hdr_segs[0], 1, allocsize,
487 1.1 hannken &vaddr, BUS_DMA_NOWAIT);
488 1.1 hannken if (r != 0) {
489 1.1 hannken aprint_error_dev(sc->sc_dev,
490 1.1 hannken "DMA memory map failed, "
491 1.1 hannken "error code %d\n", r);
492 1.1 hannken goto err_dmamem_alloc;
493 1.1 hannken }
494 1.42 yamaguch
495 1.42 yamaguch #define P(p, p0, p0size) do { p0 = (void *) p; \
496 1.42 yamaguch p += p0size; } while (0)
497 1.1 hannken memset(vaddr, 0, allocsize);
498 1.43 yamaguch sc->sc_dmamem = vaddr;
499 1.1 hannken p = (intptr_t) vaddr;
500 1.42 yamaguch
501 1.46 yamaguch for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
502 1.46 yamaguch rxq = &sc->sc_rxq[qid];
503 1.46 yamaguch txq = &sc->sc_txq[qid];
504 1.46 yamaguch
505 1.46 yamaguch P(p, rxq->rxq_hdrs,
506 1.46 yamaguch sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num);
507 1.46 yamaguch P(p, txq->txq_hdrs,
508 1.46 yamaguch sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num);
509 1.46 yamaguch }
510 1.32 jdolecek if (sc->sc_has_ctrl) {
511 1.43 yamaguch P(p, ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd));
512 1.43 yamaguch P(p, ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status));
513 1.43 yamaguch P(p, ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx));
514 1.43 yamaguch P(p, ctrlq->ctrlq_mac_tbl_uc, sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0);
515 1.43 yamaguch P(p, ctrlq->ctrlq_mac_tbl_mc,
516 1.43 yamaguch (sizeof(*ctrlq->ctrlq_mac_tbl_mc)
517 1.42 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES));
518 1.46 yamaguch P(p, ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq));
519 1.1 hannken }
520 1.1 hannken
521 1.46 yamaguch allocsize2 = 0;
522 1.46 yamaguch for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
523 1.46 yamaguch int rxqsize, txqsize;
524 1.46 yamaguch
525 1.46 yamaguch rxq = &sc->sc_rxq[qid];
526 1.46 yamaguch txq = &sc->sc_txq[qid];
527 1.46 yamaguch rxqsize = rxq->rxq_vq->vq_num;
528 1.46 yamaguch txqsize = txq->txq_vq->vq_num;
529 1.46 yamaguch
530 1.46 yamaguch allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize;
531 1.46 yamaguch allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize;
532 1.46 yamaguch allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize;
533 1.46 yamaguch
534 1.46 yamaguch allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize;
535 1.46 yamaguch allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize;
536 1.46 yamaguch allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize;
537 1.46 yamaguch }
538 1.42 yamaguch vaddr = kmem_zalloc(allocsize2, KM_SLEEP);
539 1.43 yamaguch sc->sc_kmem = vaddr;
540 1.42 yamaguch p = (intptr_t) vaddr;
541 1.42 yamaguch
542 1.46 yamaguch for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
543 1.46 yamaguch int rxqsize, txqsize;
544 1.46 yamaguch rxq = &sc->sc_rxq[qid];
545 1.46 yamaguch txq = &sc->sc_txq[qid];
546 1.46 yamaguch rxqsize = rxq->rxq_vq->vq_num;
547 1.46 yamaguch txqsize = txq->txq_vq->vq_num;
548 1.46 yamaguch
549 1.46 yamaguch P(p, rxq->rxq_hdr_dmamaps, sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize);
550 1.46 yamaguch P(p, txq->txq_hdr_dmamaps, sizeof(txq->txq_hdr_dmamaps[0]) * txqsize);
551 1.46 yamaguch P(p, rxq->rxq_dmamaps, sizeof(rxq->rxq_dmamaps[0]) * rxqsize);
552 1.46 yamaguch P(p, txq->txq_dmamaps, sizeof(txq->txq_dmamaps[0]) * txqsize);
553 1.46 yamaguch P(p, rxq->rxq_mbufs, sizeof(rxq->rxq_mbufs[0]) * rxqsize);
554 1.46 yamaguch P(p, txq->txq_mbufs, sizeof(txq->txq_mbufs[0]) * txqsize);
555 1.46 yamaguch }
556 1.42 yamaguch #undef P
557 1.1 hannken
558 1.42 yamaguch #define C(map, size, nsegs, usage) \
559 1.42 yamaguch do { \
560 1.42 yamaguch r = bus_dmamap_create(virtio_dmat(vsc), size, nsegs, size, 0, \
561 1.42 yamaguch BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW, \
562 1.42 yamaguch &map); \
563 1.42 yamaguch if (r != 0) { \
564 1.42 yamaguch aprint_error_dev(sc->sc_dev, \
565 1.42 yamaguch "%s dmamap creation failed, " \
566 1.42 yamaguch "error code %d\n", usage, r); \
567 1.42 yamaguch goto err_reqs; \
568 1.42 yamaguch } \
569 1.1 hannken } while (0)
570 1.42 yamaguch #define C_L(map, buf, size, nsegs, rw, usage) \
571 1.42 yamaguch C(map, size, nsegs, usage); \
572 1.1 hannken do { \
573 1.42 yamaguch r = bus_dmamap_load(virtio_dmat(vsc), map, \
574 1.42 yamaguch buf, size, NULL, \
575 1.42 yamaguch rw | BUS_DMA_NOWAIT); \
576 1.1 hannken if (r != 0) { \
577 1.1 hannken aprint_error_dev(sc->sc_dev, \
578 1.1 hannken usage " dmamap load failed, " \
579 1.1 hannken "error code %d\n", r); \
580 1.1 hannken goto err_reqs; \
581 1.1 hannken } \
582 1.1 hannken } while (0)
583 1.1 hannken
584 1.46 yamaguch for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
585 1.46 yamaguch rxq = &sc->sc_rxq[qid];
586 1.46 yamaguch txq = &sc->sc_txq[qid];
587 1.46 yamaguch
588 1.46 yamaguch for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
589 1.46 yamaguch C_L(rxq->rxq_hdr_dmamaps[i], &rxq->rxq_hdrs[i],
590 1.46 yamaguch sizeof(rxq->rxq_hdrs[0]), 1,
591 1.46 yamaguch BUS_DMA_READ, "rx header");
592 1.46 yamaguch C(rxq->rxq_dmamaps[i], MCLBYTES, 1, "rx payload");
593 1.46 yamaguch }
594 1.46 yamaguch
595 1.46 yamaguch for (i = 0; i < txq->txq_vq->vq_num; i++) {
596 1.46 yamaguch C_L(txq->txq_hdr_dmamaps[i], &txq->txq_hdrs[i],
597 1.46 yamaguch sizeof(txq->txq_hdrs[0]), 1,
598 1.46 yamaguch BUS_DMA_READ, "tx header");
599 1.46 yamaguch C(txq->txq_dmamaps[i], ETHER_MAX_LEN,
600 1.46 yamaguch VIRTIO_NET_TX_MAXNSEGS, "tx payload");
601 1.46 yamaguch }
602 1.1 hannken }
603 1.1 hannken
604 1.32 jdolecek if (sc->sc_has_ctrl) {
605 1.1 hannken /* control vq class & command */
606 1.43 yamaguch C_L(ctrlq->ctrlq_cmd_dmamap,
607 1.43 yamaguch ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
608 1.42 yamaguch BUS_DMA_WRITE, "control command");
609 1.43 yamaguch C_L(ctrlq->ctrlq_status_dmamap,
610 1.43 yamaguch ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
611 1.42 yamaguch BUS_DMA_READ, "control status");
612 1.1 hannken
613 1.1 hannken /* control vq rx mode command parameter */
614 1.43 yamaguch C_L(ctrlq->ctrlq_rx_dmamap,
615 1.43 yamaguch ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
616 1.42 yamaguch BUS_DMA_WRITE, "rx mode control command");
617 1.1 hannken
618 1.46 yamaguch /* multiqueue set command */
619 1.46 yamaguch C_L(ctrlq->ctrlq_mq_dmamap,
620 1.46 yamaguch ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
621 1.46 yamaguch BUS_DMA_WRITE, "multiqueue set command");
622 1.46 yamaguch
623 1.1 hannken /* control vq MAC filter table for unicast */
624 1.1 hannken /* do not load now since its length is variable */
625 1.43 yamaguch C(ctrlq->ctrlq_tbl_uc_dmamap,
626 1.43 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1,
627 1.42 yamaguch "unicast MAC address filter command");
628 1.1 hannken
629 1.1 hannken /* control vq MAC filter table for multicast */
630 1.43 yamaguch C(ctrlq->ctrlq_tbl_mc_dmamap,
631 1.43 yamaguch sizeof(*ctrlq->ctrlq_mac_tbl_mc)
632 1.42 yamaguch + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
633 1.42 yamaguch "multicast MAC address filter command");
634 1.1 hannken }
635 1.42 yamaguch #undef C_L
636 1.1 hannken #undef C
637 1.1 hannken
638 1.1 hannken return 0;
639 1.1 hannken
640 1.1 hannken err_reqs:
641 1.1 hannken #define D(map) \
642 1.1 hannken do { \
643 1.42 yamaguch if (map) { \
644 1.42 yamaguch bus_dmamap_destroy(virtio_dmat(vsc), map); \
645 1.42 yamaguch map = NULL; \
646 1.1 hannken } \
647 1.1 hannken } while (0)
648 1.43 yamaguch D(ctrlq->ctrlq_tbl_mc_dmamap);
649 1.43 yamaguch D(ctrlq->ctrlq_tbl_uc_dmamap);
650 1.43 yamaguch D(ctrlq->ctrlq_rx_dmamap);
651 1.43 yamaguch D(ctrlq->ctrlq_status_dmamap);
652 1.43 yamaguch D(ctrlq->ctrlq_cmd_dmamap);
653 1.46 yamaguch for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
654 1.46 yamaguch rxq = &sc->sc_rxq[qid];
655 1.46 yamaguch txq = &sc->sc_txq[qid];
656 1.46 yamaguch
657 1.46 yamaguch for (i = 0; i < txq->txq_vq->vq_num; i++) {
658 1.46 yamaguch D(txq->txq_dmamaps[i]);
659 1.46 yamaguch D(txq->txq_hdr_dmamaps[i]);
660 1.46 yamaguch }
661 1.46 yamaguch for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
662 1.46 yamaguch D(rxq->rxq_dmamaps[i]);
663 1.46 yamaguch D(rxq->rxq_hdr_dmamaps[i]);
664 1.46 yamaguch }
665 1.1 hannken }
666 1.1 hannken #undef D
667 1.43 yamaguch if (sc->sc_kmem) {
668 1.43 yamaguch kmem_free(sc->sc_kmem, allocsize2);
669 1.43 yamaguch sc->sc_kmem = NULL;
670 1.1 hannken }
671 1.43 yamaguch bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize);
672 1.1 hannken err_dmamem_alloc:
673 1.32 jdolecek bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
674 1.1 hannken err_none:
675 1.1 hannken return -1;
676 1.1 hannken }
677 1.1 hannken
678 1.1 hannken static void
679 1.1 hannken vioif_attach(device_t parent, device_t self, void *aux)
680 1.1 hannken {
681 1.1 hannken struct vioif_softc *sc = device_private(self);
682 1.1 hannken struct virtio_softc *vsc = device_private(parent);
683 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
684 1.46 yamaguch struct vioif_txqueue *txq;
685 1.46 yamaguch struct vioif_rxqueue *rxq;
686 1.46 yamaguch uint32_t features, req_features;
687 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
688 1.43 yamaguch u_int softint_flags;
689 1.46 yamaguch int r, i, nvqs=0, req_flags;
690 1.1 hannken
691 1.32 jdolecek if (virtio_child(vsc) != NULL) {
692 1.1 hannken aprint_normal(": child already attached for %s; "
693 1.1 hannken "something wrong...\n",
694 1.1 hannken device_xname(parent));
695 1.1 hannken return;
696 1.1 hannken }
697 1.1 hannken
698 1.1 hannken sc->sc_dev = self;
699 1.1 hannken sc->sc_virtio = vsc;
700 1.34 ozaki sc->sc_link_active = false;
701 1.1 hannken
702 1.46 yamaguch sc->sc_max_nvq_pairs = 1;
703 1.46 yamaguch sc->sc_req_nvq_pairs = 1;
704 1.46 yamaguch sc->sc_act_nvq_pairs = 1;
705 1.46 yamaguch
706 1.32 jdolecek req_flags = 0;
707 1.1 hannken
708 1.7 ozaki #ifdef VIOIF_MPSAFE
709 1.32 jdolecek req_flags |= VIRTIO_F_PCI_INTR_MPSAFE;
710 1.7 ozaki #endif
711 1.12 ozaki #ifdef VIOIF_SOFTINT_INTR
712 1.32 jdolecek req_flags |= VIRTIO_F_PCI_INTR_SOFTINT;
713 1.12 ozaki #endif
714 1.32 jdolecek req_flags |= VIRTIO_F_PCI_INTR_MSIX;
715 1.32 jdolecek
716 1.46 yamaguch req_features =
717 1.46 yamaguch VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
718 1.46 yamaguch VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
719 1.46 yamaguch #ifdef VIOIF_MULTIQ
720 1.46 yamaguch req_features |= VIRTIO_NET_F_MQ;
721 1.46 yamaguch #endif
722 1.46 yamaguch virtio_child_attach_start(vsc, self, IPL_NET, NULL,
723 1.34 ozaki vioif_config_change, virtio_vq_intr, req_flags,
724 1.46 yamaguch req_features, VIRTIO_NET_FLAG_BITS);
725 1.32 jdolecek
726 1.32 jdolecek features = virtio_features(vsc);
727 1.7 ozaki
728 1.1 hannken if (features & VIRTIO_NET_F_MAC) {
729 1.1 hannken sc->sc_mac[0] = virtio_read_device_config_1(vsc,
730 1.1 hannken VIRTIO_NET_CONFIG_MAC+0);
731 1.1 hannken sc->sc_mac[1] = virtio_read_device_config_1(vsc,
732 1.1 hannken VIRTIO_NET_CONFIG_MAC+1);
733 1.1 hannken sc->sc_mac[2] = virtio_read_device_config_1(vsc,
734 1.1 hannken VIRTIO_NET_CONFIG_MAC+2);
735 1.1 hannken sc->sc_mac[3] = virtio_read_device_config_1(vsc,
736 1.1 hannken VIRTIO_NET_CONFIG_MAC+3);
737 1.1 hannken sc->sc_mac[4] = virtio_read_device_config_1(vsc,
738 1.1 hannken VIRTIO_NET_CONFIG_MAC+4);
739 1.1 hannken sc->sc_mac[5] = virtio_read_device_config_1(vsc,
740 1.1 hannken VIRTIO_NET_CONFIG_MAC+5);
741 1.1 hannken } else {
742 1.1 hannken /* code stolen from sys/net/if_tap.c */
743 1.1 hannken struct timeval tv;
744 1.1 hannken uint32_t ui;
745 1.1 hannken getmicrouptime(&tv);
746 1.1 hannken ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
747 1.1 hannken memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
748 1.1 hannken virtio_write_device_config_1(vsc,
749 1.1 hannken VIRTIO_NET_CONFIG_MAC+0,
750 1.1 hannken sc->sc_mac[0]);
751 1.1 hannken virtio_write_device_config_1(vsc,
752 1.1 hannken VIRTIO_NET_CONFIG_MAC+1,
753 1.1 hannken sc->sc_mac[1]);
754 1.1 hannken virtio_write_device_config_1(vsc,
755 1.1 hannken VIRTIO_NET_CONFIG_MAC+2,
756 1.1 hannken sc->sc_mac[2]);
757 1.1 hannken virtio_write_device_config_1(vsc,
758 1.1 hannken VIRTIO_NET_CONFIG_MAC+3,
759 1.1 hannken sc->sc_mac[3]);
760 1.1 hannken virtio_write_device_config_1(vsc,
761 1.1 hannken VIRTIO_NET_CONFIG_MAC+4,
762 1.1 hannken sc->sc_mac[4]);
763 1.1 hannken virtio_write_device_config_1(vsc,
764 1.1 hannken VIRTIO_NET_CONFIG_MAC+5,
765 1.1 hannken sc->sc_mac[5]);
766 1.1 hannken }
767 1.32 jdolecek
768 1.32 jdolecek aprint_normal_dev(self, "Ethernet address %s\n", ether_sprintf(sc->sc_mac));
769 1.1 hannken
770 1.46 yamaguch if ((features & VIRTIO_NET_F_CTRL_VQ) &&
771 1.46 yamaguch (features & VIRTIO_NET_F_CTRL_RX)) {
772 1.46 yamaguch sc->sc_has_ctrl = true;
773 1.46 yamaguch
774 1.46 yamaguch cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
775 1.46 yamaguch mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
776 1.46 yamaguch ctrlq->ctrlq_inuse = FREE;
777 1.46 yamaguch } else {
778 1.46 yamaguch sc->sc_has_ctrl = false;
779 1.46 yamaguch }
780 1.46 yamaguch
781 1.46 yamaguch if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
782 1.46 yamaguch sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
783 1.46 yamaguch VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
784 1.46 yamaguch
785 1.46 yamaguch if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
786 1.46 yamaguch goto err;
787 1.46 yamaguch
788 1.46 yamaguch /* Limit the number of queue pairs to use */
789 1.46 yamaguch sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
790 1.46 yamaguch }
791 1.46 yamaguch
792 1.46 yamaguch r = vioif_alloc_queues(sc);
793 1.46 yamaguch if (r != 0)
794 1.46 yamaguch goto err;
795 1.46 yamaguch
796 1.46 yamaguch virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
797 1.46 yamaguch
798 1.43 yamaguch #ifdef VIOIF_MPSAFE
799 1.43 yamaguch softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
800 1.43 yamaguch #else
801 1.43 yamaguch softint_flags = SOFTINT_NET;
802 1.43 yamaguch #endif
803 1.7 ozaki
804 1.17 ozaki /*
805 1.46 yamaguch * Allocating a virtqueues
806 1.17 ozaki */
807 1.46 yamaguch for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
808 1.46 yamaguch rxq = &sc->sc_rxq[i];
809 1.46 yamaguch txq = &sc->sc_txq[i];
810 1.46 yamaguch char qname[32];
811 1.46 yamaguch
812 1.46 yamaguch rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
813 1.46 yamaguch
814 1.46 yamaguch rxq->rxq_softint = softint_establish(softint_flags, vioif_rx_softint, rxq);
815 1.46 yamaguch if (rxq->rxq_softint == NULL) {
816 1.46 yamaguch aprint_error_dev(self, "cannot establish rx softint\n");
817 1.46 yamaguch goto err;
818 1.46 yamaguch }
819 1.46 yamaguch snprintf(qname, sizeof(qname), "rx%d", i);
820 1.46 yamaguch r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs,
821 1.46 yamaguch MCLBYTES+sizeof(struct virtio_net_hdr), nvqs, qname);
822 1.46 yamaguch if (r != 0)
823 1.46 yamaguch goto err;
824 1.46 yamaguch nvqs++;
825 1.46 yamaguch rxq->rxq_vq->vq_done = vioif_rx_vq_done;
826 1.46 yamaguch rxq->rxq_vq->vq_done_ctx = (void *)rxq;
827 1.46 yamaguch rxq->rxq_stopping = true;
828 1.46 yamaguch
829 1.46 yamaguch txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
830 1.46 yamaguch txq->txq_deferred_transmit = softint_establish(softint_flags,
831 1.46 yamaguch vioif_deferred_transmit, txq);
832 1.46 yamaguch if (txq->txq_deferred_transmit == NULL) {
833 1.46 yamaguch aprint_error_dev(self, "cannot establish tx softint\n");
834 1.46 yamaguch goto err;
835 1.46 yamaguch }
836 1.46 yamaguch snprintf(qname, sizeof(qname), "tx%d", i);
837 1.46 yamaguch r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs,
838 1.46 yamaguch (sizeof(struct virtio_net_hdr) + (ETHER_MAX_LEN - ETHER_HDR_LEN)),
839 1.46 yamaguch VIRTIO_NET_TX_MAXNSEGS + 1, qname);
840 1.46 yamaguch if (r != 0)
841 1.46 yamaguch goto err;
842 1.46 yamaguch nvqs++;
843 1.46 yamaguch txq->txq_vq->vq_done = vioif_tx_vq_done;
844 1.46 yamaguch txq->txq_vq->vq_done_ctx = (void *)txq;
845 1.46 yamaguch txq->txq_link_active = sc->sc_link_active;
846 1.46 yamaguch txq->txq_stopping = false;
847 1.46 yamaguch txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_NOSLEEP);
848 1.46 yamaguch if (txq->txq_intrq == NULL)
849 1.46 yamaguch goto err;
850 1.43 yamaguch }
851 1.17 ozaki
852 1.46 yamaguch if (sc->sc_has_ctrl) {
853 1.17 ozaki /*
854 1.17 ozaki * Allocating a virtqueue for control channel
855 1.17 ozaki */
856 1.46 yamaguch r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs,
857 1.17 ozaki NBPG, 1, "control");
858 1.17 ozaki if (r != 0) {
859 1.17 ozaki aprint_error_dev(self, "failed to allocate "
860 1.17 ozaki "a virtqueue for control channel\n");
861 1.46 yamaguch
862 1.46 yamaguch sc->sc_has_ctrl = false;
863 1.46 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
864 1.46 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
865 1.46 yamaguch } else {
866 1.46 yamaguch nvqs++;
867 1.46 yamaguch ctrlq->ctrlq_vq->vq_done = vioif_ctrl_vq_done;
868 1.46 yamaguch ctrlq->ctrlq_vq->vq_done_ctx = (void *) ctrlq;
869 1.1 hannken }
870 1.1 hannken }
871 1.34 ozaki
872 1.43 yamaguch sc->sc_ctl_softint = softint_establish(softint_flags, vioif_ctl_softint, sc);
873 1.34 ozaki if (sc->sc_ctl_softint == NULL) {
874 1.34 ozaki aprint_error_dev(self, "cannot establish ctl softint\n");
875 1.1 hannken goto err;
876 1.1 hannken }
877 1.1 hannken
878 1.1 hannken if (vioif_alloc_mems(sc) < 0)
879 1.1 hannken goto err;
880 1.1 hannken
881 1.32 jdolecek if (virtio_child_attach_finish(vsc) != 0)
882 1.32 jdolecek goto err;
883 1.32 jdolecek
884 1.1 hannken strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
885 1.1 hannken ifp->if_softc = sc;
886 1.1 hannken ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
887 1.45 yamaguch #ifdef VIOIF_MPSAFE
888 1.45 yamaguch ifp->if_extflags = IFEF_MPSAFE;
889 1.45 yamaguch #endif
890 1.1 hannken ifp->if_start = vioif_start;
891 1.46 yamaguch if (sc->sc_req_nvq_pairs > 1)
892 1.46 yamaguch ifp->if_transmit = vioif_transmit;
893 1.1 hannken ifp->if_ioctl = vioif_ioctl;
894 1.1 hannken ifp->if_init = vioif_init;
895 1.1 hannken ifp->if_stop = vioif_stop;
896 1.1 hannken ifp->if_capabilities = 0;
897 1.1 hannken ifp->if_watchdog = vioif_watchdog;
898 1.46 yamaguch txq = &sc->sc_txq[0];
899 1.43 yamaguch IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN));
900 1.36 jdolecek IFQ_SET_READY(&ifp->if_snd);
901 1.1 hannken
902 1.11 ozaki sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
903 1.11 ozaki
904 1.1 hannken if_attach(ifp);
905 1.28 ozaki if_deferred_start_init(ifp, NULL);
906 1.1 hannken ether_ifattach(ifp, sc->sc_mac);
907 1.1 hannken
908 1.1 hannken return;
909 1.1 hannken
910 1.1 hannken err:
911 1.46 yamaguch for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
912 1.46 yamaguch rxq = &sc->sc_rxq[i];
913 1.46 yamaguch txq = &sc->sc_txq[i];
914 1.46 yamaguch
915 1.46 yamaguch if (rxq->rxq_lock) {
916 1.46 yamaguch mutex_obj_free(rxq->rxq_lock);
917 1.46 yamaguch rxq->rxq_lock = NULL;
918 1.46 yamaguch }
919 1.46 yamaguch
920 1.46 yamaguch if (rxq->rxq_softint) {
921 1.46 yamaguch softint_disestablish(rxq->rxq_softint);
922 1.46 yamaguch rxq->rxq_softint = NULL;
923 1.46 yamaguch }
924 1.46 yamaguch
925 1.46 yamaguch if (txq->txq_lock) {
926 1.46 yamaguch mutex_obj_free(txq->txq_lock);
927 1.46 yamaguch txq->txq_lock = NULL;
928 1.46 yamaguch }
929 1.43 yamaguch
930 1.46 yamaguch if (txq->txq_deferred_transmit) {
931 1.46 yamaguch softint_disestablish(txq->txq_deferred_transmit);
932 1.46 yamaguch txq->txq_deferred_transmit = NULL;
933 1.46 yamaguch }
934 1.43 yamaguch
935 1.46 yamaguch if (txq->txq_intrq) {
936 1.46 yamaguch pcq_destroy(txq->txq_intrq);
937 1.46 yamaguch txq->txq_intrq = NULL;
938 1.46 yamaguch }
939 1.43 yamaguch }
940 1.7 ozaki
941 1.32 jdolecek if (sc->sc_has_ctrl) {
942 1.43 yamaguch cv_destroy(&ctrlq->ctrlq_wait);
943 1.43 yamaguch mutex_destroy(&ctrlq->ctrlq_wait_lock);
944 1.1 hannken }
945 1.20 christos
946 1.32 jdolecek while (nvqs > 0)
947 1.46 yamaguch virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]);
948 1.46 yamaguch
949 1.46 yamaguch vioif_free_queues(sc);
950 1.20 christos
951 1.32 jdolecek virtio_child_attach_failed(vsc);
952 1.1 hannken return;
953 1.1 hannken }
954 1.1 hannken
955 1.1 hannken /* we need interrupts to make promiscuous mode off */
956 1.1 hannken static void
957 1.1 hannken vioif_deferred_init(device_t self)
958 1.1 hannken {
959 1.1 hannken struct vioif_softc *sc = device_private(self);
960 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
961 1.1 hannken int r;
962 1.1 hannken
963 1.9 ozaki if (ifp->if_flags & IFF_PROMISC)
964 1.10 ozaki return;
965 1.9 ozaki
966 1.1 hannken r = vioif_set_promisc(sc, false);
967 1.1 hannken if (r != 0)
968 1.1 hannken aprint_error_dev(self, "resetting promisc mode failed, "
969 1.1 hannken "errror code %d\n", r);
970 1.1 hannken }
971 1.1 hannken
972 1.46 yamaguch static void
973 1.46 yamaguch vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
974 1.46 yamaguch {
975 1.46 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
976 1.46 yamaguch struct vioif_txqueue *txq;
977 1.46 yamaguch struct vioif_rxqueue *rxq;
978 1.46 yamaguch int i;
979 1.46 yamaguch
980 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
981 1.46 yamaguch txq = &sc->sc_txq[i];
982 1.46 yamaguch rxq = &sc->sc_rxq[i];
983 1.46 yamaguch
984 1.46 yamaguch virtio_start_vq_intr(vsc, txq->txq_vq);
985 1.46 yamaguch virtio_start_vq_intr(vsc, rxq->rxq_vq);
986 1.46 yamaguch }
987 1.46 yamaguch }
988 1.46 yamaguch
989 1.46 yamaguch static void
990 1.46 yamaguch vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
991 1.46 yamaguch {
992 1.46 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
993 1.46 yamaguch struct vioif_txqueue *txq;
994 1.46 yamaguch struct vioif_rxqueue *rxq;
995 1.46 yamaguch int i;
996 1.46 yamaguch
997 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
998 1.46 yamaguch txq = &sc->sc_txq[i];
999 1.46 yamaguch rxq = &sc->sc_rxq[i];
1000 1.46 yamaguch
1001 1.46 yamaguch virtio_stop_vq_intr(vsc, txq->txq_vq);
1002 1.46 yamaguch virtio_stop_vq_intr(vsc, rxq->rxq_vq);
1003 1.46 yamaguch }
1004 1.46 yamaguch }
1005 1.46 yamaguch
1006 1.1 hannken /*
1007 1.1 hannken * Interface functions for ifnet
1008 1.1 hannken */
1009 1.1 hannken static int
1010 1.1 hannken vioif_init(struct ifnet *ifp)
1011 1.1 hannken {
1012 1.1 hannken struct vioif_softc *sc = ifp->if_softc;
1013 1.33 ozaki struct virtio_softc *vsc = sc->sc_virtio;
1014 1.46 yamaguch struct vioif_rxqueue *rxq;
1015 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1016 1.46 yamaguch int r, i;
1017 1.1 hannken
1018 1.1 hannken vioif_stop(ifp, 0);
1019 1.7 ozaki
1020 1.33 ozaki virtio_reinit_start(vsc);
1021 1.33 ozaki virtio_negotiate_features(vsc, virtio_features(vsc));
1022 1.46 yamaguch
1023 1.46 yamaguch for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
1024 1.46 yamaguch rxq = &sc->sc_rxq[i];
1025 1.46 yamaguch
1026 1.46 yamaguch /* Have to set false before vioif_populate_rx_mbufs */
1027 1.46 yamaguch rxq->rxq_stopping = false;
1028 1.46 yamaguch vioif_populate_rx_mbufs(rxq);
1029 1.46 yamaguch }
1030 1.46 yamaguch
1031 1.46 yamaguch virtio_reinit_end(vsc);
1032 1.43 yamaguch virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
1033 1.46 yamaguch
1034 1.46 yamaguch r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
1035 1.46 yamaguch if (r == 0)
1036 1.46 yamaguch sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
1037 1.46 yamaguch else
1038 1.46 yamaguch sc->sc_act_nvq_pairs = 1;
1039 1.46 yamaguch
1040 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++)
1041 1.46 yamaguch sc->sc_txq[i].txq_stopping = false;
1042 1.46 yamaguch
1043 1.46 yamaguch vioif_enable_interrupt_vqpairs(sc);
1044 1.33 ozaki
1045 1.8 pooka if (!sc->sc_deferred_init_done) {
1046 1.8 pooka sc->sc_deferred_init_done = 1;
1047 1.32 jdolecek if (sc->sc_has_ctrl)
1048 1.8 pooka vioif_deferred_init(sc->sc_dev);
1049 1.8 pooka }
1050 1.8 pooka
1051 1.34 ozaki vioif_update_link_status(sc);
1052 1.1 hannken ifp->if_flags |= IFF_RUNNING;
1053 1.1 hannken ifp->if_flags &= ~IFF_OACTIVE;
1054 1.1 hannken vioif_rx_filter(sc);
1055 1.1 hannken
1056 1.1 hannken return 0;
1057 1.1 hannken }
1058 1.1 hannken
1059 1.1 hannken static void
1060 1.1 hannken vioif_stop(struct ifnet *ifp, int disable)
1061 1.1 hannken {
1062 1.1 hannken struct vioif_softc *sc = ifp->if_softc;
1063 1.1 hannken struct virtio_softc *vsc = sc->sc_virtio;
1064 1.46 yamaguch struct vioif_txqueue *txq;
1065 1.46 yamaguch struct vioif_rxqueue *rxq;
1066 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1067 1.46 yamaguch int i;
1068 1.1 hannken
1069 1.13 ozaki /* Take the locks to ensure that ongoing TX/RX finish */
1070 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1071 1.46 yamaguch txq = &sc->sc_txq[i];
1072 1.46 yamaguch rxq = &sc->sc_rxq[i];
1073 1.46 yamaguch
1074 1.46 yamaguch mutex_enter(txq->txq_lock);
1075 1.46 yamaguch txq->txq_stopping = true;
1076 1.46 yamaguch mutex_exit(txq->txq_lock);
1077 1.46 yamaguch
1078 1.46 yamaguch mutex_enter(rxq->rxq_lock);
1079 1.46 yamaguch rxq->rxq_stopping = true;
1080 1.46 yamaguch mutex_exit(rxq->rxq_lock);
1081 1.46 yamaguch }
1082 1.7 ozaki
1083 1.33 ozaki /* disable interrupts */
1084 1.46 yamaguch vioif_disable_interrupt_vqpairs(sc);
1085 1.46 yamaguch
1086 1.33 ozaki if (sc->sc_has_ctrl)
1087 1.43 yamaguch virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
1088 1.33 ozaki
1089 1.1 hannken /* only way to stop I/O and DMA is resetting... */
1090 1.1 hannken virtio_reset(vsc);
1091 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++)
1092 1.46 yamaguch vioif_rx_deq(&sc->sc_rxq[i]);
1093 1.46 yamaguch
1094 1.1 hannken ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
1095 1.34 ozaki sc->sc_link_active = false;
1096 1.1 hannken
1097 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
1098 1.46 yamaguch txq = &sc->sc_txq[i];
1099 1.46 yamaguch rxq = &sc->sc_rxq[i];
1100 1.46 yamaguch
1101 1.46 yamaguch txq->txq_link_active = false;
1102 1.46 yamaguch
1103 1.46 yamaguch if (disable)
1104 1.46 yamaguch vioif_rx_drain(rxq);
1105 1.46 yamaguch
1106 1.46 yamaguch vioif_tx_drain(txq);
1107 1.46 yamaguch }
1108 1.1 hannken }
1109 1.1 hannken
1110 1.1 hannken static void
1111 1.46 yamaguch vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq, bool is_transmit)
1112 1.1 hannken {
1113 1.1 hannken struct vioif_softc *sc = ifp->if_softc;
1114 1.1 hannken struct virtio_softc *vsc = sc->sc_virtio;
1115 1.43 yamaguch struct virtqueue *vq = txq->txq_vq;
1116 1.1 hannken struct mbuf *m;
1117 1.36 jdolecek int queued = 0;
1118 1.1 hannken
1119 1.46 yamaguch KASSERT(mutex_owned(txq->txq_lock));
1120 1.46 yamaguch
1121 1.46 yamaguch if ((ifp->if_flags & IFF_RUNNING) == 0)
1122 1.46 yamaguch return;
1123 1.7 ozaki
1124 1.46 yamaguch if (!txq->txq_link_active || txq->txq_stopping)
1125 1.46 yamaguch return;
1126 1.7 ozaki
1127 1.46 yamaguch if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit)
1128 1.46 yamaguch return;
1129 1.1 hannken
1130 1.2 jmcneill for (;;) {
1131 1.1 hannken int slot, r;
1132 1.1 hannken
1133 1.46 yamaguch if (is_transmit)
1134 1.46 yamaguch m = pcq_get(txq->txq_intrq);
1135 1.46 yamaguch else
1136 1.46 yamaguch IFQ_DEQUEUE(&ifp->if_snd, m);
1137 1.46 yamaguch
1138 1.2 jmcneill if (m == NULL)
1139 1.2 jmcneill break;
1140 1.2 jmcneill
1141 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
1142 1.1 hannken if (r == EAGAIN) {
1143 1.1 hannken ifp->if_flags |= IFF_OACTIVE;
1144 1.36 jdolecek m_freem(m);
1145 1.36 jdolecek break;
1146 1.1 hannken }
1147 1.1 hannken if (r != 0)
1148 1.1 hannken panic("enqueue_prep for a tx buffer");
1149 1.36 jdolecek
1150 1.32 jdolecek r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1151 1.43 yamaguch txq->txq_dmamaps[slot],
1152 1.1 hannken m, BUS_DMA_WRITE|BUS_DMA_NOWAIT);
1153 1.1 hannken if (r != 0) {
1154 1.36 jdolecek /* maybe just too fragmented */
1155 1.36 jdolecek struct mbuf *newm;
1156 1.36 jdolecek
1157 1.36 jdolecek newm = m_defrag(m, M_NOWAIT);
1158 1.36 jdolecek if (newm == NULL) {
1159 1.36 jdolecek aprint_error_dev(sc->sc_dev,
1160 1.36 jdolecek "m_defrag() failed\n");
1161 1.36 jdolecek goto skip;
1162 1.36 jdolecek }
1163 1.36 jdolecek
1164 1.37 jdolecek m = newm;
1165 1.36 jdolecek r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1166 1.43 yamaguch txq->txq_dmamaps[slot],
1167 1.37 jdolecek m, BUS_DMA_WRITE|BUS_DMA_NOWAIT);
1168 1.36 jdolecek if (r != 0) {
1169 1.36 jdolecek aprint_error_dev(sc->sc_dev,
1170 1.36 jdolecek "tx dmamap load failed, error code %d\n",
1171 1.36 jdolecek r);
1172 1.36 jdolecek skip:
1173 1.37 jdolecek m_freem(m);
1174 1.36 jdolecek virtio_enqueue_abort(vsc, vq, slot);
1175 1.36 jdolecek continue;
1176 1.36 jdolecek }
1177 1.1 hannken }
1178 1.36 jdolecek
1179 1.36 jdolecek /* This should actually never fail */
1180 1.1 hannken r = virtio_enqueue_reserve(vsc, vq, slot,
1181 1.43 yamaguch txq->txq_dmamaps[slot]->dm_nsegs + 1);
1182 1.1 hannken if (r != 0) {
1183 1.36 jdolecek aprint_error_dev(sc->sc_dev,
1184 1.36 jdolecek "virtio_enqueue_reserve failed, error code %d\n",
1185 1.36 jdolecek r);
1186 1.32 jdolecek bus_dmamap_unload(virtio_dmat(vsc),
1187 1.43 yamaguch txq->txq_dmamaps[slot]);
1188 1.36 jdolecek /* slot already freed by virtio_enqueue_reserve */
1189 1.37 jdolecek m_freem(m);
1190 1.36 jdolecek continue;
1191 1.1 hannken }
1192 1.7 ozaki
1193 1.43 yamaguch txq->txq_mbufs[slot] = m;
1194 1.1 hannken
1195 1.43 yamaguch memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr));
1196 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1197 1.43 yamaguch 0, txq->txq_dmamaps[slot]->dm_mapsize,
1198 1.1 hannken BUS_DMASYNC_PREWRITE);
1199 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1200 1.43 yamaguch 0, txq->txq_hdr_dmamaps[slot]->dm_mapsize,
1201 1.1 hannken BUS_DMASYNC_PREWRITE);
1202 1.43 yamaguch virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true);
1203 1.43 yamaguch virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true);
1204 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, false);
1205 1.36 jdolecek
1206 1.1 hannken queued++;
1207 1.41 msaitoh bpf_mtap(ifp, m, BPF_D_OUT);
1208 1.1 hannken }
1209 1.1 hannken
1210 1.1 hannken if (queued > 0) {
1211 1.1 hannken virtio_enqueue_commit(vsc, vq, -1, true);
1212 1.1 hannken ifp->if_timer = 5;
1213 1.1 hannken }
1214 1.46 yamaguch }
1215 1.46 yamaguch
1216 1.46 yamaguch static void
1217 1.46 yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1218 1.46 yamaguch {
1219 1.46 yamaguch
1220 1.46 yamaguch /*
1221 1.46 yamaguch * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
1222 1.46 yamaguch */
1223 1.46 yamaguch vioif_send_common_locked(ifp, txq, false);
1224 1.46 yamaguch
1225 1.46 yamaguch }
1226 1.46 yamaguch
1227 1.46 yamaguch static void
1228 1.46 yamaguch vioif_start(struct ifnet *ifp)
1229 1.46 yamaguch {
1230 1.46 yamaguch struct vioif_softc *sc = ifp->if_softc;
1231 1.46 yamaguch struct vioif_txqueue *txq = &sc->sc_txq[0];
1232 1.46 yamaguch
1233 1.46 yamaguch #ifdef VIOIF_MPSAFE
1234 1.46 yamaguch KASSERT(if_is_mpsafe(ifp));
1235 1.46 yamaguch #endif
1236 1.46 yamaguch
1237 1.46 yamaguch mutex_enter(txq->txq_lock);
1238 1.46 yamaguch if (!txq->txq_stopping)
1239 1.46 yamaguch vioif_start_locked(ifp, txq);
1240 1.46 yamaguch mutex_exit(txq->txq_lock);
1241 1.46 yamaguch }
1242 1.46 yamaguch
1243 1.46 yamaguch static inline int
1244 1.46 yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
1245 1.46 yamaguch {
1246 1.46 yamaguch struct vioif_softc *sc = ifp->if_softc;
1247 1.46 yamaguch u_int cpuid = cpu_index(curcpu());
1248 1.46 yamaguch
1249 1.46 yamaguch return cpuid % sc->sc_act_nvq_pairs;
1250 1.46 yamaguch }
1251 1.46 yamaguch
1252 1.46 yamaguch static void
1253 1.46 yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
1254 1.46 yamaguch {
1255 1.46 yamaguch
1256 1.46 yamaguch vioif_send_common_locked(ifp, txq, true);
1257 1.46 yamaguch }
1258 1.46 yamaguch
1259 1.46 yamaguch static int
1260 1.46 yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
1261 1.46 yamaguch {
1262 1.46 yamaguch struct vioif_softc *sc = ifp->if_softc;
1263 1.46 yamaguch struct vioif_txqueue *txq;
1264 1.46 yamaguch int qid;
1265 1.46 yamaguch
1266 1.46 yamaguch qid = vioif_select_txqueue(ifp, m);
1267 1.46 yamaguch txq = &sc->sc_txq[qid];
1268 1.46 yamaguch
1269 1.46 yamaguch if (__predict_false(!pcq_put(txq->txq_intrq, m))) {
1270 1.46 yamaguch m_freem(m);
1271 1.46 yamaguch return ENOBUFS;
1272 1.46 yamaguch }
1273 1.46 yamaguch
1274 1.46 yamaguch ifp->if_obytes += m->m_pkthdr.len;
1275 1.46 yamaguch if (m->m_flags & M_MCAST)
1276 1.46 yamaguch ifp->if_omcasts++;
1277 1.46 yamaguch
1278 1.46 yamaguch if (mutex_tryenter(txq->txq_lock)) {
1279 1.46 yamaguch if (!txq->txq_stopping)
1280 1.46 yamaguch vioif_transmit_locked(ifp, txq);
1281 1.46 yamaguch mutex_exit(txq->txq_lock);
1282 1.46 yamaguch }
1283 1.46 yamaguch
1284 1.46 yamaguch return 0;
1285 1.46 yamaguch }
1286 1.46 yamaguch
1287 1.46 yamaguch static void
1288 1.46 yamaguch vioif_deferred_transmit(void *arg)
1289 1.46 yamaguch {
1290 1.46 yamaguch struct vioif_txqueue *txq = arg;
1291 1.46 yamaguch struct virtio_softc *vsc = txq->txq_vq->vq_owner;
1292 1.46 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1293 1.46 yamaguch struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1294 1.7 ozaki
1295 1.46 yamaguch if (mutex_tryenter(txq->txq_lock)) {
1296 1.46 yamaguch vioif_send_common_locked(ifp, txq, true);
1297 1.46 yamaguch mutex_exit(txq->txq_lock);
1298 1.46 yamaguch }
1299 1.1 hannken }
1300 1.1 hannken
1301 1.1 hannken static int
1302 1.1 hannken vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
1303 1.1 hannken {
1304 1.1 hannken int s, r;
1305 1.1 hannken
1306 1.1 hannken s = splnet();
1307 1.1 hannken
1308 1.1 hannken r = ether_ioctl(ifp, cmd, data);
1309 1.1 hannken if ((r == 0 && cmd == SIOCSIFFLAGS) ||
1310 1.1 hannken (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) {
1311 1.1 hannken if (ifp->if_flags & IFF_RUNNING)
1312 1.1 hannken r = vioif_rx_filter(ifp->if_softc);
1313 1.1 hannken else
1314 1.1 hannken r = 0;
1315 1.1 hannken }
1316 1.1 hannken
1317 1.1 hannken splx(s);
1318 1.1 hannken
1319 1.1 hannken return r;
1320 1.1 hannken }
1321 1.1 hannken
1322 1.1 hannken void
1323 1.1 hannken vioif_watchdog(struct ifnet *ifp)
1324 1.1 hannken {
1325 1.1 hannken struct vioif_softc *sc = ifp->if_softc;
1326 1.46 yamaguch int i;
1327 1.1 hannken
1328 1.46 yamaguch if (ifp->if_flags & IFF_RUNNING) {
1329 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++)
1330 1.46 yamaguch vioif_tx_vq_done(sc->sc_txq[i].txq_vq);
1331 1.46 yamaguch }
1332 1.1 hannken }
1333 1.1 hannken
1334 1.1 hannken
1335 1.1 hannken /*
1336 1.39 dholland * Receive implementation
1337 1.1 hannken */
1338 1.39 dholland /* allocate and initialize a mbuf for receive */
1339 1.1 hannken static int
1340 1.46 yamaguch vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1341 1.1 hannken {
1342 1.46 yamaguch struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1343 1.1 hannken struct mbuf *m;
1344 1.1 hannken int r;
1345 1.1 hannken
1346 1.1 hannken MGETHDR(m, M_DONTWAIT, MT_DATA);
1347 1.1 hannken if (m == NULL)
1348 1.1 hannken return ENOBUFS;
1349 1.1 hannken MCLGET(m, M_DONTWAIT);
1350 1.1 hannken if ((m->m_flags & M_EXT) == 0) {
1351 1.1 hannken m_freem(m);
1352 1.1 hannken return ENOBUFS;
1353 1.1 hannken }
1354 1.43 yamaguch rxq->rxq_mbufs[i] = m;
1355 1.1 hannken m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
1356 1.46 yamaguch r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
1357 1.43 yamaguch rxq->rxq_dmamaps[i],
1358 1.1 hannken m, BUS_DMA_READ|BUS_DMA_NOWAIT);
1359 1.1 hannken if (r) {
1360 1.1 hannken m_freem(m);
1361 1.46 yamaguch rxq->rxq_mbufs[i] = NULL;
1362 1.1 hannken return r;
1363 1.1 hannken }
1364 1.1 hannken
1365 1.1 hannken return 0;
1366 1.1 hannken }
1367 1.1 hannken
1368 1.39 dholland /* free a mbuf for receive */
1369 1.1 hannken static void
1370 1.46 yamaguch vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i)
1371 1.1 hannken {
1372 1.46 yamaguch struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
1373 1.43 yamaguch
1374 1.46 yamaguch bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]);
1375 1.43 yamaguch m_freem(rxq->rxq_mbufs[i]);
1376 1.43 yamaguch rxq->rxq_mbufs[i] = NULL;
1377 1.1 hannken }
1378 1.1 hannken
1379 1.39 dholland /* add mbufs for all the empty receive slots */
1380 1.1 hannken static void
1381 1.46 yamaguch vioif_populate_rx_mbufs(struct vioif_rxqueue *rxq)
1382 1.1 hannken {
1383 1.43 yamaguch
1384 1.46 yamaguch mutex_enter(rxq->rxq_lock);
1385 1.46 yamaguch vioif_populate_rx_mbufs_locked(rxq);
1386 1.46 yamaguch mutex_exit(rxq->rxq_lock);
1387 1.12 ozaki }
1388 1.12 ozaki
1389 1.12 ozaki static void
1390 1.46 yamaguch vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq)
1391 1.12 ozaki {
1392 1.46 yamaguch struct virtqueue *vq = rxq->rxq_vq;
1393 1.46 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1394 1.46 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1395 1.1 hannken int i, r, ndone = 0;
1396 1.1 hannken
1397 1.46 yamaguch KASSERT(mutex_owned(rxq->rxq_lock));
1398 1.7 ozaki
1399 1.43 yamaguch if (rxq->rxq_stopping)
1400 1.12 ozaki return;
1401 1.7 ozaki
1402 1.1 hannken for (i = 0; i < vq->vq_num; i++) {
1403 1.1 hannken int slot;
1404 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
1405 1.1 hannken if (r == EAGAIN)
1406 1.1 hannken break;
1407 1.1 hannken if (r != 0)
1408 1.1 hannken panic("enqueue_prep for rx buffers");
1409 1.43 yamaguch if (rxq->rxq_mbufs[slot] == NULL) {
1410 1.46 yamaguch r = vioif_add_rx_mbuf(rxq, slot);
1411 1.1 hannken if (r != 0) {
1412 1.1 hannken printf("%s: rx mbuf allocation failed, "
1413 1.1 hannken "error code %d\n",
1414 1.1 hannken device_xname(sc->sc_dev), r);
1415 1.1 hannken break;
1416 1.1 hannken }
1417 1.1 hannken }
1418 1.1 hannken r = virtio_enqueue_reserve(vsc, vq, slot,
1419 1.43 yamaguch rxq->rxq_dmamaps[slot]->dm_nsegs + 1);
1420 1.1 hannken if (r != 0) {
1421 1.46 yamaguch vioif_free_rx_mbuf(rxq, slot);
1422 1.1 hannken break;
1423 1.1 hannken }
1424 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1425 1.1 hannken 0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD);
1426 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1427 1.1 hannken 0, MCLBYTES, BUS_DMASYNC_PREREAD);
1428 1.43 yamaguch virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot], false);
1429 1.43 yamaguch virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false);
1430 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, false);
1431 1.1 hannken ndone++;
1432 1.1 hannken }
1433 1.1 hannken if (ndone > 0)
1434 1.1 hannken virtio_enqueue_commit(vsc, vq, -1, true);
1435 1.1 hannken }
1436 1.1 hannken
1437 1.39 dholland /* dequeue received packets */
1438 1.1 hannken static int
1439 1.46 yamaguch vioif_rx_deq(struct vioif_rxqueue *rxq)
1440 1.1 hannken {
1441 1.7 ozaki int r;
1442 1.7 ozaki
1443 1.43 yamaguch KASSERT(rxq->rxq_stopping);
1444 1.7 ozaki
1445 1.46 yamaguch mutex_enter(rxq->rxq_lock);
1446 1.46 yamaguch r = vioif_rx_deq_locked(rxq);
1447 1.46 yamaguch mutex_exit(rxq->rxq_lock);
1448 1.7 ozaki
1449 1.7 ozaki return r;
1450 1.7 ozaki }
1451 1.7 ozaki
1452 1.39 dholland /* dequeue received packets */
1453 1.7 ozaki static int
1454 1.46 yamaguch vioif_rx_deq_locked(struct vioif_rxqueue *rxq)
1455 1.7 ozaki {
1456 1.43 yamaguch struct virtqueue *vq = rxq->rxq_vq;
1457 1.46 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1458 1.46 yamaguch struct vioif_softc *sc = device_private(virtio_child(vsc));
1459 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1460 1.1 hannken struct mbuf *m;
1461 1.1 hannken int r = 0;
1462 1.1 hannken int slot, len;
1463 1.1 hannken
1464 1.46 yamaguch KASSERT(mutex_owned(rxq->rxq_lock));
1465 1.7 ozaki
1466 1.1 hannken while (virtio_dequeue(vsc, vq, &slot, &len) == 0) {
1467 1.1 hannken len -= sizeof(struct virtio_net_hdr);
1468 1.1 hannken r = 1;
1469 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
1470 1.1 hannken 0, sizeof(struct virtio_net_hdr),
1471 1.1 hannken BUS_DMASYNC_POSTREAD);
1472 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
1473 1.1 hannken 0, MCLBYTES,
1474 1.1 hannken BUS_DMASYNC_POSTREAD);
1475 1.43 yamaguch m = rxq->rxq_mbufs[slot];
1476 1.1 hannken KASSERT(m != NULL);
1477 1.43 yamaguch bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]);
1478 1.46 yamaguch rxq->rxq_mbufs[slot] = NULL;
1479 1.1 hannken virtio_dequeue_commit(vsc, vq, slot);
1480 1.24 ozaki m_set_rcvif(m, ifp);
1481 1.1 hannken m->m_len = m->m_pkthdr.len = len;
1482 1.7 ozaki
1483 1.46 yamaguch mutex_exit(rxq->rxq_lock);
1484 1.22 ozaki if_percpuq_enqueue(ifp->if_percpuq, m);
1485 1.46 yamaguch mutex_enter(rxq->rxq_lock);
1486 1.7 ozaki
1487 1.43 yamaguch if (rxq->rxq_stopping)
1488 1.7 ozaki break;
1489 1.1 hannken }
1490 1.3 christos
1491 1.1 hannken return r;
1492 1.1 hannken }
1493 1.1 hannken
1494 1.1 hannken /* rx interrupt; call _dequeue above and schedule a softint */
1495 1.1 hannken static int
1496 1.1 hannken vioif_rx_vq_done(struct virtqueue *vq)
1497 1.1 hannken {
1498 1.46 yamaguch struct vioif_rxqueue *rxq = vq->vq_done_ctx;
1499 1.7 ozaki int r = 0;
1500 1.7 ozaki
1501 1.12 ozaki #ifdef VIOIF_SOFTINT_INTR
1502 1.12 ozaki KASSERT(!cpu_intr_p());
1503 1.12 ozaki #endif
1504 1.12 ozaki
1505 1.46 yamaguch mutex_enter(rxq->rxq_lock);
1506 1.7 ozaki
1507 1.43 yamaguch if (rxq->rxq_stopping)
1508 1.7 ozaki goto out;
1509 1.1 hannken
1510 1.46 yamaguch r = vioif_rx_deq_locked(rxq);
1511 1.1 hannken if (r)
1512 1.12 ozaki #ifdef VIOIF_SOFTINT_INTR
1513 1.46 yamaguch vioif_populate_rx_mbufs_locked(rxq);
1514 1.12 ozaki #else
1515 1.43 yamaguch softint_schedule(rxq->rxq_softint);
1516 1.12 ozaki #endif
1517 1.1 hannken
1518 1.7 ozaki out:
1519 1.46 yamaguch mutex_exit(rxq->rxq_lock);
1520 1.1 hannken return r;
1521 1.1 hannken }
1522 1.1 hannken
1523 1.39 dholland /* softint: enqueue receive requests for new incoming packets */
1524 1.1 hannken static void
1525 1.1 hannken vioif_rx_softint(void *arg)
1526 1.1 hannken {
1527 1.46 yamaguch struct vioif_rxqueue *rxq = arg;
1528 1.1 hannken
1529 1.46 yamaguch vioif_populate_rx_mbufs(rxq);
1530 1.1 hannken }
1531 1.1 hannken
1532 1.1 hannken /* free all the mbufs; called from if_stop(disable) */
1533 1.1 hannken static void
1534 1.46 yamaguch vioif_rx_drain(struct vioif_rxqueue *rxq)
1535 1.1 hannken {
1536 1.43 yamaguch struct virtqueue *vq = rxq->rxq_vq;
1537 1.1 hannken int i;
1538 1.1 hannken
1539 1.1 hannken for (i = 0; i < vq->vq_num; i++) {
1540 1.43 yamaguch if (rxq->rxq_mbufs[i] == NULL)
1541 1.1 hannken continue;
1542 1.46 yamaguch vioif_free_rx_mbuf(rxq, i);
1543 1.1 hannken }
1544 1.1 hannken }
1545 1.1 hannken
1546 1.1 hannken
1547 1.1 hannken /*
1548 1.1 hannken * Transmition implementation
1549 1.1 hannken */
1550 1.1 hannken /* actual transmission is done in if_start */
1551 1.1 hannken /* tx interrupt; dequeue and free mbufs */
1552 1.1 hannken /*
1553 1.1 hannken * tx interrupt is actually disabled; this should be called upon
1554 1.1 hannken * tx vq full and watchdog
1555 1.1 hannken */
1556 1.1 hannken static int
1557 1.1 hannken vioif_tx_vq_done(struct virtqueue *vq)
1558 1.1 hannken {
1559 1.1 hannken struct virtio_softc *vsc = vq->vq_owner;
1560 1.32 jdolecek struct vioif_softc *sc = device_private(virtio_child(vsc));
1561 1.23 pooka struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1562 1.46 yamaguch struct vioif_txqueue *txq = vq->vq_done_ctx;
1563 1.7 ozaki int r = 0;
1564 1.7 ozaki
1565 1.46 yamaguch mutex_enter(txq->txq_lock);
1566 1.7 ozaki
1567 1.43 yamaguch if (txq->txq_stopping)
1568 1.7 ozaki goto out;
1569 1.7 ozaki
1570 1.7 ozaki r = vioif_tx_vq_done_locked(vq);
1571 1.7 ozaki
1572 1.7 ozaki out:
1573 1.46 yamaguch mutex_exit(txq->txq_lock);
1574 1.46 yamaguch if (r) {
1575 1.28 ozaki if_schedule_deferred_start(ifp);
1576 1.46 yamaguch
1577 1.46 yamaguch KASSERT(txq->txq_deferred_transmit != NULL);
1578 1.46 yamaguch softint_schedule(txq->txq_deferred_transmit);
1579 1.46 yamaguch }
1580 1.7 ozaki return r;
1581 1.7 ozaki }
1582 1.7 ozaki
1583 1.7 ozaki static int
1584 1.7 ozaki vioif_tx_vq_done_locked(struct virtqueue *vq)
1585 1.7 ozaki {
1586 1.7 ozaki struct virtio_softc *vsc = vq->vq_owner;
1587 1.32 jdolecek struct vioif_softc *sc = device_private(virtio_child(vsc));
1588 1.46 yamaguch struct vioif_txqueue *txq = vq->vq_done_ctx;
1589 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1590 1.1 hannken struct mbuf *m;
1591 1.1 hannken int r = 0;
1592 1.1 hannken int slot, len;
1593 1.1 hannken
1594 1.46 yamaguch KASSERT(mutex_owned(txq->txq_lock));
1595 1.7 ozaki
1596 1.1 hannken while (virtio_dequeue(vsc, vq, &slot, &len) == 0) {
1597 1.1 hannken r++;
1598 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
1599 1.1 hannken 0, sizeof(struct virtio_net_hdr),
1600 1.1 hannken BUS_DMASYNC_POSTWRITE);
1601 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
1602 1.43 yamaguch 0, txq->txq_dmamaps[slot]->dm_mapsize,
1603 1.1 hannken BUS_DMASYNC_POSTWRITE);
1604 1.43 yamaguch m = txq->txq_mbufs[slot];
1605 1.43 yamaguch bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]);
1606 1.46 yamaguch txq->txq_mbufs[slot] = NULL;
1607 1.1 hannken virtio_dequeue_commit(vsc, vq, slot);
1608 1.1 hannken ifp->if_opackets++;
1609 1.1 hannken m_freem(m);
1610 1.1 hannken }
1611 1.1 hannken
1612 1.1 hannken if (r)
1613 1.1 hannken ifp->if_flags &= ~IFF_OACTIVE;
1614 1.1 hannken return r;
1615 1.1 hannken }
1616 1.1 hannken
1617 1.1 hannken /* free all the mbufs already put on vq; called from if_stop(disable) */
1618 1.1 hannken static void
1619 1.46 yamaguch vioif_tx_drain(struct vioif_txqueue *txq)
1620 1.1 hannken {
1621 1.43 yamaguch struct virtqueue *vq = txq->txq_vq;
1622 1.46 yamaguch struct virtio_softc *vsc = vq->vq_owner;
1623 1.1 hannken int i;
1624 1.1 hannken
1625 1.43 yamaguch KASSERT(txq->txq_stopping);
1626 1.7 ozaki
1627 1.1 hannken for (i = 0; i < vq->vq_num; i++) {
1628 1.43 yamaguch if (txq->txq_mbufs[i] == NULL)
1629 1.1 hannken continue;
1630 1.43 yamaguch bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]);
1631 1.43 yamaguch m_freem(txq->txq_mbufs[i]);
1632 1.43 yamaguch txq->txq_mbufs[i] = NULL;
1633 1.1 hannken }
1634 1.1 hannken }
1635 1.1 hannken
1636 1.1 hannken /*
1637 1.1 hannken * Control vq
1638 1.1 hannken */
1639 1.1 hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
1640 1.44 yamaguch static void
1641 1.44 yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
1642 1.1 hannken {
1643 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1644 1.1 hannken
1645 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
1646 1.43 yamaguch while (ctrlq->ctrlq_inuse != FREE)
1647 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
1648 1.43 yamaguch ctrlq->ctrlq_inuse = INUSE;
1649 1.44 yamaguch ctrlq->ctrlq_owner = curlwp;
1650 1.44 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
1651 1.44 yamaguch }
1652 1.44 yamaguch
1653 1.44 yamaguch static void
1654 1.44 yamaguch vioif_ctrl_release(struct vioif_softc *sc)
1655 1.44 yamaguch {
1656 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1657 1.44 yamaguch
1658 1.44 yamaguch KASSERT(ctrlq->ctrlq_inuse != FREE);
1659 1.44 yamaguch KASSERT(ctrlq->ctrlq_owner == curlwp);
1660 1.44 yamaguch
1661 1.44 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
1662 1.44 yamaguch ctrlq->ctrlq_inuse = FREE;
1663 1.46 yamaguch ctrlq->ctrlq_owner = NULL;
1664 1.44 yamaguch cv_signal(&ctrlq->ctrlq_wait);
1665 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
1666 1.44 yamaguch }
1667 1.44 yamaguch
1668 1.44 yamaguch static int
1669 1.44 yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
1670 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
1671 1.44 yamaguch {
1672 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1673 1.44 yamaguch int i, r, loaded;
1674 1.44 yamaguch
1675 1.44 yamaguch loaded = 0;
1676 1.44 yamaguch for (i = 0; i < nspecs; i++) {
1677 1.44 yamaguch r = bus_dmamap_load(virtio_dmat(vsc),
1678 1.44 yamaguch specs[i].dmamap, specs[i].buf, specs[i].bufsize,
1679 1.44 yamaguch NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT);
1680 1.44 yamaguch if (r) {
1681 1.44 yamaguch printf("%s: control command dmamap load failed, "
1682 1.44 yamaguch "error code %d\n", device_xname(sc->sc_dev), r);
1683 1.44 yamaguch goto err;
1684 1.44 yamaguch }
1685 1.44 yamaguch loaded++;
1686 1.44 yamaguch
1687 1.44 yamaguch }
1688 1.44 yamaguch
1689 1.44 yamaguch return r;
1690 1.44 yamaguch
1691 1.44 yamaguch err:
1692 1.44 yamaguch for (i = 0; i < loaded; i++) {
1693 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
1694 1.44 yamaguch }
1695 1.44 yamaguch
1696 1.44 yamaguch return r;
1697 1.44 yamaguch }
1698 1.44 yamaguch
1699 1.44 yamaguch static void
1700 1.44 yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
1701 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
1702 1.44 yamaguch {
1703 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1704 1.44 yamaguch int i;
1705 1.44 yamaguch
1706 1.44 yamaguch for (i = 0; i < nspecs; i++) {
1707 1.44 yamaguch bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
1708 1.44 yamaguch }
1709 1.44 yamaguch }
1710 1.44 yamaguch
1711 1.44 yamaguch static int
1712 1.44 yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
1713 1.44 yamaguch struct vioif_ctrl_cmdspec *specs, int nspecs)
1714 1.44 yamaguch {
1715 1.44 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1716 1.44 yamaguch struct virtqueue *vq = ctrlq->ctrlq_vq;
1717 1.44 yamaguch struct virtio_softc *vsc = sc->sc_virtio;
1718 1.44 yamaguch int i, r, slot;
1719 1.43 yamaguch
1720 1.44 yamaguch ctrlq->ctrlq_cmd->class = class;
1721 1.43 yamaguch ctrlq->ctrlq_cmd->command = cmd;
1722 1.1 hannken
1723 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
1724 1.1 hannken 0, sizeof(struct virtio_net_ctrl_cmd),
1725 1.1 hannken BUS_DMASYNC_PREWRITE);
1726 1.44 yamaguch for (i = 0; i < nspecs; i++) {
1727 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
1728 1.44 yamaguch 0, specs[i].bufsize,
1729 1.44 yamaguch BUS_DMASYNC_PREWRITE);
1730 1.44 yamaguch }
1731 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
1732 1.1 hannken 0, sizeof(struct virtio_net_ctrl_status),
1733 1.1 hannken BUS_DMASYNC_PREREAD);
1734 1.1 hannken
1735 1.1 hannken r = virtio_enqueue_prep(vsc, vq, &slot);
1736 1.1 hannken if (r != 0)
1737 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1738 1.44 yamaguch r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
1739 1.1 hannken if (r != 0)
1740 1.1 hannken panic("%s: control vq busy!?", device_xname(sc->sc_dev));
1741 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
1742 1.44 yamaguch for (i = 0; i < nspecs; i++) {
1743 1.44 yamaguch virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
1744 1.44 yamaguch }
1745 1.43 yamaguch virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
1746 1.1 hannken virtio_enqueue_commit(vsc, vq, slot, true);
1747 1.1 hannken
1748 1.1 hannken /* wait for done */
1749 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
1750 1.43 yamaguch while (ctrlq->ctrlq_inuse != DONE)
1751 1.43 yamaguch cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
1752 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
1753 1.1 hannken /* already dequeueued */
1754 1.1 hannken
1755 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
1756 1.1 hannken sizeof(struct virtio_net_ctrl_cmd),
1757 1.1 hannken BUS_DMASYNC_POSTWRITE);
1758 1.44 yamaguch for (i = 0; i < nspecs; i++) {
1759 1.44 yamaguch bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
1760 1.44 yamaguch specs[i].bufsize,
1761 1.44 yamaguch BUS_DMASYNC_POSTWRITE);
1762 1.44 yamaguch }
1763 1.43 yamaguch bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
1764 1.1 hannken sizeof(struct virtio_net_ctrl_status),
1765 1.1 hannken BUS_DMASYNC_POSTREAD);
1766 1.1 hannken
1767 1.43 yamaguch if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
1768 1.1 hannken r = 0;
1769 1.1 hannken else {
1770 1.1 hannken printf("%s: failed setting rx mode\n",
1771 1.1 hannken device_xname(sc->sc_dev));
1772 1.1 hannken r = EIO;
1773 1.1 hannken }
1774 1.1 hannken
1775 1.44 yamaguch return r;
1776 1.44 yamaguch }
1777 1.44 yamaguch
1778 1.44 yamaguch static int
1779 1.44 yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
1780 1.44 yamaguch {
1781 1.44 yamaguch struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
1782 1.44 yamaguch struct vioif_ctrl_cmdspec specs[1];
1783 1.44 yamaguch int r;
1784 1.44 yamaguch
1785 1.44 yamaguch if (!sc->sc_has_ctrl)
1786 1.44 yamaguch return ENOTSUP;
1787 1.44 yamaguch
1788 1.44 yamaguch vioif_ctrl_acquire(sc);
1789 1.44 yamaguch
1790 1.44 yamaguch rx->onoff = onoff;
1791 1.44 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
1792 1.44 yamaguch specs[0].buf = rx;
1793 1.44 yamaguch specs[0].bufsize = sizeof(*rx);
1794 1.44 yamaguch
1795 1.44 yamaguch r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
1796 1.44 yamaguch specs, __arraycount(specs));
1797 1.3 christos
1798 1.44 yamaguch vioif_ctrl_release(sc);
1799 1.1 hannken return r;
1800 1.1 hannken }
1801 1.1 hannken
1802 1.1 hannken static int
1803 1.1 hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
1804 1.1 hannken {
1805 1.1 hannken int r;
1806 1.1 hannken
1807 1.1 hannken r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
1808 1.1 hannken
1809 1.1 hannken return r;
1810 1.1 hannken }
1811 1.1 hannken
1812 1.1 hannken static int
1813 1.1 hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
1814 1.1 hannken {
1815 1.1 hannken int r;
1816 1.1 hannken
1817 1.1 hannken r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
1818 1.1 hannken
1819 1.1 hannken return r;
1820 1.1 hannken }
1821 1.1 hannken
1822 1.1 hannken /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
1823 1.1 hannken static int
1824 1.1 hannken vioif_set_rx_filter(struct vioif_softc *sc)
1825 1.1 hannken {
1826 1.43 yamaguch /* filter already set in ctrlq->ctrlq_mac_tbl */
1827 1.44 yamaguch struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
1828 1.44 yamaguch struct vioif_ctrl_cmdspec specs[2];
1829 1.44 yamaguch int nspecs = __arraycount(specs);
1830 1.44 yamaguch int r;
1831 1.44 yamaguch
1832 1.44 yamaguch mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
1833 1.44 yamaguch mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
1834 1.1 hannken
1835 1.32 jdolecek if (!sc->sc_has_ctrl)
1836 1.1 hannken return ENOTSUP;
1837 1.1 hannken
1838 1.44 yamaguch vioif_ctrl_acquire(sc);
1839 1.1 hannken
1840 1.44 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
1841 1.44 yamaguch specs[0].buf = mac_tbl_uc;
1842 1.44 yamaguch specs[0].bufsize = sizeof(*mac_tbl_uc)
1843 1.44 yamaguch + (ETHER_ADDR_LEN * mac_tbl_uc->nentries);
1844 1.44 yamaguch
1845 1.44 yamaguch specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
1846 1.44 yamaguch specs[1].buf = mac_tbl_mc;
1847 1.44 yamaguch specs[1].bufsize = sizeof(*mac_tbl_mc)
1848 1.44 yamaguch + (ETHER_ADDR_LEN * mac_tbl_mc->nentries);
1849 1.1 hannken
1850 1.44 yamaguch r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
1851 1.44 yamaguch if (r != 0)
1852 1.1 hannken goto out;
1853 1.1 hannken
1854 1.44 yamaguch r = vioif_ctrl_send_command(sc,
1855 1.44 yamaguch VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
1856 1.44 yamaguch specs, nspecs);
1857 1.1 hannken
1858 1.44 yamaguch vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
1859 1.1 hannken
1860 1.1 hannken out:
1861 1.44 yamaguch vioif_ctrl_release(sc);
1862 1.3 christos
1863 1.1 hannken return r;
1864 1.1 hannken }
1865 1.1 hannken
1866 1.46 yamaguch static int
1867 1.46 yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
1868 1.46 yamaguch {
1869 1.46 yamaguch struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
1870 1.46 yamaguch struct vioif_ctrl_cmdspec specs[1];
1871 1.46 yamaguch int r;
1872 1.46 yamaguch
1873 1.46 yamaguch if (!sc->sc_has_ctrl)
1874 1.46 yamaguch return ENOTSUP;
1875 1.46 yamaguch
1876 1.46 yamaguch if (nvq_pairs <= 1)
1877 1.46 yamaguch return EINVAL;
1878 1.46 yamaguch
1879 1.46 yamaguch vioif_ctrl_acquire(sc);
1880 1.46 yamaguch
1881 1.46 yamaguch mq->virtqueue_pairs = nvq_pairs;
1882 1.46 yamaguch specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
1883 1.46 yamaguch specs[0].buf = mq;
1884 1.46 yamaguch specs[0].bufsize = sizeof(*mq);
1885 1.46 yamaguch
1886 1.46 yamaguch r = vioif_ctrl_send_command(sc,
1887 1.46 yamaguch VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
1888 1.46 yamaguch specs, __arraycount(specs));
1889 1.46 yamaguch
1890 1.46 yamaguch vioif_ctrl_release(sc);
1891 1.46 yamaguch
1892 1.46 yamaguch return r;
1893 1.46 yamaguch }
1894 1.46 yamaguch
1895 1.1 hannken /* ctrl vq interrupt; wake up the command issuer */
1896 1.1 hannken static int
1897 1.1 hannken vioif_ctrl_vq_done(struct virtqueue *vq)
1898 1.1 hannken {
1899 1.1 hannken struct virtio_softc *vsc = vq->vq_owner;
1900 1.46 yamaguch struct vioif_ctrlqueue *ctrlq = vq->vq_done_ctx;
1901 1.1 hannken int r, slot;
1902 1.1 hannken
1903 1.1 hannken r = virtio_dequeue(vsc, vq, &slot, NULL);
1904 1.1 hannken if (r == ENOENT)
1905 1.1 hannken return 0;
1906 1.1 hannken virtio_dequeue_commit(vsc, vq, slot);
1907 1.1 hannken
1908 1.43 yamaguch mutex_enter(&ctrlq->ctrlq_wait_lock);
1909 1.43 yamaguch ctrlq->ctrlq_inuse = DONE;
1910 1.43 yamaguch cv_signal(&ctrlq->ctrlq_wait);
1911 1.43 yamaguch mutex_exit(&ctrlq->ctrlq_wait_lock);
1912 1.1 hannken
1913 1.1 hannken return 1;
1914 1.1 hannken }
1915 1.1 hannken
1916 1.1 hannken /*
1917 1.1 hannken * If IFF_PROMISC requested, set promiscuous
1918 1.1 hannken * If multicast filter small enough (<=MAXENTRIES) set rx filter
1919 1.1 hannken * If large multicast filter exist use ALLMULTI
1920 1.1 hannken */
1921 1.1 hannken /*
1922 1.1 hannken * If setting rx filter fails fall back to ALLMULTI
1923 1.1 hannken * If ALLMULTI fails fall back to PROMISC
1924 1.1 hannken */
1925 1.1 hannken static int
1926 1.1 hannken vioif_rx_filter(struct vioif_softc *sc)
1927 1.1 hannken {
1928 1.1 hannken struct ifnet *ifp = &sc->sc_ethercom.ec_if;
1929 1.1 hannken struct ether_multi *enm;
1930 1.1 hannken struct ether_multistep step;
1931 1.43 yamaguch struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
1932 1.1 hannken int nentries;
1933 1.1 hannken int promisc = 0, allmulti = 0, rxfilter = 0;
1934 1.1 hannken int r;
1935 1.1 hannken
1936 1.32 jdolecek if (!sc->sc_has_ctrl) { /* no ctrl vq; always promisc */
1937 1.1 hannken ifp->if_flags |= IFF_PROMISC;
1938 1.1 hannken return 0;
1939 1.1 hannken }
1940 1.1 hannken
1941 1.1 hannken if (ifp->if_flags & IFF_PROMISC) {
1942 1.1 hannken promisc = 1;
1943 1.1 hannken goto set;
1944 1.1 hannken }
1945 1.1 hannken
1946 1.1 hannken nentries = -1;
1947 1.30 ozaki ETHER_LOCK(&sc->sc_ethercom);
1948 1.1 hannken ETHER_FIRST_MULTI(step, &sc->sc_ethercom, enm);
1949 1.1 hannken while (nentries++, enm != NULL) {
1950 1.1 hannken if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
1951 1.1 hannken allmulti = 1;
1952 1.31 ozaki goto set_unlock;
1953 1.1 hannken }
1954 1.1 hannken if (memcmp(enm->enm_addrlo, enm->enm_addrhi,
1955 1.1 hannken ETHER_ADDR_LEN)) {
1956 1.1 hannken allmulti = 1;
1957 1.31 ozaki goto set_unlock;
1958 1.1 hannken }
1959 1.43 yamaguch memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
1960 1.1 hannken enm->enm_addrlo, ETHER_ADDR_LEN);
1961 1.1 hannken ETHER_NEXT_MULTI(step, enm);
1962 1.1 hannken }
1963 1.1 hannken rxfilter = 1;
1964 1.1 hannken
1965 1.31 ozaki set_unlock:
1966 1.30 ozaki ETHER_UNLOCK(&sc->sc_ethercom);
1967 1.30 ozaki
1968 1.31 ozaki set:
1969 1.1 hannken if (rxfilter) {
1970 1.43 yamaguch ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
1971 1.43 yamaguch ctrlq->ctrlq_mac_tbl_mc->nentries = nentries;
1972 1.1 hannken r = vioif_set_rx_filter(sc);
1973 1.1 hannken if (r != 0) {
1974 1.1 hannken rxfilter = 0;
1975 1.1 hannken allmulti = 1; /* fallback */
1976 1.1 hannken }
1977 1.1 hannken } else {
1978 1.1 hannken /* remove rx filter */
1979 1.43 yamaguch ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
1980 1.43 yamaguch ctrlq->ctrlq_mac_tbl_mc->nentries = 0;
1981 1.1 hannken r = vioif_set_rx_filter(sc);
1982 1.1 hannken /* what to do on failure? */
1983 1.1 hannken }
1984 1.1 hannken if (allmulti) {
1985 1.1 hannken r = vioif_set_allmulti(sc, true);
1986 1.1 hannken if (r != 0) {
1987 1.1 hannken allmulti = 0;
1988 1.1 hannken promisc = 1; /* fallback */
1989 1.1 hannken }
1990 1.1 hannken } else {
1991 1.1 hannken r = vioif_set_allmulti(sc, false);
1992 1.1 hannken /* what to do on failure? */
1993 1.1 hannken }
1994 1.1 hannken if (promisc) {
1995 1.1 hannken r = vioif_set_promisc(sc, true);
1996 1.1 hannken } else {
1997 1.1 hannken r = vioif_set_promisc(sc, false);
1998 1.1 hannken }
1999 1.1 hannken
2000 1.1 hannken return r;
2001 1.1 hannken }
2002 1.1 hannken
2003 1.34 ozaki static bool
2004 1.34 ozaki vioif_is_link_up(struct vioif_softc *sc)
2005 1.34 ozaki {
2006 1.34 ozaki struct virtio_softc *vsc = sc->sc_virtio;
2007 1.34 ozaki uint16_t status;
2008 1.34 ozaki
2009 1.34 ozaki if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
2010 1.34 ozaki status = virtio_read_device_config_2(vsc,
2011 1.34 ozaki VIRTIO_NET_CONFIG_STATUS);
2012 1.34 ozaki else
2013 1.34 ozaki status = VIRTIO_NET_S_LINK_UP;
2014 1.34 ozaki
2015 1.34 ozaki return ((status & VIRTIO_NET_S_LINK_UP) != 0);
2016 1.34 ozaki }
2017 1.34 ozaki
2018 1.34 ozaki /* change link status */
2019 1.34 ozaki static void
2020 1.34 ozaki vioif_update_link_status(struct vioif_softc *sc)
2021 1.34 ozaki {
2022 1.34 ozaki struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2023 1.46 yamaguch struct vioif_txqueue *txq;
2024 1.34 ozaki bool active, changed;
2025 1.46 yamaguch int link, i;
2026 1.34 ozaki
2027 1.34 ozaki active = vioif_is_link_up(sc);
2028 1.34 ozaki changed = false;
2029 1.34 ozaki
2030 1.34 ozaki if (active) {
2031 1.34 ozaki if (!sc->sc_link_active)
2032 1.34 ozaki changed = true;
2033 1.34 ozaki
2034 1.34 ozaki link = LINK_STATE_UP;
2035 1.34 ozaki sc->sc_link_active = true;
2036 1.34 ozaki } else {
2037 1.34 ozaki if (sc->sc_link_active)
2038 1.34 ozaki changed = true;
2039 1.34 ozaki
2040 1.34 ozaki link = LINK_STATE_DOWN;
2041 1.34 ozaki sc->sc_link_active = false;
2042 1.34 ozaki }
2043 1.34 ozaki
2044 1.43 yamaguch if (changed) {
2045 1.46 yamaguch for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
2046 1.46 yamaguch txq = &sc->sc_txq[i];
2047 1.46 yamaguch
2048 1.46 yamaguch mutex_enter(txq->txq_lock);
2049 1.46 yamaguch txq->txq_link_active = sc->sc_link_active;
2050 1.46 yamaguch mutex_exit(txq->txq_lock);
2051 1.46 yamaguch }
2052 1.43 yamaguch
2053 1.34 ozaki if_link_state_change(ifp, link);
2054 1.43 yamaguch }
2055 1.34 ozaki }
2056 1.34 ozaki
2057 1.34 ozaki static int
2058 1.34 ozaki vioif_config_change(struct virtio_softc *vsc)
2059 1.34 ozaki {
2060 1.34 ozaki struct vioif_softc *sc = device_private(virtio_child(vsc));
2061 1.34 ozaki
2062 1.34 ozaki #ifdef VIOIF_SOFTINT_INTR
2063 1.34 ozaki struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2064 1.34 ozaki #endif
2065 1.34 ozaki
2066 1.34 ozaki #ifdef VIOIF_SOFTINT_INTR
2067 1.34 ozaki KASSERT(!cpu_intr_p());
2068 1.34 ozaki vioif_update_link_status(sc);
2069 1.34 ozaki vioif_start(ifp);
2070 1.34 ozaki #else
2071 1.34 ozaki softint_schedule(sc->sc_ctl_softint);
2072 1.34 ozaki #endif
2073 1.34 ozaki
2074 1.34 ozaki return 0;
2075 1.34 ozaki }
2076 1.34 ozaki
2077 1.34 ozaki static void
2078 1.34 ozaki vioif_ctl_softint(void *arg)
2079 1.34 ozaki {
2080 1.34 ozaki struct vioif_softc *sc = arg;
2081 1.34 ozaki struct ifnet *ifp = &sc->sc_ethercom.ec_if;
2082 1.34 ozaki
2083 1.34 ozaki vioif_update_link_status(sc);
2084 1.34 ozaki vioif_start(ifp);
2085 1.34 ozaki }
2086 1.34 ozaki
2087 1.26 pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
2088 1.26 pgoyette
2089 1.26 pgoyette #ifdef _MODULE
2090 1.26 pgoyette #include "ioconf.c"
2091 1.26 pgoyette #endif
2092 1.26 pgoyette
2093 1.26 pgoyette static int
2094 1.26 pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
2095 1.26 pgoyette {
2096 1.26 pgoyette int error = 0;
2097 1.26 pgoyette
2098 1.26 pgoyette #ifdef _MODULE
2099 1.26 pgoyette switch (cmd) {
2100 1.26 pgoyette case MODULE_CMD_INIT:
2101 1.26 pgoyette error = config_init_component(cfdriver_ioconf_if_vioif,
2102 1.26 pgoyette cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2103 1.26 pgoyette break;
2104 1.26 pgoyette case MODULE_CMD_FINI:
2105 1.26 pgoyette error = config_fini_component(cfdriver_ioconf_if_vioif,
2106 1.26 pgoyette cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
2107 1.26 pgoyette break;
2108 1.26 pgoyette default:
2109 1.26 pgoyette error = ENOTTY;
2110 1.26 pgoyette break;
2111 1.26 pgoyette }
2112 1.26 pgoyette #endif
2113 1.26 pgoyette
2114 1.26 pgoyette return error;
2115 1.26 pgoyette }
2116