Home | History | Annotate | Line # | Download | only in pci
if_vioif.c revision 1.10
      1 /*	$NetBSD: if_vioif.c,v 1.10 2014/10/08 03:34:44 ozaki-r Exp $	*/
      2 
      3 /*
      4  * Copyright (c) 2010 Minoura Makoto.
      5  * All rights reserved.
      6  *
      7  * Redistribution and use in source and binary forms, with or without
      8  * modification, are permitted provided that the following conditions
      9  * are met:
     10  * 1. Redistributions of source code must retain the above copyright
     11  *    notice, this list of conditions and the following disclaimer.
     12  * 2. Redistributions in binary form must reproduce the above copyright
     13  *    notice, this list of conditions and the following disclaimer in the
     14  *    documentation and/or other materials provided with the distribution.
     15  *
     16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26  */
     27 
     28 #include <sys/cdefs.h>
     29 __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.10 2014/10/08 03:34:44 ozaki-r Exp $");
     30 
     31 #include <sys/param.h>
     32 #include <sys/systm.h>
     33 #include <sys/kernel.h>
     34 #include <sys/bus.h>
     35 #include <sys/condvar.h>
     36 #include <sys/device.h>
     37 #include <sys/intr.h>
     38 #include <sys/kmem.h>
     39 #include <sys/mbuf.h>
     40 #include <sys/mutex.h>
     41 #include <sys/sockio.h>
     42 
     43 #include <dev/pci/pcidevs.h>
     44 #include <dev/pci/pcireg.h>
     45 #include <dev/pci/pcivar.h>
     46 #include <dev/pci/virtioreg.h>
     47 #include <dev/pci/virtiovar.h>
     48 
     49 #include <net/if.h>
     50 #include <net/if_media.h>
     51 #include <net/if_ether.h>
     52 
     53 #include <net/bpf.h>
     54 
     55 
     56 #ifdef NET_MPSAFE
     57 #define VIOIF_MPSAFE	1
     58 #endif
     59 
     60 /*
     61  * if_vioifreg.h:
     62  */
     63 /* Configuration registers */
     64 #define VIRTIO_NET_CONFIG_MAC		0 /* 8bit x 6byte */
     65 #define VIRTIO_NET_CONFIG_STATUS	6 /* 16bit */
     66 
     67 /* Feature bits */
     68 #define VIRTIO_NET_F_CSUM	(1<<0)
     69 #define VIRTIO_NET_F_GUEST_CSUM	(1<<1)
     70 #define VIRTIO_NET_F_MAC	(1<<5)
     71 #define VIRTIO_NET_F_GSO	(1<<6)
     72 #define VIRTIO_NET_F_GUEST_TSO4	(1<<7)
     73 #define VIRTIO_NET_F_GUEST_TSO6	(1<<8)
     74 #define VIRTIO_NET_F_GUEST_ECN	(1<<9)
     75 #define VIRTIO_NET_F_GUEST_UFO	(1<<10)
     76 #define VIRTIO_NET_F_HOST_TSO4	(1<<11)
     77 #define VIRTIO_NET_F_HOST_TSO6	(1<<12)
     78 #define VIRTIO_NET_F_HOST_ECN	(1<<13)
     79 #define VIRTIO_NET_F_HOST_UFO	(1<<14)
     80 #define VIRTIO_NET_F_MRG_RXBUF	(1<<15)
     81 #define VIRTIO_NET_F_STATUS	(1<<16)
     82 #define VIRTIO_NET_F_CTRL_VQ	(1<<17)
     83 #define VIRTIO_NET_F_CTRL_RX	(1<<18)
     84 #define VIRTIO_NET_F_CTRL_VLAN	(1<<19)
     85 
     86 /* Status */
     87 #define VIRTIO_NET_S_LINK_UP	1
     88 
     89 /* Packet header structure */
     90 struct virtio_net_hdr {
     91 	uint8_t		flags;
     92 	uint8_t		gso_type;
     93 	uint16_t	hdr_len;
     94 	uint16_t	gso_size;
     95 	uint16_t	csum_start;
     96 	uint16_t	csum_offset;
     97 #if 0
     98 	uint16_t	num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */
     99 #endif
    100 } __packed;
    101 
    102 #define VIRTIO_NET_HDR_F_NEEDS_CSUM	1 /* flags */
    103 #define VIRTIO_NET_HDR_GSO_NONE		0 /* gso_type */
    104 #define VIRTIO_NET_HDR_GSO_TCPV4	1 /* gso_type */
    105 #define VIRTIO_NET_HDR_GSO_UDP		3 /* gso_type */
    106 #define VIRTIO_NET_HDR_GSO_TCPV6	4 /* gso_type */
    107 #define VIRTIO_NET_HDR_GSO_ECN		0x80 /* gso_type, |'ed */
    108 
    109 #define VIRTIO_NET_MAX_GSO_LEN		(65536+ETHER_HDR_LEN)
    110 
    111 /* Control virtqueue */
    112 struct virtio_net_ctrl_cmd {
    113 	uint8_t	class;
    114 	uint8_t	command;
    115 } __packed;
    116 #define VIRTIO_NET_CTRL_RX		0
    117 # define VIRTIO_NET_CTRL_RX_PROMISC	0
    118 # define VIRTIO_NET_CTRL_RX_ALLMULTI	1
    119 
    120 #define VIRTIO_NET_CTRL_MAC		1
    121 # define VIRTIO_NET_CTRL_MAC_TABLE_SET	0
    122 
    123 #define VIRTIO_NET_CTRL_VLAN		2
    124 # define VIRTIO_NET_CTRL_VLAN_ADD	0
    125 # define VIRTIO_NET_CTRL_VLAN_DEL	1
    126 
    127 struct virtio_net_ctrl_status {
    128 	uint8_t	ack;
    129 } __packed;
    130 #define VIRTIO_NET_OK			0
    131 #define VIRTIO_NET_ERR			1
    132 
    133 struct virtio_net_ctrl_rx {
    134 	uint8_t	onoff;
    135 } __packed;
    136 
    137 struct virtio_net_ctrl_mac_tbl {
    138 	uint32_t nentries;
    139 	uint8_t macs[][ETHER_ADDR_LEN];
    140 } __packed;
    141 
    142 struct virtio_net_ctrl_vlan {
    143 	uint16_t id;
    144 } __packed;
    145 
    146 
    147 /*
    148  * if_vioifvar.h:
    149  */
    150 struct vioif_softc {
    151 	device_t		sc_dev;
    152 
    153 	struct virtio_softc	*sc_virtio;
    154 	struct virtqueue	sc_vq[3];
    155 
    156 	uint8_t			sc_mac[ETHER_ADDR_LEN];
    157 	struct ethercom		sc_ethercom;
    158 	short			sc_deferred_init_done;
    159 
    160 	/* bus_dmamem */
    161 	bus_dma_segment_t	sc_hdr_segs[1];
    162 	struct virtio_net_hdr	*sc_hdrs;
    163 #define sc_rx_hdrs	sc_hdrs
    164 	struct virtio_net_hdr	*sc_tx_hdrs;
    165 	struct virtio_net_ctrl_cmd *sc_ctrl_cmd;
    166 	struct virtio_net_ctrl_status *sc_ctrl_status;
    167 	struct virtio_net_ctrl_rx *sc_ctrl_rx;
    168 	struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_uc;
    169 	struct virtio_net_ctrl_mac_tbl *sc_ctrl_mac_tbl_mc;
    170 
    171 	/* kmem */
    172 	bus_dmamap_t		*sc_arrays;
    173 #define sc_rxhdr_dmamaps sc_arrays
    174 	bus_dmamap_t		*sc_txhdr_dmamaps;
    175 	bus_dmamap_t		*sc_rx_dmamaps;
    176 	bus_dmamap_t		*sc_tx_dmamaps;
    177 	struct mbuf		**sc_rx_mbufs;
    178 	struct mbuf		**sc_tx_mbufs;
    179 
    180 	bus_dmamap_t		sc_ctrl_cmd_dmamap;
    181 	bus_dmamap_t		sc_ctrl_status_dmamap;
    182 	bus_dmamap_t		sc_ctrl_rx_dmamap;
    183 	bus_dmamap_t		sc_ctrl_tbl_uc_dmamap;
    184 	bus_dmamap_t		sc_ctrl_tbl_mc_dmamap;
    185 
    186 	void			*sc_rx_softint;
    187 
    188 	enum {
    189 		FREE, INUSE, DONE
    190 	}			sc_ctrl_inuse;
    191 	kcondvar_t		sc_ctrl_wait;
    192 	kmutex_t		sc_ctrl_wait_lock;
    193 	kmutex_t		*sc_tx_lock;
    194 	kmutex_t		*sc_rx_lock;
    195 	bool			sc_stopping;
    196 };
    197 #define VIRTIO_NET_TX_MAXNSEGS		(16) /* XXX */
    198 #define VIRTIO_NET_CTRL_MAC_MAXENTRIES	(64) /* XXX */
    199 
    200 #define VIOIF_TX_LOCK(_sc)	if ((_sc)->sc_tx_lock) mutex_enter((_sc)->sc_tx_lock)
    201 #define VIOIF_TX_UNLOCK(_sc)	if ((_sc)->sc_tx_lock) mutex_exit((_sc)->sc_tx_lock)
    202 #define VIOIF_TX_LOCKED(_sc)	(!(_sc)->sc_tx_lock || mutex_owned((_sc)->sc_tx_lock))
    203 #define VIOIF_RX_LOCK(_sc)	if ((_sc)->sc_rx_lock) mutex_enter((_sc)->sc_rx_lock)
    204 #define VIOIF_RX_UNLOCK(_sc)	if ((_sc)->sc_rx_lock) mutex_exit((_sc)->sc_rx_lock)
    205 #define VIOIF_RX_LOCKED(_sc)	(!(_sc)->sc_rx_lock || mutex_owned((_sc)->sc_rx_lock))
    206 
    207 /* cfattach interface functions */
    208 static int	vioif_match(device_t, cfdata_t, void *);
    209 static void	vioif_attach(device_t, device_t, void *);
    210 static void	vioif_deferred_init(device_t);
    211 
    212 /* ifnet interface functions */
    213 static int	vioif_init(struct ifnet *);
    214 static void	vioif_stop(struct ifnet *, int);
    215 static void	vioif_start(struct ifnet *);
    216 static int	vioif_ioctl(struct ifnet *, u_long, void *);
    217 static void	vioif_watchdog(struct ifnet *);
    218 
    219 /* rx */
    220 static int	vioif_add_rx_mbuf(struct vioif_softc *, int);
    221 static void	vioif_free_rx_mbuf(struct vioif_softc *, int);
    222 static void	vioif_populate_rx_mbufs(struct vioif_softc *);
    223 static int	vioif_rx_deq(struct vioif_softc *);
    224 static int	vioif_rx_deq_locked(struct vioif_softc *);
    225 static int	vioif_rx_vq_done(struct virtqueue *);
    226 static void	vioif_rx_softint(void *);
    227 static void	vioif_rx_drain(struct vioif_softc *);
    228 
    229 /* tx */
    230 static int	vioif_tx_vq_done(struct virtqueue *);
    231 static int	vioif_tx_vq_done_locked(struct virtqueue *);
    232 static void	vioif_tx_drain(struct vioif_softc *);
    233 
    234 /* other control */
    235 static int	vioif_updown(struct vioif_softc *, bool);
    236 static int	vioif_ctrl_rx(struct vioif_softc *, int, bool);
    237 static int	vioif_set_promisc(struct vioif_softc *, bool);
    238 static int	vioif_set_allmulti(struct vioif_softc *, bool);
    239 static int	vioif_set_rx_filter(struct vioif_softc *);
    240 static int	vioif_rx_filter(struct vioif_softc *);
    241 static int	vioif_ctrl_vq_done(struct virtqueue *);
    242 
    243 CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
    244 		  vioif_match, vioif_attach, NULL, NULL);
    245 
    246 static int
    247 vioif_match(device_t parent, cfdata_t match, void *aux)
    248 {
    249 	struct virtio_softc *va = aux;
    250 
    251 	if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK)
    252 		return 1;
    253 
    254 	return 0;
    255 }
    256 
    257 /* allocate memory */
    258 /*
    259  * dma memory is used for:
    260  *   sc_rx_hdrs[slot]:	 metadata array for recieved frames (READ)
    261  *   sc_tx_hdrs[slot]:	 metadata array for frames to be sent (WRITE)
    262  *   sc_ctrl_cmd:	 command to be sent via ctrl vq (WRITE)
    263  *   sc_ctrl_status:	 return value for a command via ctrl vq (READ)
    264  *   sc_ctrl_rx:	 parameter for a VIRTIO_NET_CTRL_RX class command
    265  *			 (WRITE)
    266  *   sc_ctrl_mac_tbl_uc: unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
    267  *			 class command (WRITE)
    268  *   sc_ctrl_mac_tbl_mc: multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
    269  *			 class command (WRITE)
    270  * sc_ctrl_* structures are allocated only one each; they are protected by
    271  * sc_ctrl_inuse variable and sc_ctrl_wait condvar.
    272  */
    273 /*
    274  * dynamically allocated memory is used for:
    275  *   sc_rxhdr_dmamaps[slot]:	bus_dmamap_t array for sc_rx_hdrs[slot]
    276  *   sc_txhdr_dmamaps[slot]:	bus_dmamap_t array for sc_tx_hdrs[slot]
    277  *   sc_rx_dmamaps[slot]:	bus_dmamap_t array for recieved payload
    278  *   sc_tx_dmamaps[slot]:	bus_dmamap_t array for sent payload
    279  *   sc_rx_mbufs[slot]:		mbuf pointer array for recieved frames
    280  *   sc_tx_mbufs[slot]:		mbuf pointer array for sent frames
    281  */
    282 static int
    283 vioif_alloc_mems(struct vioif_softc *sc)
    284 {
    285 	struct virtio_softc *vsc = sc->sc_virtio;
    286 	int allocsize, allocsize2, r, rsegs, i;
    287 	void *vaddr;
    288 	intptr_t p;
    289 	int rxqsize, txqsize;
    290 
    291 	rxqsize = vsc->sc_vqs[0].vq_num;
    292 	txqsize = vsc->sc_vqs[1].vq_num;
    293 
    294 	allocsize = sizeof(struct virtio_net_hdr) * rxqsize;
    295 	allocsize += sizeof(struct virtio_net_hdr) * txqsize;
    296 	if (vsc->sc_nvqs == 3) {
    297 		allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1;
    298 		allocsize += sizeof(struct virtio_net_ctrl_status) * 1;
    299 		allocsize += sizeof(struct virtio_net_ctrl_rx) * 1;
    300 		allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
    301 			+ sizeof(struct virtio_net_ctrl_mac_tbl)
    302 			+ ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
    303 	}
    304 	r = bus_dmamem_alloc(vsc->sc_dmat, allocsize, 0, 0,
    305 			     &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
    306 	if (r != 0) {
    307 		aprint_error_dev(sc->sc_dev,
    308 				 "DMA memory allocation failed, size %d, "
    309 				 "error code %d\n", allocsize, r);
    310 		goto err_none;
    311 	}
    312 	r = bus_dmamem_map(vsc->sc_dmat,
    313 			   &sc->sc_hdr_segs[0], 1, allocsize,
    314 			   &vaddr, BUS_DMA_NOWAIT);
    315 	if (r != 0) {
    316 		aprint_error_dev(sc->sc_dev,
    317 				 "DMA memory map failed, "
    318 				 "error code %d\n", r);
    319 		goto err_dmamem_alloc;
    320 	}
    321 	sc->sc_hdrs = vaddr;
    322 	memset(vaddr, 0, allocsize);
    323 	p = (intptr_t) vaddr;
    324 	p += sizeof(struct virtio_net_hdr) * rxqsize;
    325 #define P(name,size)	do { sc->sc_ ##name = (void*) p;	\
    326 			     p += size; } while (0)
    327 	P(tx_hdrs, sizeof(struct virtio_net_hdr) * txqsize);
    328 	if (vsc->sc_nvqs == 3) {
    329 		P(ctrl_cmd, sizeof(struct virtio_net_ctrl_cmd));
    330 		P(ctrl_status, sizeof(struct virtio_net_ctrl_status));
    331 		P(ctrl_rx, sizeof(struct virtio_net_ctrl_rx));
    332 		P(ctrl_mac_tbl_uc, sizeof(struct virtio_net_ctrl_mac_tbl));
    333 		P(ctrl_mac_tbl_mc,
    334 		  (sizeof(struct virtio_net_ctrl_mac_tbl)
    335 		   + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES));
    336 	}
    337 #undef P
    338 
    339 	allocsize2 = sizeof(bus_dmamap_t) * (rxqsize + txqsize);
    340 	allocsize2 += sizeof(bus_dmamap_t) * (rxqsize + txqsize);
    341 	allocsize2 += sizeof(struct mbuf*) * (rxqsize + txqsize);
    342 	sc->sc_arrays = kmem_zalloc(allocsize2, KM_SLEEP);
    343 	if (sc->sc_arrays == NULL)
    344 		goto err_dmamem_map;
    345 	sc->sc_txhdr_dmamaps = sc->sc_arrays + rxqsize;
    346 	sc->sc_rx_dmamaps = sc->sc_txhdr_dmamaps + txqsize;
    347 	sc->sc_tx_dmamaps = sc->sc_rx_dmamaps + rxqsize;
    348 	sc->sc_rx_mbufs = (void*) (sc->sc_tx_dmamaps + txqsize);
    349 	sc->sc_tx_mbufs = sc->sc_rx_mbufs + rxqsize;
    350 
    351 #define C(map, buf, size, nsegs, rw, usage)				\
    352 	do {								\
    353 		r = bus_dmamap_create(vsc->sc_dmat, size, nsegs, size, 0, \
    354 				      BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW,	\
    355 				      &sc->sc_ ##map);			\
    356 		if (r != 0) {						\
    357 			aprint_error_dev(sc->sc_dev,			\
    358 					 usage " dmamap creation failed, " \
    359 					 "error code %d\n", r);		\
    360 					 goto err_reqs;			\
    361 		}							\
    362 	} while (0)
    363 #define C_L1(map, buf, size, nsegs, rw, usage)				\
    364 	C(map, buf, size, nsegs, rw, usage);				\
    365 	do {								\
    366 		r = bus_dmamap_load(vsc->sc_dmat, sc->sc_ ##map,	\
    367 				    &sc->sc_ ##buf, size, NULL,		\
    368 				    BUS_DMA_ ##rw | BUS_DMA_NOWAIT);	\
    369 		if (r != 0) {						\
    370 			aprint_error_dev(sc->sc_dev,			\
    371 					 usage " dmamap load failed, "	\
    372 					 "error code %d\n", r);		\
    373 			goto err_reqs;					\
    374 		}							\
    375 	} while (0)
    376 #define C_L2(map, buf, size, nsegs, rw, usage)				\
    377 	C(map, buf, size, nsegs, rw, usage);				\
    378 	do {								\
    379 		r = bus_dmamap_load(vsc->sc_dmat, sc->sc_ ##map,	\
    380 				    sc->sc_ ##buf, size, NULL,		\
    381 				    BUS_DMA_ ##rw | BUS_DMA_NOWAIT);	\
    382 		if (r != 0) {						\
    383 			aprint_error_dev(sc->sc_dev,			\
    384 					 usage " dmamap load failed, "	\
    385 					 "error code %d\n", r);		\
    386 			goto err_reqs;					\
    387 		}							\
    388 	} while (0)
    389 	for (i = 0; i < rxqsize; i++) {
    390 		C_L1(rxhdr_dmamaps[i], rx_hdrs[i],
    391 		    sizeof(struct virtio_net_hdr), 1,
    392 		    READ, "rx header");
    393 		C(rx_dmamaps[i], NULL, MCLBYTES, 1, 0, "rx payload");
    394 	}
    395 
    396 	for (i = 0; i < txqsize; i++) {
    397 		C_L1(txhdr_dmamaps[i], rx_hdrs[i],
    398 		    sizeof(struct virtio_net_hdr), 1,
    399 		    WRITE, "tx header");
    400 		C(tx_dmamaps[i], NULL, ETHER_MAX_LEN, 256 /* XXX */, 0,
    401 		  "tx payload");
    402 	}
    403 
    404 	if (vsc->sc_nvqs == 3) {
    405 		/* control vq class & command */
    406 		C_L2(ctrl_cmd_dmamap, ctrl_cmd,
    407 		    sizeof(struct virtio_net_ctrl_cmd), 1, WRITE,
    408 		    "control command");
    409 
    410 		/* control vq status */
    411 		C_L2(ctrl_status_dmamap, ctrl_status,
    412 		    sizeof(struct virtio_net_ctrl_status), 1, READ,
    413 		    "control status");
    414 
    415 		/* control vq rx mode command parameter */
    416 		C_L2(ctrl_rx_dmamap, ctrl_rx,
    417 		    sizeof(struct virtio_net_ctrl_rx), 1, WRITE,
    418 		    "rx mode control command");
    419 
    420 		/* control vq MAC filter table for unicast */
    421 		/* do not load now since its length is variable */
    422 		C(ctrl_tbl_uc_dmamap, NULL,
    423 		  sizeof(struct virtio_net_ctrl_mac_tbl) + 0, 1, WRITE,
    424 		  "unicast MAC address filter command");
    425 
    426 		/* control vq MAC filter table for multicast */
    427 		C(ctrl_tbl_mc_dmamap, NULL,
    428 		  (sizeof(struct virtio_net_ctrl_mac_tbl)
    429 		   + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES),
    430 		  1, WRITE, "multicast MAC address filter command");
    431 	}
    432 #undef C_L2
    433 #undef C_L1
    434 #undef C
    435 
    436 	return 0;
    437 
    438 err_reqs:
    439 #define D(map)								\
    440 	do {								\
    441 		if (sc->sc_ ##map) {					\
    442 			bus_dmamap_destroy(vsc->sc_dmat, sc->sc_ ##map); \
    443 			sc->sc_ ##map = NULL;				\
    444 		}							\
    445 	} while (0)
    446 	D(ctrl_tbl_mc_dmamap);
    447 	D(ctrl_tbl_uc_dmamap);
    448 	D(ctrl_rx_dmamap);
    449 	D(ctrl_status_dmamap);
    450 	D(ctrl_cmd_dmamap);
    451 	for (i = 0; i < txqsize; i++) {
    452 		D(tx_dmamaps[i]);
    453 		D(txhdr_dmamaps[i]);
    454 	}
    455 	for (i = 0; i < rxqsize; i++) {
    456 		D(rx_dmamaps[i]);
    457 		D(rxhdr_dmamaps[i]);
    458 	}
    459 #undef D
    460 	if (sc->sc_arrays) {
    461 		kmem_free(sc->sc_arrays, allocsize2);
    462 		sc->sc_arrays = 0;
    463 	}
    464 err_dmamem_map:
    465 	bus_dmamem_unmap(vsc->sc_dmat, sc->sc_hdrs, allocsize);
    466 err_dmamem_alloc:
    467 	bus_dmamem_free(vsc->sc_dmat, &sc->sc_hdr_segs[0], 1);
    468 err_none:
    469 	return -1;
    470 }
    471 
    472 static void
    473 vioif_attach(device_t parent, device_t self, void *aux)
    474 {
    475 	struct vioif_softc *sc = device_private(self);
    476 	struct virtio_softc *vsc = device_private(parent);
    477 	uint32_t features;
    478 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    479 	u_int flags;
    480 
    481 	if (vsc->sc_child != NULL) {
    482 		aprint_normal(": child already attached for %s; "
    483 			      "something wrong...\n",
    484 			      device_xname(parent));
    485 		return;
    486 	}
    487 
    488 	sc->sc_dev = self;
    489 	sc->sc_virtio = vsc;
    490 
    491 	vsc->sc_child = self;
    492 	vsc->sc_ipl = IPL_NET;
    493 	vsc->sc_vqs = &sc->sc_vq[0];
    494 	vsc->sc_config_change = 0;
    495 	vsc->sc_intrhand = virtio_vq_intr;
    496 	vsc->sc_flags = 0;
    497 
    498 #ifdef VIOIF_MPSAFE
    499 	vsc->sc_flags |= VIRTIO_F_PCI_INTR_MPSAFE;
    500 #endif
    501 
    502 	features = virtio_negotiate_features(vsc,
    503 					     (VIRTIO_NET_F_MAC |
    504 					      VIRTIO_NET_F_STATUS |
    505 					      VIRTIO_NET_F_CTRL_VQ |
    506 					      VIRTIO_NET_F_CTRL_RX |
    507 					      VIRTIO_F_NOTIFY_ON_EMPTY));
    508 	if (features & VIRTIO_NET_F_MAC) {
    509 		sc->sc_mac[0] = virtio_read_device_config_1(vsc,
    510 						    VIRTIO_NET_CONFIG_MAC+0);
    511 		sc->sc_mac[1] = virtio_read_device_config_1(vsc,
    512 						    VIRTIO_NET_CONFIG_MAC+1);
    513 		sc->sc_mac[2] = virtio_read_device_config_1(vsc,
    514 						    VIRTIO_NET_CONFIG_MAC+2);
    515 		sc->sc_mac[3] = virtio_read_device_config_1(vsc,
    516 						    VIRTIO_NET_CONFIG_MAC+3);
    517 		sc->sc_mac[4] = virtio_read_device_config_1(vsc,
    518 						    VIRTIO_NET_CONFIG_MAC+4);
    519 		sc->sc_mac[5] = virtio_read_device_config_1(vsc,
    520 						    VIRTIO_NET_CONFIG_MAC+5);
    521 	} else {
    522 		/* code stolen from sys/net/if_tap.c */
    523 		struct timeval tv;
    524 		uint32_t ui;
    525 		getmicrouptime(&tv);
    526 		ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
    527 		memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
    528 		virtio_write_device_config_1(vsc,
    529 					     VIRTIO_NET_CONFIG_MAC+0,
    530 					     sc->sc_mac[0]);
    531 		virtio_write_device_config_1(vsc,
    532 					     VIRTIO_NET_CONFIG_MAC+1,
    533 					     sc->sc_mac[1]);
    534 		virtio_write_device_config_1(vsc,
    535 					     VIRTIO_NET_CONFIG_MAC+2,
    536 					     sc->sc_mac[2]);
    537 		virtio_write_device_config_1(vsc,
    538 					     VIRTIO_NET_CONFIG_MAC+3,
    539 					     sc->sc_mac[3]);
    540 		virtio_write_device_config_1(vsc,
    541 					     VIRTIO_NET_CONFIG_MAC+4,
    542 					     sc->sc_mac[4]);
    543 		virtio_write_device_config_1(vsc,
    544 					     VIRTIO_NET_CONFIG_MAC+5,
    545 					     sc->sc_mac[5]);
    546 	}
    547 	aprint_normal(": Ethernet address %s\n", ether_sprintf(sc->sc_mac));
    548 	aprint_naive("\n");
    549 
    550 	if (virtio_alloc_vq(vsc, &sc->sc_vq[0], 0,
    551 			    MCLBYTES+sizeof(struct virtio_net_hdr), 2,
    552 			    "rx") != 0) {
    553 		goto err;
    554 	}
    555 	vsc->sc_nvqs = 1;
    556 	sc->sc_vq[0].vq_done = vioif_rx_vq_done;
    557 	if (virtio_alloc_vq(vsc, &sc->sc_vq[1], 1,
    558 			    (sizeof(struct virtio_net_hdr)
    559 			     + (ETHER_MAX_LEN - ETHER_HDR_LEN)),
    560 			    VIRTIO_NET_TX_MAXNSEGS + 1,
    561 			    "tx") != 0) {
    562 		goto err;
    563 	}
    564 
    565 #ifdef VIOIF_MPSAFE
    566 	sc->sc_tx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
    567 	sc->sc_rx_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
    568 #else
    569 	sc->sc_tx_lock = NULL;
    570 	sc->sc_rx_lock = NULL;
    571 #endif
    572 	sc->sc_stopping = false;
    573 
    574 	vsc->sc_nvqs = 2;
    575 	sc->sc_vq[1].vq_done = vioif_tx_vq_done;
    576 	virtio_start_vq_intr(vsc, &sc->sc_vq[0]);
    577 	virtio_stop_vq_intr(vsc, &sc->sc_vq[1]); /* not urgent; do it later */
    578 	if ((features & VIRTIO_NET_F_CTRL_VQ)
    579 	    && (features & VIRTIO_NET_F_CTRL_RX)) {
    580 		if (virtio_alloc_vq(vsc, &sc->sc_vq[2], 2,
    581 				    NBPG, 1, "control") == 0) {
    582 			sc->sc_vq[2].vq_done = vioif_ctrl_vq_done;
    583 			cv_init(&sc->sc_ctrl_wait, "ctrl_vq");
    584 			mutex_init(&sc->sc_ctrl_wait_lock,
    585 				   MUTEX_DEFAULT, IPL_NET);
    586 			sc->sc_ctrl_inuse = FREE;
    587 			virtio_start_vq_intr(vsc, &sc->sc_vq[2]);
    588 			vsc->sc_nvqs = 3;
    589 		}
    590 	}
    591 
    592 #ifdef VIOIF_MPSAFE
    593 	flags = SOFTINT_NET | SOFTINT_MPSAFE;
    594 #else
    595 	flags = SOFTINT_NET;
    596 #endif
    597 	sc->sc_rx_softint = softint_establish(flags, vioif_rx_softint, sc);
    598 	if (sc->sc_rx_softint == NULL) {
    599 		aprint_error_dev(self, "cannot establish softint\n");
    600 		goto err;
    601 	}
    602 
    603 	if (vioif_alloc_mems(sc) < 0)
    604 		goto err;
    605 
    606 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
    607 	ifp->if_softc = sc;
    608 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
    609 	ifp->if_start = vioif_start;
    610 	ifp->if_ioctl = vioif_ioctl;
    611 	ifp->if_init = vioif_init;
    612 	ifp->if_stop = vioif_stop;
    613 	ifp->if_capabilities = 0;
    614 	ifp->if_watchdog = vioif_watchdog;
    615 
    616 	if_attach(ifp);
    617 	ether_ifattach(ifp, sc->sc_mac);
    618 
    619 	return;
    620 
    621 err:
    622 	if (sc->sc_tx_lock)
    623 		mutex_obj_free(sc->sc_tx_lock);
    624 	if (sc->sc_rx_lock)
    625 		mutex_obj_free(sc->sc_rx_lock);
    626 
    627 	if (vsc->sc_nvqs == 3) {
    628 		virtio_free_vq(vsc, &sc->sc_vq[2]);
    629 		cv_destroy(&sc->sc_ctrl_wait);
    630 		mutex_destroy(&sc->sc_ctrl_wait_lock);
    631 		vsc->sc_nvqs = 2;
    632 	}
    633 	if (vsc->sc_nvqs == 2) {
    634 		virtio_free_vq(vsc, &sc->sc_vq[1]);
    635 		vsc->sc_nvqs = 1;
    636 	}
    637 	if (vsc->sc_nvqs == 1) {
    638 		virtio_free_vq(vsc, &sc->sc_vq[0]);
    639 		vsc->sc_nvqs = 0;
    640 	}
    641 	vsc->sc_child = (void*)1;
    642 	return;
    643 }
    644 
    645 /* we need interrupts to make promiscuous mode off */
    646 static void
    647 vioif_deferred_init(device_t self)
    648 {
    649 	struct vioif_softc *sc = device_private(self);
    650 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    651 	int r;
    652 
    653 	if (ifp->if_flags & IFF_PROMISC)
    654 		return;
    655 
    656 	r =  vioif_set_promisc(sc, false);
    657 	if (r != 0)
    658 		aprint_error_dev(self, "resetting promisc mode failed, "
    659 				 "errror code %d\n", r);
    660 }
    661 
    662 /*
    663  * Interface functions for ifnet
    664  */
    665 static int
    666 vioif_init(struct ifnet *ifp)
    667 {
    668 	struct vioif_softc *sc = ifp->if_softc;
    669 
    670 	vioif_stop(ifp, 0);
    671 
    672 	if (!sc->sc_deferred_init_done) {
    673 		struct virtio_softc *vsc = sc->sc_virtio;
    674 
    675 		sc->sc_deferred_init_done = 1;
    676 		if (vsc->sc_nvqs == 3)
    677 			vioif_deferred_init(sc->sc_dev);
    678 	}
    679 
    680 	/* Have to set false before vioif_populate_rx_mbufs */
    681 	sc->sc_stopping = false;
    682 
    683 	vioif_populate_rx_mbufs(sc);
    684 
    685 	vioif_updown(sc, true);
    686 	ifp->if_flags |= IFF_RUNNING;
    687 	ifp->if_flags &= ~IFF_OACTIVE;
    688 	vioif_rx_filter(sc);
    689 
    690 	return 0;
    691 }
    692 
    693 static void
    694 vioif_stop(struct ifnet *ifp, int disable)
    695 {
    696 	struct vioif_softc *sc = ifp->if_softc;
    697 	struct virtio_softc *vsc = sc->sc_virtio;
    698 
    699 	sc->sc_stopping = true;
    700 
    701 	/* only way to stop I/O and DMA is resetting... */
    702 	virtio_reset(vsc);
    703 	vioif_rx_deq(sc);
    704 	vioif_tx_drain(sc);
    705 	ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
    706 
    707 	if (disable)
    708 		vioif_rx_drain(sc);
    709 
    710 	virtio_reinit_start(vsc);
    711 	virtio_negotiate_features(vsc, vsc->sc_features);
    712 	virtio_start_vq_intr(vsc, &sc->sc_vq[0]);
    713 	virtio_stop_vq_intr(vsc, &sc->sc_vq[1]);
    714 	if (vsc->sc_nvqs >= 3)
    715 		virtio_start_vq_intr(vsc, &sc->sc_vq[2]);
    716 	virtio_reinit_end(vsc);
    717 	vioif_updown(sc, false);
    718 }
    719 
    720 static void
    721 vioif_start(struct ifnet *ifp)
    722 {
    723 	struct vioif_softc *sc = ifp->if_softc;
    724 	struct virtio_softc *vsc = sc->sc_virtio;
    725 	struct virtqueue *vq = &sc->sc_vq[1]; /* tx vq */
    726 	struct mbuf *m;
    727 	int queued = 0, retry = 0;
    728 
    729 	VIOIF_TX_LOCK(sc);
    730 
    731 	if ((ifp->if_flags & (IFF_RUNNING|IFF_OACTIVE)) != IFF_RUNNING)
    732 		goto out;
    733 
    734 	if (sc->sc_stopping)
    735 		goto out;
    736 
    737 	for (;;) {
    738 		int slot, r;
    739 
    740 		IFQ_DEQUEUE(&ifp->if_snd, m);
    741 
    742 		if (m == NULL)
    743 			break;
    744 
    745 		r = virtio_enqueue_prep(vsc, vq, &slot);
    746 		if (r == EAGAIN) {
    747 			ifp->if_flags |= IFF_OACTIVE;
    748 			vioif_tx_vq_done_locked(vq);
    749 			if (retry++ == 0)
    750 				continue;
    751 			else
    752 				break;
    753 		}
    754 		if (r != 0)
    755 			panic("enqueue_prep for a tx buffer");
    756 		r = bus_dmamap_load_mbuf(vsc->sc_dmat,
    757 					 sc->sc_tx_dmamaps[slot],
    758 					 m, BUS_DMA_WRITE|BUS_DMA_NOWAIT);
    759 		if (r != 0) {
    760 			virtio_enqueue_abort(vsc, vq, slot);
    761 			printf("%s: tx dmamap load failed, error code %d\n",
    762 			       device_xname(sc->sc_dev), r);
    763 			break;
    764 		}
    765 		r = virtio_enqueue_reserve(vsc, vq, slot,
    766 					sc->sc_tx_dmamaps[slot]->dm_nsegs + 1);
    767 		if (r != 0) {
    768 			bus_dmamap_unload(vsc->sc_dmat,
    769 					  sc->sc_tx_dmamaps[slot]);
    770 			ifp->if_flags |= IFF_OACTIVE;
    771 			vioif_tx_vq_done_locked(vq);
    772 			if (retry++ == 0)
    773 				continue;
    774 			else
    775 				break;
    776 		}
    777 
    778 		sc->sc_tx_mbufs[slot] = m;
    779 
    780 		memset(&sc->sc_tx_hdrs[slot], 0, sizeof(struct virtio_net_hdr));
    781 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_tx_dmamaps[slot],
    782 				0, sc->sc_tx_dmamaps[slot]->dm_mapsize,
    783 				BUS_DMASYNC_PREWRITE);
    784 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_txhdr_dmamaps[slot],
    785 				0, sc->sc_txhdr_dmamaps[slot]->dm_mapsize,
    786 				BUS_DMASYNC_PREWRITE);
    787 		virtio_enqueue(vsc, vq, slot, sc->sc_txhdr_dmamaps[slot], true);
    788 		virtio_enqueue(vsc, vq, slot, sc->sc_tx_dmamaps[slot], true);
    789 		virtio_enqueue_commit(vsc, vq, slot, false);
    790 		queued++;
    791 		bpf_mtap(ifp, m);
    792 	}
    793 
    794 	if (m != NULL) {
    795 		ifp->if_flags |= IFF_OACTIVE;
    796 		m_freem(m);
    797 	}
    798 
    799 	if (queued > 0) {
    800 		virtio_enqueue_commit(vsc, vq, -1, true);
    801 		ifp->if_timer = 5;
    802 	}
    803 
    804 out:
    805 	VIOIF_TX_UNLOCK(sc);
    806 }
    807 
    808 static int
    809 vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
    810 {
    811 	int s, r;
    812 
    813 	s = splnet();
    814 
    815 	r = ether_ioctl(ifp, cmd, data);
    816 	if ((r == 0 && cmd == SIOCSIFFLAGS) ||
    817 	    (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) {
    818 		if (ifp->if_flags & IFF_RUNNING)
    819 			r = vioif_rx_filter(ifp->if_softc);
    820 		else
    821 			r = 0;
    822 	}
    823 
    824 	splx(s);
    825 
    826 	return r;
    827 }
    828 
    829 void
    830 vioif_watchdog(struct ifnet *ifp)
    831 {
    832 	struct vioif_softc *sc = ifp->if_softc;
    833 
    834 	if (ifp->if_flags & IFF_RUNNING)
    835 		vioif_tx_vq_done(&sc->sc_vq[1]);
    836 }
    837 
    838 
    839 /*
    840  * Recieve implementation
    841  */
    842 /* allocate and initialize a mbuf for recieve */
    843 static int
    844 vioif_add_rx_mbuf(struct vioif_softc *sc, int i)
    845 {
    846 	struct mbuf *m;
    847 	int r;
    848 
    849 	MGETHDR(m, M_DONTWAIT, MT_DATA);
    850 	if (m == NULL)
    851 		return ENOBUFS;
    852 	MCLGET(m, M_DONTWAIT);
    853 	if ((m->m_flags & M_EXT) == 0) {
    854 		m_freem(m);
    855 		return ENOBUFS;
    856 	}
    857 	sc->sc_rx_mbufs[i] = m;
    858 	m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
    859 	r = bus_dmamap_load_mbuf(sc->sc_virtio->sc_dmat,
    860 				 sc->sc_rx_dmamaps[i],
    861 				 m, BUS_DMA_READ|BUS_DMA_NOWAIT);
    862 	if (r) {
    863 		m_freem(m);
    864 		sc->sc_rx_mbufs[i] = 0;
    865 		return r;
    866 	}
    867 
    868 	return 0;
    869 }
    870 
    871 /* free a mbuf for recieve */
    872 static void
    873 vioif_free_rx_mbuf(struct vioif_softc *sc, int i)
    874 {
    875 	bus_dmamap_unload(sc->sc_virtio->sc_dmat, sc->sc_rx_dmamaps[i]);
    876 	m_freem(sc->sc_rx_mbufs[i]);
    877 	sc->sc_rx_mbufs[i] = NULL;
    878 }
    879 
    880 /* add mbufs for all the empty recieve slots */
    881 static void
    882 vioif_populate_rx_mbufs(struct vioif_softc *sc)
    883 {
    884 	struct virtio_softc *vsc = sc->sc_virtio;
    885 	int i, r, ndone = 0;
    886 	struct virtqueue *vq = &sc->sc_vq[0]; /* rx vq */
    887 
    888 	VIOIF_RX_LOCK(sc);
    889 
    890 	if (sc->sc_stopping)
    891 		goto out;
    892 
    893 	for (i = 0; i < vq->vq_num; i++) {
    894 		int slot;
    895 		r = virtio_enqueue_prep(vsc, vq, &slot);
    896 		if (r == EAGAIN)
    897 			break;
    898 		if (r != 0)
    899 			panic("enqueue_prep for rx buffers");
    900 		if (sc->sc_rx_mbufs[slot] == NULL) {
    901 			r = vioif_add_rx_mbuf(sc, slot);
    902 			if (r != 0) {
    903 				printf("%s: rx mbuf allocation failed, "
    904 				       "error code %d\n",
    905 				       device_xname(sc->sc_dev), r);
    906 				break;
    907 			}
    908 		}
    909 		r = virtio_enqueue_reserve(vsc, vq, slot,
    910 					sc->sc_rx_dmamaps[slot]->dm_nsegs + 1);
    911 		if (r != 0) {
    912 			vioif_free_rx_mbuf(sc, slot);
    913 			break;
    914 		}
    915 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_rxhdr_dmamaps[slot],
    916 			0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD);
    917 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_rx_dmamaps[slot],
    918 			0, MCLBYTES, BUS_DMASYNC_PREREAD);
    919 		virtio_enqueue(vsc, vq, slot, sc->sc_rxhdr_dmamaps[slot], false);
    920 		virtio_enqueue(vsc, vq, slot, sc->sc_rx_dmamaps[slot], false);
    921 		virtio_enqueue_commit(vsc, vq, slot, false);
    922 		ndone++;
    923 	}
    924 	if (ndone > 0)
    925 		virtio_enqueue_commit(vsc, vq, -1, true);
    926 
    927 out:
    928 	VIOIF_RX_UNLOCK(sc);
    929 }
    930 
    931 /* dequeue recieved packets */
    932 static int
    933 vioif_rx_deq(struct vioif_softc *sc)
    934 {
    935 	int r;
    936 
    937 	KASSERT(sc->sc_stopping);
    938 
    939 	VIOIF_RX_LOCK(sc);
    940 	r = vioif_rx_deq_locked(sc);
    941 	VIOIF_RX_UNLOCK(sc);
    942 
    943 	return r;
    944 }
    945 
    946 /* dequeue recieved packets */
    947 static int
    948 vioif_rx_deq_locked(struct vioif_softc *sc)
    949 {
    950 	struct virtio_softc *vsc = sc->sc_virtio;
    951 	struct virtqueue *vq = &sc->sc_vq[0];
    952 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    953 	struct mbuf *m;
    954 	int r = 0;
    955 	int slot, len;
    956 
    957 	KASSERT(VIOIF_RX_LOCKED(sc));
    958 
    959 	while (virtio_dequeue(vsc, vq, &slot, &len) == 0) {
    960 		len -= sizeof(struct virtio_net_hdr);
    961 		r = 1;
    962 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_rxhdr_dmamaps[slot],
    963 				0, sizeof(struct virtio_net_hdr),
    964 				BUS_DMASYNC_POSTREAD);
    965 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_rx_dmamaps[slot],
    966 				0, MCLBYTES,
    967 				BUS_DMASYNC_POSTREAD);
    968 		m = sc->sc_rx_mbufs[slot];
    969 		KASSERT(m != NULL);
    970 		bus_dmamap_unload(vsc->sc_dmat, sc->sc_rx_dmamaps[slot]);
    971 		sc->sc_rx_mbufs[slot] = 0;
    972 		virtio_dequeue_commit(vsc, vq, slot);
    973 		m->m_pkthdr.rcvif = ifp;
    974 		m->m_len = m->m_pkthdr.len = len;
    975 		ifp->if_ipackets++;
    976 		bpf_mtap(ifp, m);
    977 
    978 		VIOIF_RX_UNLOCK(sc);
    979 		(*ifp->if_input)(ifp, m);
    980 		VIOIF_RX_LOCK(sc);
    981 
    982 		if (sc->sc_stopping)
    983 			break;
    984 	}
    985 
    986 	return r;
    987 }
    988 
    989 /* rx interrupt; call _dequeue above and schedule a softint */
    990 static int
    991 vioif_rx_vq_done(struct virtqueue *vq)
    992 {
    993 	struct virtio_softc *vsc = vq->vq_owner;
    994 	struct vioif_softc *sc = device_private(vsc->sc_child);
    995 	int r = 0;
    996 
    997 	VIOIF_RX_LOCK(sc);
    998 
    999 	if (sc->sc_stopping)
   1000 		goto out;
   1001 
   1002 	r = vioif_rx_deq_locked(sc);
   1003 	if (r)
   1004 		softint_schedule(sc->sc_rx_softint);
   1005 
   1006 out:
   1007 	VIOIF_RX_UNLOCK(sc);
   1008 	return r;
   1009 }
   1010 
   1011 /* softint: enqueue recieve requests for new incoming packets */
   1012 static void
   1013 vioif_rx_softint(void *arg)
   1014 {
   1015 	struct vioif_softc *sc = arg;
   1016 
   1017 	vioif_populate_rx_mbufs(sc);
   1018 }
   1019 
   1020 /* free all the mbufs; called from if_stop(disable) */
   1021 static void
   1022 vioif_rx_drain(struct vioif_softc *sc)
   1023 {
   1024 	struct virtqueue *vq = &sc->sc_vq[0];
   1025 	int i;
   1026 
   1027 	for (i = 0; i < vq->vq_num; i++) {
   1028 		if (sc->sc_rx_mbufs[i] == NULL)
   1029 			continue;
   1030 		vioif_free_rx_mbuf(sc, i);
   1031 	}
   1032 }
   1033 
   1034 
   1035 /*
   1036  * Transmition implementation
   1037  */
   1038 /* actual transmission is done in if_start */
   1039 /* tx interrupt; dequeue and free mbufs */
   1040 /*
   1041  * tx interrupt is actually disabled; this should be called upon
   1042  * tx vq full and watchdog
   1043  */
   1044 static int
   1045 vioif_tx_vq_done(struct virtqueue *vq)
   1046 {
   1047 	struct virtio_softc *vsc = vq->vq_owner;
   1048 	struct vioif_softc *sc = device_private(vsc->sc_child);
   1049 	int r = 0;
   1050 
   1051 	VIOIF_TX_LOCK(sc);
   1052 
   1053 	if (sc->sc_stopping)
   1054 		goto out;
   1055 
   1056 	r = vioif_tx_vq_done_locked(vq);
   1057 
   1058 out:
   1059 	VIOIF_TX_UNLOCK(sc);
   1060 	return r;
   1061 }
   1062 
   1063 static int
   1064 vioif_tx_vq_done_locked(struct virtqueue *vq)
   1065 {
   1066 	struct virtio_softc *vsc = vq->vq_owner;
   1067 	struct vioif_softc *sc = device_private(vsc->sc_child);
   1068 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1069 	struct mbuf *m;
   1070 	int r = 0;
   1071 	int slot, len;
   1072 
   1073 	KASSERT(VIOIF_TX_LOCKED(sc));
   1074 
   1075 	while (virtio_dequeue(vsc, vq, &slot, &len) == 0) {
   1076 		r++;
   1077 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_txhdr_dmamaps[slot],
   1078 				0, sizeof(struct virtio_net_hdr),
   1079 				BUS_DMASYNC_POSTWRITE);
   1080 		bus_dmamap_sync(vsc->sc_dmat, sc->sc_tx_dmamaps[slot],
   1081 				0, sc->sc_tx_dmamaps[slot]->dm_mapsize,
   1082 				BUS_DMASYNC_POSTWRITE);
   1083 		m = sc->sc_tx_mbufs[slot];
   1084 		bus_dmamap_unload(vsc->sc_dmat, sc->sc_tx_dmamaps[slot]);
   1085 		sc->sc_tx_mbufs[slot] = 0;
   1086 		virtio_dequeue_commit(vsc, vq, slot);
   1087 		ifp->if_opackets++;
   1088 		m_freem(m);
   1089 	}
   1090 
   1091 	if (r)
   1092 		ifp->if_flags &= ~IFF_OACTIVE;
   1093 	return r;
   1094 }
   1095 
   1096 /* free all the mbufs already put on vq; called from if_stop(disable) */
   1097 static void
   1098 vioif_tx_drain(struct vioif_softc *sc)
   1099 {
   1100 	struct virtio_softc *vsc = sc->sc_virtio;
   1101 	struct virtqueue *vq = &sc->sc_vq[1];
   1102 	int i;
   1103 
   1104 	KASSERT(sc->sc_stopping);
   1105 
   1106 	for (i = 0; i < vq->vq_num; i++) {
   1107 		if (sc->sc_tx_mbufs[i] == NULL)
   1108 			continue;
   1109 		bus_dmamap_unload(vsc->sc_dmat, sc->sc_tx_dmamaps[i]);
   1110 		m_freem(sc->sc_tx_mbufs[i]);
   1111 		sc->sc_tx_mbufs[i] = NULL;
   1112 	}
   1113 }
   1114 
   1115 /*
   1116  * Control vq
   1117  */
   1118 /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
   1119 static int
   1120 vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
   1121 {
   1122 	struct virtio_softc *vsc = sc->sc_virtio;
   1123 	struct virtqueue *vq = &sc->sc_vq[2];
   1124 	int r, slot;
   1125 
   1126 	if (vsc->sc_nvqs < 3)
   1127 		return ENOTSUP;
   1128 
   1129 	mutex_enter(&sc->sc_ctrl_wait_lock);
   1130 	while (sc->sc_ctrl_inuse != FREE)
   1131 		cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock);
   1132 	sc->sc_ctrl_inuse = INUSE;
   1133 	mutex_exit(&sc->sc_ctrl_wait_lock);
   1134 
   1135 	sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_RX;
   1136 	sc->sc_ctrl_cmd->command = cmd;
   1137 	sc->sc_ctrl_rx->onoff = onoff;
   1138 
   1139 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_cmd_dmamap,
   1140 			0, sizeof(struct virtio_net_ctrl_cmd),
   1141 			BUS_DMASYNC_PREWRITE);
   1142 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_rx_dmamap,
   1143 			0, sizeof(struct virtio_net_ctrl_rx),
   1144 			BUS_DMASYNC_PREWRITE);
   1145 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_status_dmamap,
   1146 			0, sizeof(struct virtio_net_ctrl_status),
   1147 			BUS_DMASYNC_PREREAD);
   1148 
   1149 	r = virtio_enqueue_prep(vsc, vq, &slot);
   1150 	if (r != 0)
   1151 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   1152 	r = virtio_enqueue_reserve(vsc, vq, slot, 3);
   1153 	if (r != 0)
   1154 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   1155 	virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true);
   1156 	virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_rx_dmamap, true);
   1157 	virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false);
   1158 	virtio_enqueue_commit(vsc, vq, slot, true);
   1159 
   1160 	/* wait for done */
   1161 	mutex_enter(&sc->sc_ctrl_wait_lock);
   1162 	while (sc->sc_ctrl_inuse != DONE)
   1163 		cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock);
   1164 	mutex_exit(&sc->sc_ctrl_wait_lock);
   1165 	/* already dequeueued */
   1166 
   1167 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_cmd_dmamap, 0,
   1168 			sizeof(struct virtio_net_ctrl_cmd),
   1169 			BUS_DMASYNC_POSTWRITE);
   1170 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_rx_dmamap, 0,
   1171 			sizeof(struct virtio_net_ctrl_rx),
   1172 			BUS_DMASYNC_POSTWRITE);
   1173 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_status_dmamap, 0,
   1174 			sizeof(struct virtio_net_ctrl_status),
   1175 			BUS_DMASYNC_POSTREAD);
   1176 
   1177 	if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK)
   1178 		r = 0;
   1179 	else {
   1180 		printf("%s: failed setting rx mode\n",
   1181 		       device_xname(sc->sc_dev));
   1182 		r = EIO;
   1183 	}
   1184 
   1185 	mutex_enter(&sc->sc_ctrl_wait_lock);
   1186 	sc->sc_ctrl_inuse = FREE;
   1187 	cv_signal(&sc->sc_ctrl_wait);
   1188 	mutex_exit(&sc->sc_ctrl_wait_lock);
   1189 
   1190 	return r;
   1191 }
   1192 
   1193 static int
   1194 vioif_set_promisc(struct vioif_softc *sc, bool onoff)
   1195 {
   1196 	int r;
   1197 
   1198 	r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
   1199 
   1200 	return r;
   1201 }
   1202 
   1203 static int
   1204 vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
   1205 {
   1206 	int r;
   1207 
   1208 	r = vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
   1209 
   1210 	return r;
   1211 }
   1212 
   1213 /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
   1214 static int
   1215 vioif_set_rx_filter(struct vioif_softc *sc)
   1216 {
   1217 	/* filter already set in sc_ctrl_mac_tbl */
   1218 	struct virtio_softc *vsc = sc->sc_virtio;
   1219 	struct virtqueue *vq = &sc->sc_vq[2];
   1220 	int r, slot;
   1221 
   1222 	if (vsc->sc_nvqs < 3)
   1223 		return ENOTSUP;
   1224 
   1225 	mutex_enter(&sc->sc_ctrl_wait_lock);
   1226 	while (sc->sc_ctrl_inuse != FREE)
   1227 		cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock);
   1228 	sc->sc_ctrl_inuse = INUSE;
   1229 	mutex_exit(&sc->sc_ctrl_wait_lock);
   1230 
   1231 	sc->sc_ctrl_cmd->class = VIRTIO_NET_CTRL_MAC;
   1232 	sc->sc_ctrl_cmd->command = VIRTIO_NET_CTRL_MAC_TABLE_SET;
   1233 
   1234 	r = bus_dmamap_load(vsc->sc_dmat, sc->sc_ctrl_tbl_uc_dmamap,
   1235 			    sc->sc_ctrl_mac_tbl_uc,
   1236 			    (sizeof(struct virtio_net_ctrl_mac_tbl)
   1237 			  + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries),
   1238 			    NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT);
   1239 	if (r) {
   1240 		printf("%s: control command dmamap load failed, "
   1241 		       "error code %d\n", device_xname(sc->sc_dev), r);
   1242 		goto out;
   1243 	}
   1244 	r = bus_dmamap_load(vsc->sc_dmat, sc->sc_ctrl_tbl_mc_dmamap,
   1245 			    sc->sc_ctrl_mac_tbl_mc,
   1246 			    (sizeof(struct virtio_net_ctrl_mac_tbl)
   1247 			  + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries),
   1248 			    NULL, BUS_DMA_WRITE|BUS_DMA_NOWAIT);
   1249 	if (r) {
   1250 		printf("%s: control command dmamap load failed, "
   1251 		       "error code %d\n", device_xname(sc->sc_dev), r);
   1252 		bus_dmamap_unload(vsc->sc_dmat, sc->sc_ctrl_tbl_uc_dmamap);
   1253 		goto out;
   1254 	}
   1255 
   1256 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_cmd_dmamap,
   1257 			0, sizeof(struct virtio_net_ctrl_cmd),
   1258 			BUS_DMASYNC_PREWRITE);
   1259 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_tbl_uc_dmamap, 0,
   1260 			(sizeof(struct virtio_net_ctrl_mac_tbl)
   1261 			 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries),
   1262 			BUS_DMASYNC_PREWRITE);
   1263 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_tbl_mc_dmamap, 0,
   1264 			(sizeof(struct virtio_net_ctrl_mac_tbl)
   1265 			 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries),
   1266 			BUS_DMASYNC_PREWRITE);
   1267 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_status_dmamap,
   1268 			0, sizeof(struct virtio_net_ctrl_status),
   1269 			BUS_DMASYNC_PREREAD);
   1270 
   1271 	r = virtio_enqueue_prep(vsc, vq, &slot);
   1272 	if (r != 0)
   1273 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   1274 	r = virtio_enqueue_reserve(vsc, vq, slot, 4);
   1275 	if (r != 0)
   1276 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   1277 	virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_cmd_dmamap, true);
   1278 	virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_uc_dmamap, true);
   1279 	virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_tbl_mc_dmamap, true);
   1280 	virtio_enqueue(vsc, vq, slot, sc->sc_ctrl_status_dmamap, false);
   1281 	virtio_enqueue_commit(vsc, vq, slot, true);
   1282 
   1283 	/* wait for done */
   1284 	mutex_enter(&sc->sc_ctrl_wait_lock);
   1285 	while (sc->sc_ctrl_inuse != DONE)
   1286 		cv_wait(&sc->sc_ctrl_wait, &sc->sc_ctrl_wait_lock);
   1287 	mutex_exit(&sc->sc_ctrl_wait_lock);
   1288 	/* already dequeueued */
   1289 
   1290 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_cmd_dmamap, 0,
   1291 			sizeof(struct virtio_net_ctrl_cmd),
   1292 			BUS_DMASYNC_POSTWRITE);
   1293 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_tbl_uc_dmamap, 0,
   1294 			(sizeof(struct virtio_net_ctrl_mac_tbl)
   1295 			 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_uc->nentries),
   1296 			BUS_DMASYNC_POSTWRITE);
   1297 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_tbl_mc_dmamap, 0,
   1298 			(sizeof(struct virtio_net_ctrl_mac_tbl)
   1299 			 + ETHER_ADDR_LEN * sc->sc_ctrl_mac_tbl_mc->nentries),
   1300 			BUS_DMASYNC_POSTWRITE);
   1301 	bus_dmamap_sync(vsc->sc_dmat, sc->sc_ctrl_status_dmamap, 0,
   1302 			sizeof(struct virtio_net_ctrl_status),
   1303 			BUS_DMASYNC_POSTREAD);
   1304 	bus_dmamap_unload(vsc->sc_dmat, sc->sc_ctrl_tbl_uc_dmamap);
   1305 	bus_dmamap_unload(vsc->sc_dmat, sc->sc_ctrl_tbl_mc_dmamap);
   1306 
   1307 	if (sc->sc_ctrl_status->ack == VIRTIO_NET_OK)
   1308 		r = 0;
   1309 	else {
   1310 		printf("%s: failed setting rx filter\n",
   1311 		       device_xname(sc->sc_dev));
   1312 		r = EIO;
   1313 	}
   1314 
   1315 out:
   1316 	mutex_enter(&sc->sc_ctrl_wait_lock);
   1317 	sc->sc_ctrl_inuse = FREE;
   1318 	cv_signal(&sc->sc_ctrl_wait);
   1319 	mutex_exit(&sc->sc_ctrl_wait_lock);
   1320 
   1321 	return r;
   1322 }
   1323 
   1324 /* ctrl vq interrupt; wake up the command issuer */
   1325 static int
   1326 vioif_ctrl_vq_done(struct virtqueue *vq)
   1327 {
   1328 	struct virtio_softc *vsc = vq->vq_owner;
   1329 	struct vioif_softc *sc = device_private(vsc->sc_child);
   1330 	int r, slot;
   1331 
   1332 	r = virtio_dequeue(vsc, vq, &slot, NULL);
   1333 	if (r == ENOENT)
   1334 		return 0;
   1335 	virtio_dequeue_commit(vsc, vq, slot);
   1336 
   1337 	mutex_enter(&sc->sc_ctrl_wait_lock);
   1338 	sc->sc_ctrl_inuse = DONE;
   1339 	cv_signal(&sc->sc_ctrl_wait);
   1340 	mutex_exit(&sc->sc_ctrl_wait_lock);
   1341 
   1342 	return 1;
   1343 }
   1344 
   1345 /*
   1346  * If IFF_PROMISC requested,  set promiscuous
   1347  * If multicast filter small enough (<=MAXENTRIES) set rx filter
   1348  * If large multicast filter exist use ALLMULTI
   1349  */
   1350 /*
   1351  * If setting rx filter fails fall back to ALLMULTI
   1352  * If ALLMULTI fails fall back to PROMISC
   1353  */
   1354 static int
   1355 vioif_rx_filter(struct vioif_softc *sc)
   1356 {
   1357 	struct virtio_softc *vsc = sc->sc_virtio;
   1358 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1359 	struct ether_multi *enm;
   1360 	struct ether_multistep step;
   1361 	int nentries;
   1362 	int promisc = 0, allmulti = 0, rxfilter = 0;
   1363 	int r;
   1364 
   1365 	if (vsc->sc_nvqs < 3) {	/* no ctrl vq; always promisc */
   1366 		ifp->if_flags |= IFF_PROMISC;
   1367 		return 0;
   1368 	}
   1369 
   1370 	if (ifp->if_flags & IFF_PROMISC) {
   1371 		promisc = 1;
   1372 		goto set;
   1373 	}
   1374 
   1375 	nentries = -1;
   1376 	ETHER_FIRST_MULTI(step, &sc->sc_ethercom, enm);
   1377 	while (nentries++, enm != NULL) {
   1378 		if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
   1379 			allmulti = 1;
   1380 			goto set;
   1381 		}
   1382 		if (memcmp(enm->enm_addrlo, enm->enm_addrhi,
   1383 			   ETHER_ADDR_LEN)) {
   1384 			allmulti = 1;
   1385 			goto set;
   1386 		}
   1387 		memcpy(sc->sc_ctrl_mac_tbl_mc->macs[nentries],
   1388 		       enm->enm_addrlo, ETHER_ADDR_LEN);
   1389 		ETHER_NEXT_MULTI(step, enm);
   1390 	}
   1391 	rxfilter = 1;
   1392 
   1393 set:
   1394 	if (rxfilter) {
   1395 		sc->sc_ctrl_mac_tbl_uc->nentries = 0;
   1396 		sc->sc_ctrl_mac_tbl_mc->nentries = nentries;
   1397 		r = vioif_set_rx_filter(sc);
   1398 		if (r != 0) {
   1399 			rxfilter = 0;
   1400 			allmulti = 1; /* fallback */
   1401 		}
   1402 	} else {
   1403 		/* remove rx filter */
   1404 		sc->sc_ctrl_mac_tbl_uc->nentries = 0;
   1405 		sc->sc_ctrl_mac_tbl_mc->nentries = 0;
   1406 		r = vioif_set_rx_filter(sc);
   1407 		/* what to do on failure? */
   1408 	}
   1409 	if (allmulti) {
   1410 		r = vioif_set_allmulti(sc, true);
   1411 		if (r != 0) {
   1412 			allmulti = 0;
   1413 			promisc = 1; /* fallback */
   1414 		}
   1415 	} else {
   1416 		r = vioif_set_allmulti(sc, false);
   1417 		/* what to do on failure? */
   1418 	}
   1419 	if (promisc) {
   1420 		r = vioif_set_promisc(sc, true);
   1421 	} else {
   1422 		r = vioif_set_promisc(sc, false);
   1423 	}
   1424 
   1425 	return r;
   1426 }
   1427 
   1428 /* change link status */
   1429 static int
   1430 vioif_updown(struct vioif_softc *sc, bool isup)
   1431 {
   1432 	struct virtio_softc *vsc = sc->sc_virtio;
   1433 
   1434 	if (!(vsc->sc_features & VIRTIO_NET_F_STATUS))
   1435 		return ENODEV;
   1436 	virtio_write_device_config_1(vsc,
   1437 				     VIRTIO_NET_CONFIG_STATUS,
   1438 				     isup?VIRTIO_NET_S_LINK_UP:0);
   1439 	return 0;
   1440 }
   1441