Home | History | Annotate | Line # | Download | only in pci
if_vioif.c revision 1.60
      1  1.60  yamaguch /*	$NetBSD: if_vioif.c,v 1.60 2020/05/25 09:25:31 yamaguchi Exp $	*/
      2   1.1   hannken 
      3   1.1   hannken /*
      4   1.1   hannken  * Copyright (c) 2010 Minoura Makoto.
      5   1.1   hannken  * All rights reserved.
      6   1.1   hannken  *
      7   1.1   hannken  * Redistribution and use in source and binary forms, with or without
      8   1.1   hannken  * modification, are permitted provided that the following conditions
      9   1.1   hannken  * are met:
     10   1.1   hannken  * 1. Redistributions of source code must retain the above copyright
     11   1.1   hannken  *    notice, this list of conditions and the following disclaimer.
     12   1.1   hannken  * 2. Redistributions in binary form must reproduce the above copyright
     13   1.1   hannken  *    notice, this list of conditions and the following disclaimer in the
     14   1.1   hannken  *    documentation and/or other materials provided with the distribution.
     15   1.1   hannken  *
     16   1.1   hannken  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     17   1.1   hannken  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     18   1.1   hannken  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     19   1.1   hannken  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     20   1.1   hannken  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     21   1.1   hannken  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     22   1.1   hannken  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     23   1.1   hannken  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     24   1.1   hannken  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     25   1.1   hannken  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     26   1.1   hannken  */
     27   1.1   hannken 
     28   1.1   hannken #include <sys/cdefs.h>
     29  1.60  yamaguch __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.60 2020/05/25 09:25:31 yamaguchi Exp $");
     30  1.15     ozaki 
     31  1.15     ozaki #ifdef _KERNEL_OPT
     32  1.15     ozaki #include "opt_net_mpsafe.h"
     33  1.15     ozaki #endif
     34   1.1   hannken 
     35   1.1   hannken #include <sys/param.h>
     36   1.1   hannken #include <sys/systm.h>
     37   1.1   hannken #include <sys/kernel.h>
     38  1.55  yamaguch #include <sys/atomic.h>
     39   1.1   hannken #include <sys/bus.h>
     40   1.1   hannken #include <sys/condvar.h>
     41   1.1   hannken #include <sys/device.h>
     42   1.1   hannken #include <sys/intr.h>
     43   1.1   hannken #include <sys/kmem.h>
     44   1.1   hannken #include <sys/mbuf.h>
     45   1.1   hannken #include <sys/mutex.h>
     46   1.1   hannken #include <sys/sockio.h>
     47  1.12     ozaki #include <sys/cpu.h>
     48  1.26  pgoyette #include <sys/module.h>
     49  1.46  yamaguch #include <sys/pcq.h>
     50  1.55  yamaguch #include <sys/workqueue.h>
     51   1.1   hannken 
     52   1.1   hannken #include <dev/pci/virtioreg.h>
     53   1.1   hannken #include <dev/pci/virtiovar.h>
     54   1.1   hannken 
     55   1.1   hannken #include <net/if.h>
     56   1.1   hannken #include <net/if_media.h>
     57   1.1   hannken #include <net/if_ether.h>
     58   1.1   hannken 
     59   1.1   hannken #include <net/bpf.h>
     60   1.1   hannken 
     61  1.26  pgoyette #include "ioconf.h"
     62   1.1   hannken 
     63   1.7     ozaki #ifdef NET_MPSAFE
     64   1.7     ozaki #define VIOIF_MPSAFE	1
     65  1.46  yamaguch #define VIOIF_MULTIQ	1
     66   1.7     ozaki #endif
     67   1.7     ozaki 
     68   1.1   hannken /*
     69   1.1   hannken  * if_vioifreg.h:
     70   1.1   hannken  */
     71   1.1   hannken /* Configuration registers */
     72   1.1   hannken #define VIRTIO_NET_CONFIG_MAC		0 /* 8bit x 6byte */
     73   1.1   hannken #define VIRTIO_NET_CONFIG_STATUS	6 /* 16bit */
     74  1.46  yamaguch #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS	8 /* 16bit */
     75   1.1   hannken 
     76   1.1   hannken /* Feature bits */
     77  1.46  yamaguch #define VIRTIO_NET_F_CSUM		__BIT(0)
     78  1.46  yamaguch #define VIRTIO_NET_F_GUEST_CSUM		__BIT(1)
     79  1.46  yamaguch #define VIRTIO_NET_F_MAC		__BIT(5)
     80  1.46  yamaguch #define VIRTIO_NET_F_GSO		__BIT(6)
     81  1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO4		__BIT(7)
     82  1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO6		__BIT(8)
     83  1.46  yamaguch #define VIRTIO_NET_F_GUEST_ECN		__BIT(9)
     84  1.46  yamaguch #define VIRTIO_NET_F_GUEST_UFO		__BIT(10)
     85  1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO4		__BIT(11)
     86  1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO6		__BIT(12)
     87  1.46  yamaguch #define VIRTIO_NET_F_HOST_ECN		__BIT(13)
     88  1.46  yamaguch #define VIRTIO_NET_F_HOST_UFO		__BIT(14)
     89  1.46  yamaguch #define VIRTIO_NET_F_MRG_RXBUF		__BIT(15)
     90  1.46  yamaguch #define VIRTIO_NET_F_STATUS		__BIT(16)
     91  1.46  yamaguch #define VIRTIO_NET_F_CTRL_VQ		__BIT(17)
     92  1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX		__BIT(18)
     93  1.46  yamaguch #define VIRTIO_NET_F_CTRL_VLAN		__BIT(19)
     94  1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA	__BIT(20)
     95  1.46  yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE	__BIT(21)
     96  1.46  yamaguch #define VIRTIO_NET_F_MQ			__BIT(22)
     97   1.1   hannken 
     98  1.18  christos #define VIRTIO_NET_FLAG_BITS \
     99  1.18  christos 	VIRTIO_COMMON_FLAG_BITS \
    100  1.46  yamaguch 	"\x17""MQ" \
    101  1.46  yamaguch 	"\x16""GUEST_ANNOUNCE" \
    102  1.46  yamaguch 	"\x15""CTRL_RX_EXTRA" \
    103  1.18  christos 	"\x14""CTRL_VLAN" \
    104  1.18  christos 	"\x13""CTRL_RX" \
    105  1.18  christos 	"\x12""CTRL_VQ" \
    106  1.18  christos 	"\x11""STATUS" \
    107  1.18  christos 	"\x10""MRG_RXBUF" \
    108  1.18  christos 	"\x0f""HOST_UFO" \
    109  1.18  christos 	"\x0e""HOST_ECN" \
    110  1.18  christos 	"\x0d""HOST_TSO6" \
    111  1.18  christos 	"\x0c""HOST_TSO4" \
    112  1.18  christos 	"\x0b""GUEST_UFO" \
    113  1.18  christos 	"\x0a""GUEST_ECN" \
    114  1.18  christos 	"\x09""GUEST_TSO6" \
    115  1.18  christos 	"\x08""GUEST_TSO4" \
    116  1.18  christos 	"\x07""GSO" \
    117  1.18  christos 	"\x06""MAC" \
    118  1.18  christos 	"\x02""GUEST_CSUM" \
    119  1.18  christos 	"\x01""CSUM"
    120  1.18  christos 
    121   1.1   hannken /* Status */
    122   1.1   hannken #define VIRTIO_NET_S_LINK_UP	1
    123   1.1   hannken 
    124   1.1   hannken /* Packet header structure */
    125   1.1   hannken struct virtio_net_hdr {
    126   1.1   hannken 	uint8_t		flags;
    127   1.1   hannken 	uint8_t		gso_type;
    128   1.1   hannken 	uint16_t	hdr_len;
    129   1.1   hannken 	uint16_t	gso_size;
    130   1.1   hannken 	uint16_t	csum_start;
    131   1.1   hannken 	uint16_t	csum_offset;
    132   1.1   hannken #if 0
    133   1.1   hannken 	uint16_t	num_buffers; /* if VIRTIO_NET_F_MRG_RXBUF enabled */
    134   1.1   hannken #endif
    135   1.1   hannken } __packed;
    136   1.1   hannken 
    137   1.1   hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM	1 /* flags */
    138   1.1   hannken #define VIRTIO_NET_HDR_GSO_NONE		0 /* gso_type */
    139   1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV4	1 /* gso_type */
    140   1.1   hannken #define VIRTIO_NET_HDR_GSO_UDP		3 /* gso_type */
    141   1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV6	4 /* gso_type */
    142   1.1   hannken #define VIRTIO_NET_HDR_GSO_ECN		0x80 /* gso_type, |'ed */
    143   1.1   hannken 
    144   1.1   hannken #define VIRTIO_NET_MAX_GSO_LEN		(65536+ETHER_HDR_LEN)
    145   1.1   hannken 
    146   1.1   hannken /* Control virtqueue */
    147   1.1   hannken struct virtio_net_ctrl_cmd {
    148   1.1   hannken 	uint8_t	class;
    149   1.1   hannken 	uint8_t	command;
    150   1.1   hannken } __packed;
    151   1.1   hannken #define VIRTIO_NET_CTRL_RX		0
    152   1.1   hannken # define VIRTIO_NET_CTRL_RX_PROMISC	0
    153   1.1   hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI	1
    154   1.1   hannken 
    155   1.1   hannken #define VIRTIO_NET_CTRL_MAC		1
    156   1.1   hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET	0
    157   1.1   hannken 
    158   1.1   hannken #define VIRTIO_NET_CTRL_VLAN		2
    159   1.1   hannken # define VIRTIO_NET_CTRL_VLAN_ADD	0
    160   1.1   hannken # define VIRTIO_NET_CTRL_VLAN_DEL	1
    161   1.1   hannken 
    162  1.46  yamaguch #define VIRTIO_NET_CTRL_MQ			4
    163  1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET	0
    164  1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN	1
    165  1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX	0x8000
    166  1.46  yamaguch 
    167   1.1   hannken struct virtio_net_ctrl_status {
    168   1.1   hannken 	uint8_t	ack;
    169   1.1   hannken } __packed;
    170   1.1   hannken #define VIRTIO_NET_OK			0
    171   1.1   hannken #define VIRTIO_NET_ERR			1
    172   1.1   hannken 
    173   1.1   hannken struct virtio_net_ctrl_rx {
    174   1.1   hannken 	uint8_t	onoff;
    175   1.1   hannken } __packed;
    176   1.1   hannken 
    177   1.1   hannken struct virtio_net_ctrl_mac_tbl {
    178   1.1   hannken 	uint32_t nentries;
    179   1.1   hannken 	uint8_t macs[][ETHER_ADDR_LEN];
    180   1.1   hannken } __packed;
    181   1.1   hannken 
    182   1.1   hannken struct virtio_net_ctrl_vlan {
    183   1.1   hannken 	uint16_t id;
    184   1.1   hannken } __packed;
    185   1.1   hannken 
    186  1.46  yamaguch struct virtio_net_ctrl_mq {
    187  1.46  yamaguch 	uint16_t virtqueue_pairs;
    188  1.46  yamaguch } __packed;
    189  1.46  yamaguch 
    190  1.44  yamaguch struct vioif_ctrl_cmdspec {
    191  1.44  yamaguch 	bus_dmamap_t	dmamap;
    192  1.44  yamaguch 	void		*buf;
    193  1.44  yamaguch 	bus_size_t	bufsize;
    194  1.44  yamaguch };
    195   1.1   hannken 
    196   1.1   hannken /*
    197   1.1   hannken  * if_vioifvar.h:
    198   1.1   hannken  */
    199  1.43  yamaguch 
    200  1.43  yamaguch /*
    201  1.43  yamaguch  * Locking notes:
    202  1.43  yamaguch  * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and
    203  1.57  yamaguch  *   a field in vioif_rxqueue is protected by rxq_lock (a spin mutex).
    204  1.43  yamaguch  *      - more than one lock cannot be held at onece
    205  1.43  yamaguch  * + ctrlq_inuse is protected by ctrlq_wait_lock.
    206  1.43  yamaguch  *      - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
    207  1.43  yamaguch  *      - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock
    208  1.43  yamaguch  */
    209  1.43  yamaguch 
    210  1.55  yamaguch struct vioif_work {
    211  1.55  yamaguch 	struct work	 cookie;
    212  1.55  yamaguch 	void		(*func)(void *);
    213  1.55  yamaguch 	void		*arg;
    214  1.55  yamaguch 	unsigned int	 added;
    215  1.55  yamaguch };
    216  1.55  yamaguch 
    217  1.43  yamaguch struct vioif_txqueue {
    218  1.43  yamaguch 	kmutex_t		*txq_lock;	/* lock for tx operations */
    219  1.43  yamaguch 
    220  1.43  yamaguch 	struct virtqueue	*txq_vq;
    221  1.43  yamaguch 	bool			txq_stopping;
    222  1.43  yamaguch 	bool			txq_link_active;
    223  1.46  yamaguch 	pcq_t			*txq_intrq;
    224  1.43  yamaguch 
    225  1.43  yamaguch 	struct virtio_net_hdr	*txq_hdrs;
    226  1.43  yamaguch 	bus_dmamap_t		*txq_hdr_dmamaps;
    227  1.43  yamaguch 
    228  1.43  yamaguch 	struct mbuf		**txq_mbufs;
    229  1.43  yamaguch 	bus_dmamap_t		*txq_dmamaps;
    230  1.46  yamaguch 
    231  1.46  yamaguch 	void			*txq_deferred_transmit;
    232  1.55  yamaguch 	void			*txq_handle_si;
    233  1.55  yamaguch 	struct vioif_work	 txq_work;
    234  1.55  yamaguch 	bool			 txq_workqueue;
    235  1.55  yamaguch 	bool			 txq_active;
    236  1.43  yamaguch };
    237  1.43  yamaguch 
    238  1.43  yamaguch struct vioif_rxqueue {
    239  1.43  yamaguch 	kmutex_t		*rxq_lock;	/* lock for rx operations */
    240  1.43  yamaguch 
    241  1.43  yamaguch 	struct virtqueue	*rxq_vq;
    242  1.43  yamaguch 	bool			rxq_stopping;
    243  1.43  yamaguch 
    244  1.43  yamaguch 	struct virtio_net_hdr	*rxq_hdrs;
    245  1.43  yamaguch 	bus_dmamap_t		*rxq_hdr_dmamaps;
    246  1.43  yamaguch 
    247  1.43  yamaguch 	struct mbuf		**rxq_mbufs;
    248  1.43  yamaguch 	bus_dmamap_t		*rxq_dmamaps;
    249  1.43  yamaguch 
    250  1.43  yamaguch 	void			*rxq_softint;
    251  1.55  yamaguch 	void			*rxq_handle_si;
    252  1.55  yamaguch 	struct vioif_work	 rxq_work;
    253  1.55  yamaguch 	bool			 rxq_workqueue;
    254  1.55  yamaguch 	bool			 rxq_active;
    255  1.43  yamaguch };
    256  1.43  yamaguch 
    257  1.43  yamaguch struct vioif_ctrlqueue {
    258  1.43  yamaguch 	struct virtqueue		*ctrlq_vq;
    259  1.43  yamaguch 	enum {
    260  1.43  yamaguch 		FREE, INUSE, DONE
    261  1.43  yamaguch 	}				ctrlq_inuse;
    262  1.43  yamaguch 	kcondvar_t			ctrlq_wait;
    263  1.43  yamaguch 	kmutex_t			ctrlq_wait_lock;
    264  1.44  yamaguch 	struct lwp			*ctrlq_owner;
    265  1.43  yamaguch 
    266  1.43  yamaguch 	struct virtio_net_ctrl_cmd	*ctrlq_cmd;
    267  1.43  yamaguch 	struct virtio_net_ctrl_status	*ctrlq_status;
    268  1.43  yamaguch 	struct virtio_net_ctrl_rx	*ctrlq_rx;
    269  1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_uc;
    270  1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_mc;
    271  1.46  yamaguch 	struct virtio_net_ctrl_mq	*ctrlq_mq;
    272  1.43  yamaguch 
    273  1.43  yamaguch 	bus_dmamap_t			ctrlq_cmd_dmamap;
    274  1.43  yamaguch 	bus_dmamap_t			ctrlq_status_dmamap;
    275  1.43  yamaguch 	bus_dmamap_t			ctrlq_rx_dmamap;
    276  1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_uc_dmamap;
    277  1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_mc_dmamap;
    278  1.46  yamaguch 	bus_dmamap_t			ctrlq_mq_dmamap;
    279  1.43  yamaguch };
    280  1.43  yamaguch 
    281   1.1   hannken struct vioif_softc {
    282   1.1   hannken 	device_t		sc_dev;
    283  1.55  yamaguch 	struct sysctllog	*sc_sysctllog;
    284   1.1   hannken 
    285   1.1   hannken 	struct virtio_softc	*sc_virtio;
    286  1.46  yamaguch 	struct virtqueue	*sc_vqs;
    287  1.46  yamaguch 
    288  1.46  yamaguch 	int			sc_max_nvq_pairs;
    289  1.46  yamaguch 	int			sc_req_nvq_pairs;
    290  1.46  yamaguch 	int			sc_act_nvq_pairs;
    291   1.1   hannken 
    292   1.1   hannken 	uint8_t			sc_mac[ETHER_ADDR_LEN];
    293   1.1   hannken 	struct ethercom		sc_ethercom;
    294   1.8     pooka 	short			sc_deferred_init_done;
    295  1.34     ozaki 	bool			sc_link_active;
    296   1.1   hannken 
    297  1.46  yamaguch 	struct vioif_txqueue	*sc_txq;
    298  1.46  yamaguch 	struct vioif_rxqueue	*sc_rxq;
    299  1.43  yamaguch 
    300  1.43  yamaguch 	bool			sc_has_ctrl;
    301  1.43  yamaguch 	struct vioif_ctrlqueue	sc_ctrlq;
    302  1.43  yamaguch 
    303   1.1   hannken 	bus_dma_segment_t	sc_hdr_segs[1];
    304  1.43  yamaguch 	void			*sc_dmamem;
    305  1.43  yamaguch 	void			*sc_kmem;
    306   1.1   hannken 
    307  1.34     ozaki 	void			*sc_ctl_softint;
    308  1.55  yamaguch 
    309  1.55  yamaguch 	struct workqueue	*sc_txrx_workqueue;
    310  1.55  yamaguch 	bool			 sc_txrx_workqueue_sysctl;
    311  1.55  yamaguch 	u_int			 sc_tx_intr_process_limit;
    312  1.55  yamaguch 	u_int			 sc_tx_process_limit;
    313  1.55  yamaguch 	u_int			 sc_rx_intr_process_limit;
    314  1.55  yamaguch 	u_int			 sc_rx_process_limit;
    315   1.1   hannken };
    316   1.1   hannken #define VIRTIO_NET_TX_MAXNSEGS		(16) /* XXX */
    317   1.1   hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES	(64) /* XXX */
    318   1.1   hannken 
    319  1.55  yamaguch #define VIOIF_TX_INTR_PROCESS_LIMIT	256
    320  1.55  yamaguch #define VIOIF_TX_PROCESS_LIMIT		256
    321  1.55  yamaguch #define VIOIF_RX_INTR_PROCESS_LIMIT	0U
    322  1.55  yamaguch #define VIOIF_RX_PROCESS_LIMIT		256
    323  1.55  yamaguch 
    324  1.55  yamaguch #define VIOIF_WORKQUEUE_PRI		PRI_SOFTNET
    325  1.55  yamaguch 
    326   1.1   hannken /* cfattach interface functions */
    327   1.1   hannken static int	vioif_match(device_t, cfdata_t, void *);
    328   1.1   hannken static void	vioif_attach(device_t, device_t, void *);
    329   1.1   hannken static void	vioif_deferred_init(device_t);
    330  1.55  yamaguch static int	vioif_finalize_teardown(device_t);
    331   1.1   hannken 
    332   1.1   hannken /* ifnet interface functions */
    333   1.1   hannken static int	vioif_init(struct ifnet *);
    334   1.1   hannken static void	vioif_stop(struct ifnet *, int);
    335   1.1   hannken static void	vioif_start(struct ifnet *);
    336  1.46  yamaguch static void	vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
    337  1.46  yamaguch static int	vioif_transmit(struct ifnet *, struct mbuf *);
    338  1.46  yamaguch static void	vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
    339   1.1   hannken static int	vioif_ioctl(struct ifnet *, u_long, void *);
    340   1.1   hannken static void	vioif_watchdog(struct ifnet *);
    341   1.1   hannken 
    342   1.1   hannken /* rx */
    343  1.46  yamaguch static int	vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
    344  1.46  yamaguch static void	vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
    345  1.46  yamaguch static void	vioif_populate_rx_mbufs(struct vioif_rxqueue *);
    346  1.46  yamaguch static void	vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *);
    347  1.55  yamaguch static void	vioif_rx_queue_clear(struct vioif_rxqueue *);
    348  1.55  yamaguch static bool	vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *,
    349  1.55  yamaguch 		    struct vioif_rxqueue *, u_int);
    350  1.54  yamaguch static int	vioif_rx_intr(void *);
    351  1.55  yamaguch static void	vioif_rx_handle(void *);
    352  1.55  yamaguch static void	vioif_rx_sched_handle(struct vioif_softc *,
    353  1.55  yamaguch 		    struct vioif_rxqueue *);
    354   1.1   hannken static void	vioif_rx_softint(void *);
    355  1.46  yamaguch static void	vioif_rx_drain(struct vioif_rxqueue *);
    356   1.1   hannken 
    357   1.1   hannken /* tx */
    358  1.54  yamaguch static int	vioif_tx_intr(void *);
    359  1.55  yamaguch static void	vioif_tx_handle(void *);
    360  1.55  yamaguch static void	vioif_tx_sched_handle(struct vioif_softc *,
    361  1.55  yamaguch 		    struct vioif_txqueue *);
    362  1.55  yamaguch static void	vioif_tx_queue_clear(struct vioif_txqueue *);
    363  1.55  yamaguch static bool	vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *,
    364  1.55  yamaguch 		    struct vioif_txqueue *, u_int);
    365  1.46  yamaguch static void	vioif_tx_drain(struct vioif_txqueue *);
    366  1.46  yamaguch static void	vioif_deferred_transmit(void *);
    367   1.1   hannken 
    368  1.55  yamaguch /* workqueue */
    369  1.55  yamaguch static struct workqueue*
    370  1.55  yamaguch 		vioif_workq_create(const char *, pri_t, int, int);
    371  1.55  yamaguch static void	vioif_workq_destroy(struct workqueue *);
    372  1.55  yamaguch static void	vioif_workq_work(struct work *, void *);
    373  1.55  yamaguch static void	vioif_work_set(struct vioif_work *, void(*)(void *), void *);
    374  1.55  yamaguch static void	vioif_work_add(struct workqueue *, struct vioif_work *);
    375  1.55  yamaguch static void	vioif_work_wait(struct workqueue *, struct vioif_work *);
    376  1.55  yamaguch 
    377   1.1   hannken /* other control */
    378  1.34     ozaki static bool	vioif_is_link_up(struct vioif_softc *);
    379  1.34     ozaki static void	vioif_update_link_status(struct vioif_softc *);
    380   1.1   hannken static int	vioif_ctrl_rx(struct vioif_softc *, int, bool);
    381   1.1   hannken static int	vioif_set_promisc(struct vioif_softc *, bool);
    382   1.1   hannken static int	vioif_set_allmulti(struct vioif_softc *, bool);
    383   1.1   hannken static int	vioif_set_rx_filter(struct vioif_softc *);
    384   1.1   hannken static int	vioif_rx_filter(struct vioif_softc *);
    385  1.54  yamaguch static int	vioif_ctrl_intr(void *);
    386  1.34     ozaki static int	vioif_config_change(struct virtio_softc *);
    387  1.34     ozaki static void	vioif_ctl_softint(void *);
    388  1.46  yamaguch static int	vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
    389  1.46  yamaguch static void	vioif_enable_interrupt_vqpairs(struct vioif_softc *);
    390  1.46  yamaguch static void	vioif_disable_interrupt_vqpairs(struct vioif_softc *);
    391  1.55  yamaguch static int	vioif_setup_sysctl(struct vioif_softc *);
    392   1.1   hannken 
    393   1.1   hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
    394   1.1   hannken 		  vioif_match, vioif_attach, NULL, NULL);
    395   1.1   hannken 
    396   1.1   hannken static int
    397   1.1   hannken vioif_match(device_t parent, cfdata_t match, void *aux)
    398   1.1   hannken {
    399  1.32  jdolecek 	struct virtio_attach_args *va = aux;
    400   1.1   hannken 
    401   1.1   hannken 	if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_NETWORK)
    402   1.1   hannken 		return 1;
    403   1.1   hannken 
    404   1.1   hannken 	return 0;
    405   1.1   hannken }
    406   1.1   hannken 
    407  1.58  yamaguch static int
    408  1.58  yamaguch vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
    409  1.58  yamaguch     bus_size_t size, int nsegs, const char *usage)
    410  1.58  yamaguch {
    411  1.58  yamaguch 	int r;
    412  1.58  yamaguch 
    413  1.58  yamaguch 	r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
    414  1.58  yamaguch 	    nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
    415  1.58  yamaguch 
    416  1.58  yamaguch 	if (r != 0) {
    417  1.58  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
    418  1.58  yamaguch 		    "error code %d\n", usage, r);
    419  1.58  yamaguch 	}
    420  1.58  yamaguch 
    421  1.58  yamaguch 	return r;
    422  1.58  yamaguch }
    423  1.58  yamaguch 
    424  1.58  yamaguch static void
    425  1.58  yamaguch vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
    426  1.58  yamaguch {
    427  1.58  yamaguch 
    428  1.58  yamaguch 	if (*map) {
    429  1.58  yamaguch 		bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
    430  1.58  yamaguch 		*map = NULL;
    431  1.58  yamaguch 	}
    432  1.58  yamaguch }
    433  1.58  yamaguch 
    434  1.58  yamaguch static int
    435  1.58  yamaguch vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
    436  1.58  yamaguch     void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
    437  1.58  yamaguch {
    438  1.58  yamaguch 	int r;
    439  1.58  yamaguch 
    440  1.58  yamaguch 	r = vioif_dmamap_create(sc, map, size, nsegs, usage);
    441  1.58  yamaguch 	if (r != 0)
    442  1.58  yamaguch 		return 1;
    443  1.58  yamaguch 
    444  1.58  yamaguch 	r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
    445  1.58  yamaguch 	    size, NULL, rw | BUS_DMA_NOWAIT);
    446  1.58  yamaguch 	if (r != 0) {
    447  1.58  yamaguch 		vioif_dmamap_destroy(sc, map);
    448  1.58  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
    449  1.58  yamaguch 		    "error code %d\n", usage, r);
    450  1.58  yamaguch 	}
    451  1.58  yamaguch 
    452  1.58  yamaguch 	return r;
    453  1.58  yamaguch }
    454  1.58  yamaguch 
    455  1.58  yamaguch static void *
    456  1.58  yamaguch vioif_assign_mem(intptr_t *p, size_t size)
    457  1.58  yamaguch {
    458  1.58  yamaguch 	intptr_t rv;
    459  1.58  yamaguch 
    460  1.58  yamaguch 	rv = *p;
    461  1.58  yamaguch 	*p += size;
    462  1.58  yamaguch 
    463  1.58  yamaguch 	return (void *)rv;
    464  1.58  yamaguch }
    465  1.58  yamaguch 
    466  1.51       chs static void
    467  1.46  yamaguch vioif_alloc_queues(struct vioif_softc *sc)
    468  1.46  yamaguch {
    469  1.46  yamaguch 	int nvq_pairs = sc->sc_max_nvq_pairs;
    470  1.46  yamaguch 	int nvqs = nvq_pairs * 2;
    471  1.46  yamaguch 	int i;
    472  1.46  yamaguch 
    473  1.46  yamaguch 	KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
    474  1.46  yamaguch 
    475  1.46  yamaguch 	sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
    476  1.51       chs 	    KM_SLEEP);
    477  1.46  yamaguch 	sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
    478  1.51       chs 	    KM_SLEEP);
    479  1.46  yamaguch 
    480  1.46  yamaguch 	if (sc->sc_has_ctrl)
    481  1.46  yamaguch 		nvqs++;
    482  1.46  yamaguch 
    483  1.51       chs 	sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
    484  1.46  yamaguch 	nvqs = 0;
    485  1.46  yamaguch 	for (i = 0; i < nvq_pairs; i++) {
    486  1.46  yamaguch 		sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
    487  1.46  yamaguch 		sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
    488  1.46  yamaguch 	}
    489  1.46  yamaguch 
    490  1.46  yamaguch 	if (sc->sc_has_ctrl)
    491  1.46  yamaguch 		sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
    492  1.46  yamaguch }
    493  1.46  yamaguch 
    494  1.46  yamaguch static void
    495  1.46  yamaguch vioif_free_queues(struct vioif_softc *sc)
    496  1.46  yamaguch {
    497  1.46  yamaguch 	int nvq_pairs = sc->sc_max_nvq_pairs;
    498  1.46  yamaguch 	int nvqs = nvq_pairs * 2;
    499  1.46  yamaguch 
    500  1.46  yamaguch 	if (sc->sc_ctrlq.ctrlq_vq)
    501  1.46  yamaguch 		nvqs++;
    502  1.46  yamaguch 
    503  1.46  yamaguch 	if (sc->sc_txq) {
    504  1.46  yamaguch 		kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
    505  1.46  yamaguch 		sc->sc_txq = NULL;
    506  1.46  yamaguch 	}
    507  1.46  yamaguch 
    508  1.46  yamaguch 	if (sc->sc_rxq) {
    509  1.46  yamaguch 		kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
    510  1.46  yamaguch 		sc->sc_rxq = NULL;
    511  1.46  yamaguch 	}
    512  1.46  yamaguch 
    513  1.46  yamaguch 	if (sc->sc_vqs) {
    514  1.46  yamaguch 		kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
    515  1.46  yamaguch 		sc->sc_vqs = NULL;
    516  1.46  yamaguch 	}
    517  1.46  yamaguch }
    518  1.46  yamaguch 
    519   1.1   hannken /* allocate memory */
    520   1.1   hannken /*
    521   1.1   hannken  * dma memory is used for:
    522  1.43  yamaguch  *   rxq_hdrs[slot]:	 metadata array for received frames (READ)
    523  1.43  yamaguch  *   txq_hdrs[slot]:	 metadata array for frames to be sent (WRITE)
    524  1.43  yamaguch  *   ctrlq_cmd:		 command to be sent via ctrl vq (WRITE)
    525  1.43  yamaguch  *   ctrlq_status:	 return value for a command via ctrl vq (READ)
    526  1.43  yamaguch  *   ctrlq_rx:		 parameter for a VIRTIO_NET_CTRL_RX class command
    527   1.1   hannken  *			 (WRITE)
    528  1.43  yamaguch  *   ctrlq_mac_tbl_uc:	 unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
    529   1.1   hannken  *			 class command (WRITE)
    530  1.43  yamaguch  *   ctrlq_mac_tbl_mc:	 multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
    531   1.1   hannken  *			 class command (WRITE)
    532  1.43  yamaguch  * ctrlq_* structures are allocated only one each; they are protected by
    533  1.43  yamaguch  * ctrlq_inuse variable and ctrlq_wait condvar.
    534   1.1   hannken  */
    535   1.1   hannken /*
    536   1.1   hannken  * dynamically allocated memory is used for:
    537  1.43  yamaguch  *   rxq_hdr_dmamaps[slot]:	bus_dmamap_t array for sc_rx_hdrs[slot]
    538  1.43  yamaguch  *   txq_hdr_dmamaps[slot]:	bus_dmamap_t array for sc_tx_hdrs[slot]
    539  1.43  yamaguch  *   rxq_dmamaps[slot]:		bus_dmamap_t array for received payload
    540  1.43  yamaguch  *   txq_dmamaps[slot]:		bus_dmamap_t array for sent payload
    541  1.43  yamaguch  *   rxq_mbufs[slot]:		mbuf pointer array for received frames
    542  1.43  yamaguch  *   txq_mbufs[slot]:		mbuf pointer array for sent frames
    543   1.1   hannken  */
    544   1.1   hannken static int
    545   1.1   hannken vioif_alloc_mems(struct vioif_softc *sc)
    546   1.1   hannken {
    547   1.1   hannken 	struct virtio_softc *vsc = sc->sc_virtio;
    548  1.46  yamaguch 	struct vioif_txqueue *txq;
    549  1.46  yamaguch 	struct vioif_rxqueue *rxq;
    550  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    551  1.46  yamaguch 	int allocsize, allocsize2, r, rsegs, i, qid;
    552   1.1   hannken 	void *vaddr;
    553   1.1   hannken 	intptr_t p;
    554   1.1   hannken 
    555  1.46  yamaguch 	allocsize = 0;
    556  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    557  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    558  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    559  1.46  yamaguch 
    560  1.46  yamaguch 		allocsize +=
    561  1.46  yamaguch 		    sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num;
    562  1.46  yamaguch 		allocsize +=
    563  1.46  yamaguch 		    sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num;
    564  1.46  yamaguch 	}
    565  1.32  jdolecek 	if (sc->sc_has_ctrl) {
    566   1.1   hannken 		allocsize += sizeof(struct virtio_net_ctrl_cmd) * 1;
    567   1.1   hannken 		allocsize += sizeof(struct virtio_net_ctrl_status) * 1;
    568   1.1   hannken 		allocsize += sizeof(struct virtio_net_ctrl_rx) * 1;
    569   1.1   hannken 		allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
    570  1.50  christos 		    + sizeof(struct virtio_net_ctrl_mac_tbl)
    571  1.50  christos 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
    572  1.46  yamaguch 		allocsize += sizeof(struct virtio_net_ctrl_mq) * 1;
    573   1.1   hannken 	}
    574  1.32  jdolecek 	r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0,
    575  1.50  christos 	    &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
    576   1.1   hannken 	if (r != 0) {
    577   1.1   hannken 		aprint_error_dev(sc->sc_dev,
    578  1.50  christos 		    "DMA memory allocation failed, size %d, "
    579  1.50  christos 		    "error code %d\n", allocsize, r);
    580   1.1   hannken 		goto err_none;
    581   1.1   hannken 	}
    582  1.32  jdolecek 	r = bus_dmamem_map(virtio_dmat(vsc),
    583  1.50  christos 	    &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT);
    584   1.1   hannken 	if (r != 0) {
    585   1.1   hannken 		aprint_error_dev(sc->sc_dev,
    586  1.50  christos 		    "DMA memory map failed, error code %d\n", r);
    587   1.1   hannken 		goto err_dmamem_alloc;
    588   1.1   hannken 	}
    589  1.42  yamaguch 
    590   1.1   hannken 	memset(vaddr, 0, allocsize);
    591  1.43  yamaguch 	sc->sc_dmamem = vaddr;
    592   1.1   hannken 	p = (intptr_t) vaddr;
    593  1.42  yamaguch 
    594  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    595  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    596  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    597  1.46  yamaguch 
    598  1.58  yamaguch 		rxq->rxq_hdrs = vioif_assign_mem(&p,
    599  1.46  yamaguch 		    sizeof(rxq->rxq_hdrs[0]) * rxq->rxq_vq->vq_num);
    600  1.58  yamaguch 		txq->txq_hdrs = vioif_assign_mem(&p,
    601  1.46  yamaguch 		    sizeof(txq->txq_hdrs[0]) * txq->txq_vq->vq_num);
    602  1.46  yamaguch 	}
    603  1.32  jdolecek 	if (sc->sc_has_ctrl) {
    604  1.58  yamaguch 		ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
    605  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_cmd));
    606  1.58  yamaguch 		ctrlq->ctrlq_status = vioif_assign_mem(&p,
    607  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_status));
    608  1.58  yamaguch 		ctrlq->ctrlq_rx = vioif_assign_mem(&p,
    609  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_rx));
    610  1.58  yamaguch 		ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
    611  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc));
    612  1.58  yamaguch 		ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
    613  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
    614  1.50  christos 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
    615  1.58  yamaguch 		ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
    616   1.1   hannken 	}
    617   1.1   hannken 
    618  1.46  yamaguch 	allocsize2 = 0;
    619  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    620  1.46  yamaguch 		int rxqsize, txqsize;
    621  1.46  yamaguch 
    622  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    623  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    624  1.46  yamaguch 		rxqsize = rxq->rxq_vq->vq_num;
    625  1.46  yamaguch 		txqsize = txq->txq_vq->vq_num;
    626  1.46  yamaguch 
    627  1.46  yamaguch 		allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize;
    628  1.46  yamaguch 		allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize;
    629  1.46  yamaguch 		allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize;
    630  1.46  yamaguch 
    631  1.46  yamaguch 		allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize;
    632  1.46  yamaguch 		allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize;
    633  1.46  yamaguch 		allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize;
    634  1.46  yamaguch 	}
    635  1.42  yamaguch 	vaddr = kmem_zalloc(allocsize2, KM_SLEEP);
    636  1.43  yamaguch 	sc->sc_kmem = vaddr;
    637  1.42  yamaguch 	p = (intptr_t) vaddr;
    638  1.42  yamaguch 
    639  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    640  1.46  yamaguch 		int rxqsize, txqsize;
    641  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    642  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    643  1.46  yamaguch 		rxqsize = rxq->rxq_vq->vq_num;
    644  1.46  yamaguch 		txqsize = txq->txq_vq->vq_num;
    645  1.46  yamaguch 
    646  1.58  yamaguch 		rxq->rxq_hdr_dmamaps = vioif_assign_mem(&p,
    647  1.48   msaitoh 		    sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize);
    648  1.58  yamaguch 		txq->txq_hdr_dmamaps = vioif_assign_mem(&p,
    649  1.48   msaitoh 		    sizeof(txq->txq_hdr_dmamaps[0]) * txqsize);
    650  1.58  yamaguch 		rxq->rxq_dmamaps = vioif_assign_mem(&p,
    651  1.58  yamaguch 		    sizeof(rxq->rxq_dmamaps[0]) * rxqsize);
    652  1.58  yamaguch 		txq->txq_dmamaps = vioif_assign_mem(&p,
    653  1.58  yamaguch 		    sizeof(txq->txq_dmamaps[0]) * txqsize);
    654  1.58  yamaguch 		rxq->rxq_mbufs = vioif_assign_mem(&p,
    655  1.58  yamaguch 		    sizeof(rxq->rxq_mbufs[0]) * rxqsize);
    656  1.58  yamaguch 		txq->txq_mbufs = vioif_assign_mem(&p,
    657  1.58  yamaguch 		    sizeof(txq->txq_mbufs[0]) * txqsize);
    658  1.58  yamaguch 	}
    659   1.1   hannken 
    660  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    661  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    662  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    663  1.46  yamaguch 
    664  1.46  yamaguch 		for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
    665  1.58  yamaguch 			r = vioif_dmamap_create_load(sc, &rxq->rxq_hdr_dmamaps[i],
    666  1.58  yamaguch 			    &rxq->rxq_hdrs[i], sizeof(rxq->rxq_hdrs[0]), 1,
    667  1.46  yamaguch 			    BUS_DMA_READ, "rx header");
    668  1.58  yamaguch 			if (r != 0)
    669  1.58  yamaguch 				goto err_reqs;
    670  1.58  yamaguch 
    671  1.58  yamaguch 			r = vioif_dmamap_create(sc, &rxq->rxq_dmamaps[i],
    672  1.58  yamaguch 			    MCLBYTES, 1, "rx payload");
    673  1.58  yamaguch 			if (r != 0)
    674  1.58  yamaguch 				goto err_reqs;
    675  1.46  yamaguch 		}
    676  1.46  yamaguch 
    677  1.46  yamaguch 		for (i = 0; i < txq->txq_vq->vq_num; i++) {
    678  1.58  yamaguch 			r = vioif_dmamap_create_load(sc, &txq->txq_hdr_dmamaps[i],
    679  1.58  yamaguch 			    &txq->txq_hdrs[i], sizeof(txq->txq_hdrs[0]), 1,
    680  1.46  yamaguch 			    BUS_DMA_READ, "tx header");
    681  1.58  yamaguch 			if (r != 0)
    682  1.58  yamaguch 				goto err_reqs;
    683  1.58  yamaguch 
    684  1.58  yamaguch 			r = vioif_dmamap_create(sc, &txq->txq_dmamaps[i], ETHER_MAX_LEN,
    685  1.46  yamaguch 			    VIRTIO_NET_TX_MAXNSEGS, "tx payload");
    686  1.58  yamaguch 			if (r != 0)
    687  1.58  yamaguch 				goto err_reqs;
    688  1.46  yamaguch 		}
    689   1.1   hannken 	}
    690   1.1   hannken 
    691  1.32  jdolecek 	if (sc->sc_has_ctrl) {
    692   1.1   hannken 		/* control vq class & command */
    693  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
    694  1.43  yamaguch 		    ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
    695  1.42  yamaguch 		    BUS_DMA_WRITE, "control command");
    696  1.58  yamaguch 		if (r != 0)
    697  1.58  yamaguch 			goto err_reqs;
    698  1.58  yamaguch 
    699  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
    700  1.43  yamaguch 		    ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
    701  1.42  yamaguch 		    BUS_DMA_READ, "control status");
    702  1.58  yamaguch 		if (r != 0)
    703  1.58  yamaguch 			goto err_reqs;
    704   1.1   hannken 
    705   1.1   hannken 		/* control vq rx mode command parameter */
    706  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
    707  1.43  yamaguch 		    ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
    708  1.42  yamaguch 		    BUS_DMA_WRITE, "rx mode control command");
    709  1.58  yamaguch 		if (r != 0)
    710  1.58  yamaguch 			goto err_reqs;
    711   1.1   hannken 
    712  1.46  yamaguch 		/* multiqueue set command */
    713  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
    714  1.46  yamaguch 		    ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
    715  1.46  yamaguch 		    BUS_DMA_WRITE, "multiqueue set command");
    716  1.58  yamaguch 		if (r != 0)
    717  1.58  yamaguch 			goto err_reqs;
    718  1.46  yamaguch 
    719   1.1   hannken 		/* control vq MAC filter table for unicast */
    720   1.1   hannken 		/* do not load now since its length is variable */
    721  1.58  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
    722  1.43  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc) + 0, 1,
    723  1.42  yamaguch 		    "unicast MAC address filter command");
    724  1.58  yamaguch 		if (r != 0)
    725  1.58  yamaguch 			goto err_reqs;
    726   1.1   hannken 
    727   1.1   hannken 		/* control vq MAC filter table for multicast */
    728  1.58  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
    729  1.43  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
    730  1.42  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
    731  1.42  yamaguch 		    "multicast MAC address filter command");
    732   1.1   hannken 	}
    733   1.1   hannken 
    734   1.1   hannken 	return 0;
    735   1.1   hannken 
    736   1.1   hannken err_reqs:
    737  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
    738  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
    739  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
    740  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
    741  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
    742  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    743  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    744  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    745  1.46  yamaguch 
    746  1.46  yamaguch 		for (i = 0; i < txq->txq_vq->vq_num; i++) {
    747  1.58  yamaguch 			vioif_dmamap_destroy(sc, &txq->txq_dmamaps[i]);
    748  1.58  yamaguch 			vioif_dmamap_destroy(sc, &txq->txq_hdr_dmamaps[i]);
    749  1.46  yamaguch 		}
    750  1.46  yamaguch 		for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
    751  1.58  yamaguch 			vioif_dmamap_destroy(sc, &rxq->rxq_dmamaps[i]);
    752  1.58  yamaguch 			vioif_dmamap_destroy(sc, &rxq->rxq_hdr_dmamaps[i]);
    753  1.46  yamaguch 		}
    754   1.1   hannken 	}
    755  1.43  yamaguch 	if (sc->sc_kmem) {
    756  1.43  yamaguch 		kmem_free(sc->sc_kmem, allocsize2);
    757  1.43  yamaguch 		sc->sc_kmem = NULL;
    758   1.1   hannken 	}
    759  1.43  yamaguch 	bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize);
    760   1.1   hannken err_dmamem_alloc:
    761  1.32  jdolecek 	bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
    762   1.1   hannken err_none:
    763   1.1   hannken 	return -1;
    764   1.1   hannken }
    765   1.1   hannken 
    766   1.1   hannken static void
    767   1.1   hannken vioif_attach(device_t parent, device_t self, void *aux)
    768   1.1   hannken {
    769   1.1   hannken 	struct vioif_softc *sc = device_private(self);
    770   1.1   hannken 	struct virtio_softc *vsc = device_private(parent);
    771  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    772  1.46  yamaguch 	struct vioif_txqueue *txq;
    773  1.46  yamaguch 	struct vioif_rxqueue *rxq;
    774  1.46  yamaguch 	uint32_t features, req_features;
    775   1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    776  1.43  yamaguch 	u_int softint_flags;
    777  1.46  yamaguch 	int r, i, nvqs=0, req_flags;
    778  1.55  yamaguch 	char xnamebuf[MAXCOMLEN];
    779   1.1   hannken 
    780  1.32  jdolecek 	if (virtio_child(vsc) != NULL) {
    781   1.1   hannken 		aprint_normal(": child already attached for %s; "
    782  1.50  christos 		    "something wrong...\n", device_xname(parent));
    783   1.1   hannken 		return;
    784   1.1   hannken 	}
    785   1.1   hannken 
    786   1.1   hannken 	sc->sc_dev = self;
    787   1.1   hannken 	sc->sc_virtio = vsc;
    788  1.34     ozaki 	sc->sc_link_active = false;
    789   1.1   hannken 
    790  1.46  yamaguch 	sc->sc_max_nvq_pairs = 1;
    791  1.46  yamaguch 	sc->sc_req_nvq_pairs = 1;
    792  1.46  yamaguch 	sc->sc_act_nvq_pairs = 1;
    793  1.55  yamaguch 	sc->sc_txrx_workqueue_sysctl = true;
    794  1.55  yamaguch 	sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
    795  1.55  yamaguch 	sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
    796  1.55  yamaguch 	sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
    797  1.55  yamaguch 	sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
    798  1.55  yamaguch 
    799  1.55  yamaguch 	snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
    800  1.55  yamaguch 	sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
    801  1.55  yamaguch 	    IPL_NET, WQ_PERCPU | WQ_MPSAFE);
    802  1.55  yamaguch 	if (sc->sc_txrx_workqueue == NULL)
    803  1.55  yamaguch 		goto err;
    804  1.46  yamaguch 
    805  1.32  jdolecek 	req_flags = 0;
    806   1.1   hannken 
    807   1.7     ozaki #ifdef VIOIF_MPSAFE
    808  1.32  jdolecek 	req_flags |= VIRTIO_F_PCI_INTR_MPSAFE;
    809   1.7     ozaki #endif
    810  1.32  jdolecek 	req_flags |= VIRTIO_F_PCI_INTR_MSIX;
    811  1.32  jdolecek 
    812  1.46  yamaguch 	req_features =
    813  1.46  yamaguch 	    VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
    814  1.46  yamaguch 	    VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
    815  1.46  yamaguch #ifdef VIOIF_MULTIQ
    816  1.46  yamaguch 	req_features |= VIRTIO_NET_F_MQ;
    817  1.46  yamaguch #endif
    818  1.46  yamaguch 	virtio_child_attach_start(vsc, self, IPL_NET, NULL,
    819  1.54  yamaguch 	    vioif_config_change, virtio_vq_intrhand, req_flags,
    820  1.46  yamaguch 	    req_features, VIRTIO_NET_FLAG_BITS);
    821  1.32  jdolecek 
    822  1.32  jdolecek 	features = virtio_features(vsc);
    823   1.7     ozaki 
    824   1.1   hannken 	if (features & VIRTIO_NET_F_MAC) {
    825  1.50  christos 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    826  1.50  christos 			sc->sc_mac[i] = virtio_read_device_config_1(vsc,
    827  1.50  christos 			    VIRTIO_NET_CONFIG_MAC + i);
    828  1.50  christos 		}
    829   1.1   hannken 	} else {
    830   1.1   hannken 		/* code stolen from sys/net/if_tap.c */
    831   1.1   hannken 		struct timeval tv;
    832   1.1   hannken 		uint32_t ui;
    833   1.1   hannken 		getmicrouptime(&tv);
    834   1.1   hannken 		ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
    835   1.1   hannken 		memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
    836  1.50  christos 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    837  1.50  christos 			virtio_write_device_config_1(vsc,
    838  1.50  christos 			    VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
    839  1.50  christos 		}
    840   1.1   hannken 	}
    841  1.32  jdolecek 
    842  1.50  christos 	aprint_normal_dev(self, "Ethernet address %s\n",
    843  1.50  christos 	    ether_sprintf(sc->sc_mac));
    844   1.1   hannken 
    845  1.46  yamaguch 	if ((features & VIRTIO_NET_F_CTRL_VQ) &&
    846  1.46  yamaguch 	    (features & VIRTIO_NET_F_CTRL_RX)) {
    847  1.46  yamaguch 		sc->sc_has_ctrl = true;
    848  1.46  yamaguch 
    849  1.46  yamaguch 		cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
    850  1.46  yamaguch 		mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
    851  1.46  yamaguch 		ctrlq->ctrlq_inuse = FREE;
    852  1.46  yamaguch 	} else {
    853  1.46  yamaguch 		sc->sc_has_ctrl = false;
    854  1.46  yamaguch 	}
    855  1.46  yamaguch 
    856  1.46  yamaguch 	if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
    857  1.46  yamaguch 		sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
    858  1.46  yamaguch 		    VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
    859  1.46  yamaguch 
    860  1.46  yamaguch 		if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
    861  1.46  yamaguch 			goto err;
    862  1.46  yamaguch 
    863  1.46  yamaguch 		/* Limit the number of queue pairs to use */
    864  1.46  yamaguch 		sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
    865  1.46  yamaguch 	}
    866  1.46  yamaguch 
    867  1.51       chs 	vioif_alloc_queues(sc);
    868  1.46  yamaguch 	virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
    869  1.46  yamaguch 
    870  1.43  yamaguch #ifdef VIOIF_MPSAFE
    871  1.43  yamaguch 	softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
    872  1.43  yamaguch #else
    873  1.43  yamaguch 	softint_flags = SOFTINT_NET;
    874  1.43  yamaguch #endif
    875   1.7     ozaki 
    876  1.17     ozaki 	/*
    877  1.57  yamaguch 	 * Allocating virtqueues
    878  1.17     ozaki 	 */
    879  1.46  yamaguch 	for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
    880  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
    881  1.46  yamaguch 		txq = &sc->sc_txq[i];
    882  1.46  yamaguch 		char qname[32];
    883  1.46  yamaguch 
    884  1.46  yamaguch 		rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
    885  1.46  yamaguch 
    886  1.50  christos 		rxq->rxq_softint = softint_establish(softint_flags,
    887  1.50  christos 		    vioif_rx_softint, rxq);
    888  1.46  yamaguch 		if (rxq->rxq_softint == NULL) {
    889  1.46  yamaguch 			aprint_error_dev(self, "cannot establish rx softint\n");
    890  1.46  yamaguch 			goto err;
    891  1.46  yamaguch 		}
    892  1.55  yamaguch 		rxq->rxq_handle_si = softint_establish(softint_flags,
    893  1.55  yamaguch 		    vioif_rx_handle, rxq);
    894  1.55  yamaguch 		if (rxq->rxq_handle_si == NULL) {
    895  1.55  yamaguch 			aprint_error_dev(self, "cannot establish rx softint\n");
    896  1.55  yamaguch 			goto err;
    897  1.55  yamaguch 		}
    898  1.55  yamaguch 
    899  1.46  yamaguch 		snprintf(qname, sizeof(qname), "rx%d", i);
    900  1.46  yamaguch 		r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs,
    901  1.56  yamaguch 		    MCLBYTES+sizeof(struct virtio_net_hdr), 2, qname);
    902  1.46  yamaguch 		if (r != 0)
    903  1.46  yamaguch 			goto err;
    904  1.46  yamaguch 		nvqs++;
    905  1.54  yamaguch 		rxq->rxq_vq->vq_intrhand = vioif_rx_intr;
    906  1.54  yamaguch 		rxq->rxq_vq->vq_intrhand_arg = (void *)rxq;
    907  1.46  yamaguch 		rxq->rxq_stopping = true;
    908  1.55  yamaguch 		vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq);
    909  1.46  yamaguch 
    910  1.46  yamaguch 		txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
    911  1.55  yamaguch 
    912  1.46  yamaguch 		txq->txq_deferred_transmit = softint_establish(softint_flags,
    913  1.46  yamaguch 		    vioif_deferred_transmit, txq);
    914  1.46  yamaguch 		if (txq->txq_deferred_transmit == NULL) {
    915  1.46  yamaguch 			aprint_error_dev(self, "cannot establish tx softint\n");
    916  1.46  yamaguch 			goto err;
    917  1.46  yamaguch 		}
    918  1.55  yamaguch 		txq->txq_handle_si = softint_establish(softint_flags,
    919  1.55  yamaguch 		    vioif_tx_handle, txq);
    920  1.55  yamaguch 		if (txq->txq_handle_si == NULL) {
    921  1.55  yamaguch 			aprint_error_dev(self, "cannot establish tx softint\n");
    922  1.55  yamaguch 			goto err;
    923  1.55  yamaguch 		}
    924  1.55  yamaguch 
    925  1.46  yamaguch 		snprintf(qname, sizeof(qname), "tx%d", i);
    926  1.46  yamaguch 		r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs,
    927  1.50  christos 		    sizeof(struct virtio_net_hdr)
    928  1.50  christos 		    + (ETHER_MAX_LEN - ETHER_HDR_LEN),
    929  1.46  yamaguch 		    VIRTIO_NET_TX_MAXNSEGS + 1, qname);
    930  1.46  yamaguch 		if (r != 0)
    931  1.46  yamaguch 			goto err;
    932  1.46  yamaguch 		nvqs++;
    933  1.54  yamaguch 		txq->txq_vq->vq_intrhand = vioif_tx_intr;
    934  1.54  yamaguch 		txq->txq_vq->vq_intrhand_arg = (void *)txq;
    935  1.46  yamaguch 		txq->txq_link_active = sc->sc_link_active;
    936  1.46  yamaguch 		txq->txq_stopping = false;
    937  1.51       chs 		txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP);
    938  1.55  yamaguch 		vioif_work_set(&txq->txq_work, vioif_tx_handle, txq);
    939  1.43  yamaguch 	}
    940  1.17     ozaki 
    941  1.46  yamaguch 	if (sc->sc_has_ctrl) {
    942  1.17     ozaki 		/*
    943  1.17     ozaki 		 * Allocating a virtqueue for control channel
    944  1.17     ozaki 		 */
    945  1.46  yamaguch 		r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs,
    946  1.17     ozaki 		    NBPG, 1, "control");
    947  1.17     ozaki 		if (r != 0) {
    948  1.17     ozaki 			aprint_error_dev(self, "failed to allocate "
    949  1.50  christos 			    "a virtqueue for control channel, error code %d\n",
    950  1.50  christos 			    r);
    951  1.46  yamaguch 
    952  1.46  yamaguch 			sc->sc_has_ctrl = false;
    953  1.46  yamaguch 			cv_destroy(&ctrlq->ctrlq_wait);
    954  1.46  yamaguch 			mutex_destroy(&ctrlq->ctrlq_wait_lock);
    955  1.46  yamaguch 		} else {
    956  1.46  yamaguch 			nvqs++;
    957  1.54  yamaguch 			ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
    958  1.54  yamaguch 			ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
    959   1.1   hannken 		}
    960   1.1   hannken 	}
    961  1.34     ozaki 
    962  1.48   msaitoh 	sc->sc_ctl_softint = softint_establish(softint_flags,
    963  1.48   msaitoh 	    vioif_ctl_softint, sc);
    964  1.34     ozaki 	if (sc->sc_ctl_softint == NULL) {
    965  1.34     ozaki 		aprint_error_dev(self, "cannot establish ctl softint\n");
    966   1.1   hannken 		goto err;
    967   1.1   hannken 	}
    968   1.1   hannken 
    969   1.1   hannken 	if (vioif_alloc_mems(sc) < 0)
    970   1.1   hannken 		goto err;
    971   1.1   hannken 
    972  1.32  jdolecek 	if (virtio_child_attach_finish(vsc) != 0)
    973  1.32  jdolecek 		goto err;
    974  1.32  jdolecek 
    975  1.55  yamaguch 	if (vioif_setup_sysctl(sc) != 0) {
    976  1.55  yamaguch 		aprint_error_dev(self, "unable to create sysctl node\n");
    977  1.55  yamaguch 		/* continue */
    978  1.55  yamaguch 	}
    979  1.55  yamaguch 
    980   1.1   hannken 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
    981   1.1   hannken 	ifp->if_softc = sc;
    982   1.1   hannken 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
    983  1.45  yamaguch #ifdef VIOIF_MPSAFE
    984  1.45  yamaguch 	ifp->if_extflags = IFEF_MPSAFE;
    985  1.45  yamaguch #endif
    986   1.1   hannken 	ifp->if_start = vioif_start;
    987  1.46  yamaguch 	if (sc->sc_req_nvq_pairs > 1)
    988  1.46  yamaguch 		ifp->if_transmit = vioif_transmit;
    989   1.1   hannken 	ifp->if_ioctl = vioif_ioctl;
    990   1.1   hannken 	ifp->if_init = vioif_init;
    991   1.1   hannken 	ifp->if_stop = vioif_stop;
    992   1.1   hannken 	ifp->if_capabilities = 0;
    993   1.1   hannken 	ifp->if_watchdog = vioif_watchdog;
    994  1.46  yamaguch 	txq = &sc->sc_txq[0];
    995  1.43  yamaguch 	IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN));
    996  1.36  jdolecek 	IFQ_SET_READY(&ifp->if_snd);
    997   1.1   hannken 
    998  1.11     ozaki 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
    999  1.11     ozaki 
   1000   1.1   hannken 	if_attach(ifp);
   1001  1.28     ozaki 	if_deferred_start_init(ifp, NULL);
   1002   1.1   hannken 	ether_ifattach(ifp, sc->sc_mac);
   1003   1.1   hannken 
   1004   1.1   hannken 	return;
   1005   1.1   hannken 
   1006   1.1   hannken err:
   1007  1.46  yamaguch 	for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
   1008  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1009  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1010  1.46  yamaguch 
   1011  1.46  yamaguch 		if (rxq->rxq_lock) {
   1012  1.46  yamaguch 			mutex_obj_free(rxq->rxq_lock);
   1013  1.46  yamaguch 			rxq->rxq_lock = NULL;
   1014  1.46  yamaguch 		}
   1015  1.46  yamaguch 
   1016  1.46  yamaguch 		if (rxq->rxq_softint) {
   1017  1.46  yamaguch 			softint_disestablish(rxq->rxq_softint);
   1018  1.46  yamaguch 			rxq->rxq_softint = NULL;
   1019  1.46  yamaguch 		}
   1020  1.46  yamaguch 
   1021  1.55  yamaguch 		if (rxq->rxq_handle_si) {
   1022  1.55  yamaguch 			softint_disestablish(rxq->rxq_handle_si);
   1023  1.55  yamaguch 			rxq->rxq_handle_si = NULL;
   1024  1.55  yamaguch 		}
   1025  1.55  yamaguch 
   1026  1.46  yamaguch 		if (txq->txq_lock) {
   1027  1.46  yamaguch 			mutex_obj_free(txq->txq_lock);
   1028  1.46  yamaguch 			txq->txq_lock = NULL;
   1029  1.46  yamaguch 		}
   1030  1.43  yamaguch 
   1031  1.55  yamaguch 		if (txq->txq_handle_si) {
   1032  1.55  yamaguch 			softint_disestablish(txq->txq_handle_si);
   1033  1.55  yamaguch 			txq->txq_handle_si = NULL;
   1034  1.55  yamaguch 		}
   1035  1.55  yamaguch 
   1036  1.46  yamaguch 		if (txq->txq_deferred_transmit) {
   1037  1.46  yamaguch 			softint_disestablish(txq->txq_deferred_transmit);
   1038  1.46  yamaguch 			txq->txq_deferred_transmit = NULL;
   1039  1.46  yamaguch 		}
   1040  1.43  yamaguch 
   1041  1.46  yamaguch 		if (txq->txq_intrq) {
   1042  1.46  yamaguch 			pcq_destroy(txq->txq_intrq);
   1043  1.46  yamaguch 			txq->txq_intrq = NULL;
   1044  1.46  yamaguch 		}
   1045  1.43  yamaguch 	}
   1046   1.7     ozaki 
   1047  1.32  jdolecek 	if (sc->sc_has_ctrl) {
   1048  1.43  yamaguch 		cv_destroy(&ctrlq->ctrlq_wait);
   1049  1.43  yamaguch 		mutex_destroy(&ctrlq->ctrlq_wait_lock);
   1050   1.1   hannken 	}
   1051  1.20  christos 
   1052  1.32  jdolecek 	while (nvqs > 0)
   1053  1.46  yamaguch 		virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]);
   1054  1.46  yamaguch 
   1055  1.46  yamaguch 	vioif_free_queues(sc);
   1056  1.55  yamaguch 	virtio_child_attach_failed(vsc);
   1057  1.55  yamaguch 	config_finalize_register(self, vioif_finalize_teardown);
   1058  1.20  christos 
   1059   1.1   hannken 	return;
   1060   1.1   hannken }
   1061   1.1   hannken 
   1062  1.55  yamaguch static int
   1063  1.55  yamaguch vioif_finalize_teardown(device_t self)
   1064  1.55  yamaguch {
   1065  1.55  yamaguch 	struct vioif_softc *sc = device_private(self);
   1066  1.55  yamaguch 
   1067  1.55  yamaguch 	if (sc->sc_txrx_workqueue != NULL) {
   1068  1.55  yamaguch 		vioif_workq_destroy(sc->sc_txrx_workqueue);
   1069  1.55  yamaguch 		sc->sc_txrx_workqueue = NULL;
   1070  1.55  yamaguch 	}
   1071  1.55  yamaguch 
   1072  1.55  yamaguch 	return 0;
   1073  1.55  yamaguch }
   1074  1.55  yamaguch 
   1075   1.1   hannken /* we need interrupts to make promiscuous mode off */
   1076   1.1   hannken static void
   1077   1.1   hannken vioif_deferred_init(device_t self)
   1078   1.1   hannken {
   1079   1.1   hannken 	struct vioif_softc *sc = device_private(self);
   1080   1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1081   1.1   hannken 	int r;
   1082   1.1   hannken 
   1083   1.9     ozaki 	if (ifp->if_flags & IFF_PROMISC)
   1084  1.10     ozaki 		return;
   1085   1.9     ozaki 
   1086   1.1   hannken 	r =  vioif_set_promisc(sc, false);
   1087   1.1   hannken 	if (r != 0)
   1088   1.1   hannken 		aprint_error_dev(self, "resetting promisc mode failed, "
   1089  1.50  christos 		    "error code %d\n", r);
   1090   1.1   hannken }
   1091   1.1   hannken 
   1092  1.46  yamaguch static void
   1093  1.46  yamaguch vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
   1094  1.46  yamaguch {
   1095  1.46  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1096  1.46  yamaguch 	struct vioif_txqueue *txq;
   1097  1.46  yamaguch 	struct vioif_rxqueue *rxq;
   1098  1.46  yamaguch 	int i;
   1099  1.46  yamaguch 
   1100  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1101  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1102  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1103  1.46  yamaguch 
   1104  1.46  yamaguch 		virtio_start_vq_intr(vsc, txq->txq_vq);
   1105  1.46  yamaguch 		virtio_start_vq_intr(vsc, rxq->rxq_vq);
   1106  1.46  yamaguch 	}
   1107  1.46  yamaguch }
   1108  1.46  yamaguch 
   1109  1.46  yamaguch static void
   1110  1.46  yamaguch vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
   1111  1.46  yamaguch {
   1112  1.46  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1113  1.46  yamaguch 	struct vioif_txqueue *txq;
   1114  1.46  yamaguch 	struct vioif_rxqueue *rxq;
   1115  1.46  yamaguch 	int i;
   1116  1.46  yamaguch 
   1117  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1118  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1119  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1120  1.46  yamaguch 
   1121  1.46  yamaguch 		virtio_stop_vq_intr(vsc, txq->txq_vq);
   1122  1.46  yamaguch 		virtio_stop_vq_intr(vsc, rxq->rxq_vq);
   1123  1.46  yamaguch 	}
   1124  1.46  yamaguch }
   1125  1.46  yamaguch 
   1126   1.1   hannken /*
   1127   1.1   hannken  * Interface functions for ifnet
   1128   1.1   hannken  */
   1129   1.1   hannken static int
   1130   1.1   hannken vioif_init(struct ifnet *ifp)
   1131   1.1   hannken {
   1132   1.1   hannken 	struct vioif_softc *sc = ifp->if_softc;
   1133  1.33     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   1134  1.46  yamaguch 	struct vioif_rxqueue *rxq;
   1135  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1136  1.46  yamaguch 	int r, i;
   1137   1.1   hannken 
   1138   1.1   hannken 	vioif_stop(ifp, 0);
   1139   1.7     ozaki 
   1140  1.33     ozaki 	virtio_reinit_start(vsc);
   1141  1.33     ozaki 	virtio_negotiate_features(vsc, virtio_features(vsc));
   1142  1.46  yamaguch 
   1143  1.46  yamaguch 	for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
   1144  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1145  1.46  yamaguch 
   1146  1.46  yamaguch 		/* Have to set false before vioif_populate_rx_mbufs */
   1147  1.46  yamaguch 		rxq->rxq_stopping = false;
   1148  1.46  yamaguch 		vioif_populate_rx_mbufs(rxq);
   1149  1.46  yamaguch 	}
   1150  1.46  yamaguch 
   1151  1.46  yamaguch 	virtio_reinit_end(vsc);
   1152  1.47  yamaguch 
   1153  1.47  yamaguch 	if (sc->sc_has_ctrl)
   1154  1.43  yamaguch 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
   1155  1.46  yamaguch 
   1156  1.46  yamaguch 	r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
   1157  1.46  yamaguch 	if (r == 0)
   1158  1.46  yamaguch 		sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
   1159  1.46  yamaguch 	else
   1160  1.46  yamaguch 		sc->sc_act_nvq_pairs = 1;
   1161  1.46  yamaguch 
   1162  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++)
   1163  1.46  yamaguch 		sc->sc_txq[i].txq_stopping = false;
   1164  1.46  yamaguch 
   1165  1.46  yamaguch 	vioif_enable_interrupt_vqpairs(sc);
   1166  1.33     ozaki 
   1167   1.8     pooka 	if (!sc->sc_deferred_init_done) {
   1168   1.8     pooka 		sc->sc_deferred_init_done = 1;
   1169  1.32  jdolecek 		if (sc->sc_has_ctrl)
   1170   1.8     pooka 			vioif_deferred_init(sc->sc_dev);
   1171   1.8     pooka 	}
   1172   1.8     pooka 
   1173  1.34     ozaki 	vioif_update_link_status(sc);
   1174   1.1   hannken 	ifp->if_flags |= IFF_RUNNING;
   1175   1.1   hannken 	ifp->if_flags &= ~IFF_OACTIVE;
   1176   1.1   hannken 	vioif_rx_filter(sc);
   1177   1.1   hannken 
   1178   1.1   hannken 	return 0;
   1179   1.1   hannken }
   1180   1.1   hannken 
   1181   1.1   hannken static void
   1182   1.1   hannken vioif_stop(struct ifnet *ifp, int disable)
   1183   1.1   hannken {
   1184   1.1   hannken 	struct vioif_softc *sc = ifp->if_softc;
   1185   1.1   hannken 	struct virtio_softc *vsc = sc->sc_virtio;
   1186  1.46  yamaguch 	struct vioif_txqueue *txq;
   1187  1.46  yamaguch 	struct vioif_rxqueue *rxq;
   1188  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1189  1.46  yamaguch 	int i;
   1190   1.1   hannken 
   1191  1.13     ozaki 	/* Take the locks to ensure that ongoing TX/RX finish */
   1192  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1193  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1194  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1195  1.46  yamaguch 
   1196  1.46  yamaguch 		mutex_enter(txq->txq_lock);
   1197  1.46  yamaguch 		txq->txq_stopping = true;
   1198  1.46  yamaguch 		mutex_exit(txq->txq_lock);
   1199  1.46  yamaguch 
   1200  1.46  yamaguch 		mutex_enter(rxq->rxq_lock);
   1201  1.46  yamaguch 		rxq->rxq_stopping = true;
   1202  1.46  yamaguch 		mutex_exit(rxq->rxq_lock);
   1203  1.46  yamaguch 	}
   1204   1.7     ozaki 
   1205  1.33     ozaki 	/* disable interrupts */
   1206  1.46  yamaguch 	vioif_disable_interrupt_vqpairs(sc);
   1207  1.46  yamaguch 
   1208  1.33     ozaki 	if (sc->sc_has_ctrl)
   1209  1.43  yamaguch 		virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
   1210  1.33     ozaki 
   1211   1.1   hannken 	/* only way to stop I/O and DMA is resetting... */
   1212   1.1   hannken 	virtio_reset(vsc);
   1213  1.55  yamaguch 
   1214  1.55  yamaguch 	/* rendezvous for finish of handlers */
   1215  1.55  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1216  1.55  yamaguch 		txq = &sc->sc_txq[i];
   1217  1.55  yamaguch 		rxq = &sc->sc_rxq[i];
   1218  1.55  yamaguch 
   1219  1.55  yamaguch 		mutex_enter(txq->txq_lock);
   1220  1.55  yamaguch 		mutex_exit(txq->txq_lock);
   1221  1.55  yamaguch 
   1222  1.55  yamaguch 		mutex_enter(rxq->rxq_lock);
   1223  1.55  yamaguch 		mutex_exit(rxq->rxq_lock);
   1224  1.55  yamaguch 
   1225  1.55  yamaguch 		vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work);
   1226  1.55  yamaguch 		vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work);
   1227  1.55  yamaguch 	}
   1228  1.55  yamaguch 
   1229  1.55  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1230  1.55  yamaguch 		vioif_rx_queue_clear(&sc->sc_rxq[i]);
   1231  1.55  yamaguch 		vioif_tx_queue_clear(&sc->sc_txq[i]);
   1232  1.55  yamaguch 	}
   1233  1.46  yamaguch 
   1234   1.1   hannken 	ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
   1235  1.34     ozaki 	sc->sc_link_active = false;
   1236   1.1   hannken 
   1237  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1238  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1239  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1240  1.46  yamaguch 
   1241  1.46  yamaguch 		txq->txq_link_active = false;
   1242  1.46  yamaguch 
   1243  1.46  yamaguch 		if (disable)
   1244  1.46  yamaguch 			vioif_rx_drain(rxq);
   1245  1.46  yamaguch 
   1246  1.46  yamaguch 		vioif_tx_drain(txq);
   1247  1.46  yamaguch 	}
   1248   1.1   hannken }
   1249   1.1   hannken 
   1250   1.1   hannken static void
   1251  1.48   msaitoh vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq,
   1252  1.48   msaitoh     bool is_transmit)
   1253   1.1   hannken {
   1254   1.1   hannken 	struct vioif_softc *sc = ifp->if_softc;
   1255   1.1   hannken 	struct virtio_softc *vsc = sc->sc_virtio;
   1256  1.43  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1257   1.1   hannken 	struct mbuf *m;
   1258  1.36  jdolecek 	int queued = 0;
   1259   1.1   hannken 
   1260  1.46  yamaguch 	KASSERT(mutex_owned(txq->txq_lock));
   1261  1.46  yamaguch 
   1262  1.46  yamaguch 	if ((ifp->if_flags & IFF_RUNNING) == 0)
   1263  1.46  yamaguch 		return;
   1264   1.7     ozaki 
   1265  1.46  yamaguch 	if (!txq->txq_link_active || txq->txq_stopping)
   1266  1.46  yamaguch 		return;
   1267   1.7     ozaki 
   1268  1.46  yamaguch 	if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit)
   1269  1.46  yamaguch 		return;
   1270   1.1   hannken 
   1271   1.2  jmcneill 	for (;;) {
   1272   1.1   hannken 		int slot, r;
   1273   1.1   hannken 
   1274  1.46  yamaguch 		if (is_transmit)
   1275  1.46  yamaguch 			m = pcq_get(txq->txq_intrq);
   1276  1.46  yamaguch 		else
   1277  1.46  yamaguch 			IFQ_DEQUEUE(&ifp->if_snd, m);
   1278  1.46  yamaguch 
   1279   1.2  jmcneill 		if (m == NULL)
   1280   1.2  jmcneill 			break;
   1281   1.2  jmcneill 
   1282   1.1   hannken 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1283   1.1   hannken 		if (r == EAGAIN) {
   1284   1.1   hannken 			ifp->if_flags |= IFF_OACTIVE;
   1285  1.36  jdolecek 			m_freem(m);
   1286  1.36  jdolecek 			break;
   1287   1.1   hannken 		}
   1288   1.1   hannken 		if (r != 0)
   1289   1.1   hannken 			panic("enqueue_prep for a tx buffer");
   1290  1.36  jdolecek 
   1291  1.32  jdolecek 		r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1292  1.50  christos 		    txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   1293   1.1   hannken 		if (r != 0) {
   1294  1.36  jdolecek 			/* maybe just too fragmented */
   1295  1.36  jdolecek 			struct mbuf *newm;
   1296  1.36  jdolecek 
   1297  1.36  jdolecek 			newm = m_defrag(m, M_NOWAIT);
   1298  1.36  jdolecek 			if (newm == NULL) {
   1299  1.36  jdolecek 				aprint_error_dev(sc->sc_dev,
   1300  1.36  jdolecek 				    "m_defrag() failed\n");
   1301  1.36  jdolecek 				goto skip;
   1302  1.36  jdolecek 			}
   1303  1.36  jdolecek 
   1304  1.37  jdolecek 			m = newm;
   1305  1.36  jdolecek 			r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1306  1.50  christos 			    txq->txq_dmamaps[slot], m,
   1307  1.50  christos 			    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   1308  1.36  jdolecek 			if (r != 0) {
   1309  1.36  jdolecek 				aprint_error_dev(sc->sc_dev,
   1310  1.49   msaitoh 				    "tx dmamap load failed, error code %d\n",
   1311  1.36  jdolecek 				    r);
   1312  1.36  jdolecek skip:
   1313  1.37  jdolecek 				m_freem(m);
   1314  1.36  jdolecek 				virtio_enqueue_abort(vsc, vq, slot);
   1315  1.36  jdolecek 				continue;
   1316  1.36  jdolecek 			}
   1317   1.1   hannken 		}
   1318  1.36  jdolecek 
   1319  1.36  jdolecek 		/* This should actually never fail */
   1320   1.1   hannken 		r = virtio_enqueue_reserve(vsc, vq, slot,
   1321  1.43  yamaguch 					txq->txq_dmamaps[slot]->dm_nsegs + 1);
   1322   1.1   hannken 		if (r != 0) {
   1323  1.36  jdolecek 			aprint_error_dev(sc->sc_dev,
   1324  1.49   msaitoh 			    "virtio_enqueue_reserve failed, error code %d\n",
   1325  1.36  jdolecek 			    r);
   1326  1.32  jdolecek 			bus_dmamap_unload(virtio_dmat(vsc),
   1327  1.43  yamaguch 					  txq->txq_dmamaps[slot]);
   1328  1.36  jdolecek 			/* slot already freed by virtio_enqueue_reserve */
   1329  1.37  jdolecek 			m_freem(m);
   1330  1.36  jdolecek 			continue;
   1331   1.1   hannken 		}
   1332   1.7     ozaki 
   1333  1.43  yamaguch 		txq->txq_mbufs[slot] = m;
   1334   1.1   hannken 
   1335  1.43  yamaguch 		memset(&txq->txq_hdrs[slot], 0, sizeof(struct virtio_net_hdr));
   1336  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
   1337  1.50  christos 		    0, txq->txq_dmamaps[slot]->dm_mapsize,
   1338  1.50  christos 		    BUS_DMASYNC_PREWRITE);
   1339  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
   1340  1.50  christos 		    0, txq->txq_hdr_dmamaps[slot]->dm_mapsize,
   1341  1.50  christos 		    BUS_DMASYNC_PREWRITE);
   1342  1.43  yamaguch 		virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true);
   1343  1.43  yamaguch 		virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true);
   1344   1.1   hannken 		virtio_enqueue_commit(vsc, vq, slot, false);
   1345  1.36  jdolecek 
   1346   1.1   hannken 		queued++;
   1347  1.41   msaitoh 		bpf_mtap(ifp, m, BPF_D_OUT);
   1348   1.1   hannken 	}
   1349   1.1   hannken 
   1350   1.1   hannken 	if (queued > 0) {
   1351   1.1   hannken 		virtio_enqueue_commit(vsc, vq, -1, true);
   1352   1.1   hannken 		ifp->if_timer = 5;
   1353   1.1   hannken 	}
   1354  1.46  yamaguch }
   1355  1.46  yamaguch 
   1356  1.46  yamaguch static void
   1357  1.46  yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
   1358  1.46  yamaguch {
   1359  1.46  yamaguch 
   1360  1.46  yamaguch 	/*
   1361  1.46  yamaguch 	 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
   1362  1.46  yamaguch 	 */
   1363  1.46  yamaguch 	vioif_send_common_locked(ifp, txq, false);
   1364  1.46  yamaguch 
   1365  1.46  yamaguch }
   1366  1.46  yamaguch 
   1367  1.46  yamaguch static void
   1368  1.46  yamaguch vioif_start(struct ifnet *ifp)
   1369  1.46  yamaguch {
   1370  1.46  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1371  1.46  yamaguch 	struct vioif_txqueue *txq = &sc->sc_txq[0];
   1372  1.46  yamaguch 
   1373  1.46  yamaguch #ifdef VIOIF_MPSAFE
   1374  1.46  yamaguch 	KASSERT(if_is_mpsafe(ifp));
   1375  1.46  yamaguch #endif
   1376  1.46  yamaguch 
   1377  1.46  yamaguch 	mutex_enter(txq->txq_lock);
   1378  1.59  yamaguch 	vioif_start_locked(ifp, txq);
   1379  1.46  yamaguch 	mutex_exit(txq->txq_lock);
   1380  1.46  yamaguch }
   1381  1.46  yamaguch 
   1382  1.46  yamaguch static inline int
   1383  1.46  yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
   1384  1.46  yamaguch {
   1385  1.46  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1386  1.46  yamaguch 	u_int cpuid = cpu_index(curcpu());
   1387  1.46  yamaguch 
   1388  1.46  yamaguch 	return cpuid % sc->sc_act_nvq_pairs;
   1389  1.46  yamaguch }
   1390  1.46  yamaguch 
   1391  1.46  yamaguch static void
   1392  1.46  yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
   1393  1.46  yamaguch {
   1394  1.46  yamaguch 
   1395  1.46  yamaguch 	vioif_send_common_locked(ifp, txq, true);
   1396  1.46  yamaguch }
   1397  1.46  yamaguch 
   1398  1.46  yamaguch static int
   1399  1.46  yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
   1400  1.46  yamaguch {
   1401  1.46  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1402  1.46  yamaguch 	struct vioif_txqueue *txq;
   1403  1.46  yamaguch 	int qid;
   1404  1.46  yamaguch 
   1405  1.46  yamaguch 	qid = vioif_select_txqueue(ifp, m);
   1406  1.46  yamaguch 	txq = &sc->sc_txq[qid];
   1407  1.46  yamaguch 
   1408  1.46  yamaguch 	if (__predict_false(!pcq_put(txq->txq_intrq, m))) {
   1409  1.46  yamaguch 		m_freem(m);
   1410  1.46  yamaguch 		return ENOBUFS;
   1411  1.46  yamaguch 	}
   1412  1.46  yamaguch 
   1413  1.52   thorpej 	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
   1414  1.52   thorpej 	if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
   1415  1.46  yamaguch 	if (m->m_flags & M_MCAST)
   1416  1.52   thorpej 		if_statinc_ref(nsr, if_omcasts);
   1417  1.52   thorpej 	IF_STAT_PUTREF(ifp);
   1418  1.46  yamaguch 
   1419  1.46  yamaguch 	if (mutex_tryenter(txq->txq_lock)) {
   1420  1.59  yamaguch 		vioif_transmit_locked(ifp, txq);
   1421  1.46  yamaguch 		mutex_exit(txq->txq_lock);
   1422  1.46  yamaguch 	}
   1423  1.46  yamaguch 
   1424  1.46  yamaguch 	return 0;
   1425  1.46  yamaguch }
   1426  1.46  yamaguch 
   1427  1.46  yamaguch static void
   1428  1.46  yamaguch vioif_deferred_transmit(void *arg)
   1429  1.46  yamaguch {
   1430  1.46  yamaguch 	struct vioif_txqueue *txq = arg;
   1431  1.46  yamaguch 	struct virtio_softc *vsc = txq->txq_vq->vq_owner;
   1432  1.46  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1433  1.46  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1434   1.7     ozaki 
   1435  1.60  yamaguch 	mutex_enter(txq->txq_lock);
   1436  1.60  yamaguch 	vioif_send_common_locked(ifp, txq, true);
   1437  1.60  yamaguch 	mutex_exit(txq->txq_lock);
   1438   1.1   hannken }
   1439   1.1   hannken 
   1440   1.1   hannken static int
   1441   1.1   hannken vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
   1442   1.1   hannken {
   1443   1.1   hannken 	int s, r;
   1444   1.1   hannken 
   1445   1.1   hannken 	s = splnet();
   1446   1.1   hannken 
   1447   1.1   hannken 	r = ether_ioctl(ifp, cmd, data);
   1448   1.1   hannken 	if ((r == 0 && cmd == SIOCSIFFLAGS) ||
   1449   1.1   hannken 	    (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI))) {
   1450   1.1   hannken 		if (ifp->if_flags & IFF_RUNNING)
   1451   1.1   hannken 			r = vioif_rx_filter(ifp->if_softc);
   1452   1.1   hannken 		else
   1453   1.1   hannken 			r = 0;
   1454   1.1   hannken 	}
   1455   1.1   hannken 
   1456   1.1   hannken 	splx(s);
   1457   1.1   hannken 
   1458   1.1   hannken 	return r;
   1459   1.1   hannken }
   1460   1.1   hannken 
   1461   1.1   hannken void
   1462   1.1   hannken vioif_watchdog(struct ifnet *ifp)
   1463   1.1   hannken {
   1464   1.1   hannken 	struct vioif_softc *sc = ifp->if_softc;
   1465  1.46  yamaguch 	int i;
   1466   1.1   hannken 
   1467  1.46  yamaguch 	if (ifp->if_flags & IFF_RUNNING) {
   1468  1.55  yamaguch 		for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1469  1.55  yamaguch 			vioif_tx_queue_clear(&sc->sc_txq[i]);
   1470  1.55  yamaguch 		}
   1471  1.46  yamaguch 	}
   1472   1.1   hannken }
   1473   1.1   hannken 
   1474   1.1   hannken /*
   1475  1.39  dholland  * Receive implementation
   1476   1.1   hannken  */
   1477  1.39  dholland /* allocate and initialize a mbuf for receive */
   1478   1.1   hannken static int
   1479  1.46  yamaguch vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i)
   1480   1.1   hannken {
   1481  1.46  yamaguch 	struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
   1482   1.1   hannken 	struct mbuf *m;
   1483   1.1   hannken 	int r;
   1484   1.1   hannken 
   1485   1.1   hannken 	MGETHDR(m, M_DONTWAIT, MT_DATA);
   1486   1.1   hannken 	if (m == NULL)
   1487   1.1   hannken 		return ENOBUFS;
   1488   1.1   hannken 	MCLGET(m, M_DONTWAIT);
   1489   1.1   hannken 	if ((m->m_flags & M_EXT) == 0) {
   1490   1.1   hannken 		m_freem(m);
   1491   1.1   hannken 		return ENOBUFS;
   1492   1.1   hannken 	}
   1493  1.43  yamaguch 	rxq->rxq_mbufs[i] = m;
   1494   1.1   hannken 	m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
   1495  1.46  yamaguch 	r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1496  1.50  christos 	    rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT);
   1497   1.1   hannken 	if (r) {
   1498   1.1   hannken 		m_freem(m);
   1499  1.46  yamaguch 		rxq->rxq_mbufs[i] = NULL;
   1500   1.1   hannken 		return r;
   1501   1.1   hannken 	}
   1502   1.1   hannken 
   1503   1.1   hannken 	return 0;
   1504   1.1   hannken }
   1505   1.1   hannken 
   1506  1.39  dholland /* free a mbuf for receive */
   1507   1.1   hannken static void
   1508  1.46  yamaguch vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i)
   1509   1.1   hannken {
   1510  1.46  yamaguch 	struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
   1511  1.43  yamaguch 
   1512  1.46  yamaguch 	bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]);
   1513  1.43  yamaguch 	m_freem(rxq->rxq_mbufs[i]);
   1514  1.43  yamaguch 	rxq->rxq_mbufs[i] = NULL;
   1515   1.1   hannken }
   1516   1.1   hannken 
   1517  1.39  dholland /* add mbufs for all the empty receive slots */
   1518   1.1   hannken static void
   1519  1.46  yamaguch vioif_populate_rx_mbufs(struct vioif_rxqueue *rxq)
   1520   1.1   hannken {
   1521  1.43  yamaguch 
   1522  1.46  yamaguch 	mutex_enter(rxq->rxq_lock);
   1523  1.46  yamaguch 	vioif_populate_rx_mbufs_locked(rxq);
   1524  1.46  yamaguch 	mutex_exit(rxq->rxq_lock);
   1525  1.12     ozaki }
   1526  1.12     ozaki 
   1527  1.12     ozaki static void
   1528  1.46  yamaguch vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *rxq)
   1529  1.12     ozaki {
   1530  1.46  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1531  1.46  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1532  1.46  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1533   1.1   hannken 	int i, r, ndone = 0;
   1534   1.1   hannken 
   1535  1.46  yamaguch 	KASSERT(mutex_owned(rxq->rxq_lock));
   1536   1.7     ozaki 
   1537  1.43  yamaguch 	if (rxq->rxq_stopping)
   1538  1.12     ozaki 		return;
   1539   1.7     ozaki 
   1540   1.1   hannken 	for (i = 0; i < vq->vq_num; i++) {
   1541   1.1   hannken 		int slot;
   1542   1.1   hannken 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1543   1.1   hannken 		if (r == EAGAIN)
   1544   1.1   hannken 			break;
   1545   1.1   hannken 		if (r != 0)
   1546   1.1   hannken 			panic("enqueue_prep for rx buffers");
   1547  1.43  yamaguch 		if (rxq->rxq_mbufs[slot] == NULL) {
   1548  1.46  yamaguch 			r = vioif_add_rx_mbuf(rxq, slot);
   1549   1.1   hannken 			if (r != 0) {
   1550  1.50  christos 				aprint_error_dev(sc->sc_dev,
   1551  1.50  christos 				    "rx mbuf allocation failed, "
   1552  1.50  christos 				    "error code %d\n", r);
   1553   1.1   hannken 				break;
   1554   1.1   hannken 			}
   1555   1.1   hannken 		}
   1556   1.1   hannken 		r = virtio_enqueue_reserve(vsc, vq, slot,
   1557  1.50  christos 		    rxq->rxq_dmamaps[slot]->dm_nsegs + 1);
   1558   1.1   hannken 		if (r != 0) {
   1559  1.46  yamaguch 			vioif_free_rx_mbuf(rxq, slot);
   1560   1.1   hannken 			break;
   1561   1.1   hannken 		}
   1562  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
   1563  1.50  christos 		    0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_PREREAD);
   1564  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
   1565  1.50  christos 		    0, MCLBYTES, BUS_DMASYNC_PREREAD);
   1566  1.50  christos 		virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot],
   1567  1.50  christos 		    false);
   1568  1.43  yamaguch 		virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false);
   1569   1.1   hannken 		virtio_enqueue_commit(vsc, vq, slot, false);
   1570   1.1   hannken 		ndone++;
   1571   1.1   hannken 	}
   1572   1.1   hannken 	if (ndone > 0)
   1573   1.1   hannken 		virtio_enqueue_commit(vsc, vq, -1, true);
   1574   1.1   hannken }
   1575   1.1   hannken 
   1576  1.55  yamaguch static void
   1577  1.55  yamaguch vioif_rx_queue_clear(struct vioif_rxqueue *rxq)
   1578   1.1   hannken {
   1579  1.55  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1580  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1581  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1582  1.55  yamaguch 	u_int limit = UINT_MAX;
   1583  1.55  yamaguch 	bool more;
   1584   1.7     ozaki 
   1585  1.43  yamaguch 	KASSERT(rxq->rxq_stopping);
   1586   1.7     ozaki 
   1587  1.46  yamaguch 	mutex_enter(rxq->rxq_lock);
   1588  1.55  yamaguch 	for (;;) {
   1589  1.55  yamaguch 		more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
   1590  1.55  yamaguch 		if (more == false)
   1591  1.55  yamaguch 			break;
   1592  1.55  yamaguch 	}
   1593  1.46  yamaguch 	mutex_exit(rxq->rxq_lock);
   1594   1.7     ozaki }
   1595   1.7     ozaki 
   1596  1.39  dholland /* dequeue received packets */
   1597  1.55  yamaguch static bool
   1598  1.55  yamaguch vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   1599  1.55  yamaguch     struct vioif_rxqueue *rxq, u_int limit)
   1600   1.7     ozaki {
   1601  1.43  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1602   1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1603   1.1   hannken 	struct mbuf *m;
   1604   1.1   hannken 	int slot, len;
   1605  1.55  yamaguch 	bool more = false, dequeued = false;
   1606   1.1   hannken 
   1607  1.46  yamaguch 	KASSERT(mutex_owned(rxq->rxq_lock));
   1608   1.7     ozaki 
   1609  1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   1610  1.55  yamaguch 		return false;
   1611  1.55  yamaguch 
   1612  1.55  yamaguch 	for (;;) {
   1613  1.55  yamaguch 		if (limit-- == 0) {
   1614  1.55  yamaguch 			more = true;
   1615  1.55  yamaguch 			break;
   1616  1.55  yamaguch 		}
   1617  1.55  yamaguch 
   1618  1.55  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   1619  1.55  yamaguch 			break;
   1620  1.55  yamaguch 
   1621  1.55  yamaguch 		dequeued = true;
   1622  1.54  yamaguch 
   1623   1.1   hannken 		len -= sizeof(struct virtio_net_hdr);
   1624  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
   1625  1.50  christos 		    0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTREAD);
   1626  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
   1627  1.50  christos 		    0, MCLBYTES, BUS_DMASYNC_POSTREAD);
   1628  1.43  yamaguch 		m = rxq->rxq_mbufs[slot];
   1629   1.1   hannken 		KASSERT(m != NULL);
   1630  1.43  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]);
   1631  1.46  yamaguch 		rxq->rxq_mbufs[slot] = NULL;
   1632   1.1   hannken 		virtio_dequeue_commit(vsc, vq, slot);
   1633  1.24     ozaki 		m_set_rcvif(m, ifp);
   1634   1.1   hannken 		m->m_len = m->m_pkthdr.len = len;
   1635   1.7     ozaki 
   1636  1.46  yamaguch 		mutex_exit(rxq->rxq_lock);
   1637  1.22     ozaki 		if_percpuq_enqueue(ifp->if_percpuq, m);
   1638  1.46  yamaguch 		mutex_enter(rxq->rxq_lock);
   1639   1.7     ozaki 
   1640  1.43  yamaguch 		if (rxq->rxq_stopping)
   1641   1.7     ozaki 			break;
   1642   1.1   hannken 	}
   1643   1.3  christos 
   1644  1.55  yamaguch 	if (dequeued)
   1645  1.55  yamaguch 		softint_schedule(rxq->rxq_softint);
   1646  1.55  yamaguch 
   1647  1.55  yamaguch 	return more;
   1648   1.1   hannken }
   1649   1.1   hannken 
   1650   1.1   hannken /* rx interrupt; call _dequeue above and schedule a softint */
   1651   1.1   hannken static int
   1652  1.54  yamaguch vioif_rx_intr(void *arg)
   1653   1.1   hannken {
   1654  1.54  yamaguch 	struct vioif_rxqueue *rxq = arg;
   1655  1.55  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1656  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1657  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1658  1.55  yamaguch 	u_int limit;
   1659  1.55  yamaguch 	bool more;
   1660  1.55  yamaguch 
   1661  1.55  yamaguch 	limit = sc->sc_rx_intr_process_limit;
   1662  1.55  yamaguch 
   1663  1.55  yamaguch 	if (atomic_load_relaxed(&rxq->rxq_active) == true)
   1664  1.55  yamaguch 		return 1;
   1665   1.7     ozaki 
   1666  1.46  yamaguch 	mutex_enter(rxq->rxq_lock);
   1667   1.7     ozaki 
   1668  1.55  yamaguch 	if (!rxq->rxq_stopping) {
   1669  1.55  yamaguch 		rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl;
   1670  1.55  yamaguch 
   1671  1.55  yamaguch 		virtio_stop_vq_intr(vsc, vq);
   1672  1.55  yamaguch 		atomic_store_relaxed(&rxq->rxq_active, true);
   1673  1.55  yamaguch 
   1674  1.55  yamaguch 		more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
   1675  1.55  yamaguch 		if (more) {
   1676  1.55  yamaguch 			vioif_rx_sched_handle(sc, rxq);
   1677  1.55  yamaguch 		} else {
   1678  1.55  yamaguch 			atomic_store_relaxed(&rxq->rxq_active, false);
   1679  1.55  yamaguch 			virtio_start_vq_intr(vsc, vq);
   1680  1.55  yamaguch 		}
   1681  1.55  yamaguch 	}
   1682  1.55  yamaguch 
   1683  1.55  yamaguch 	mutex_exit(rxq->rxq_lock);
   1684  1.55  yamaguch 	return 1;
   1685  1.55  yamaguch }
   1686  1.55  yamaguch 
   1687  1.55  yamaguch static void
   1688  1.55  yamaguch vioif_rx_handle(void *xrxq)
   1689  1.55  yamaguch {
   1690  1.55  yamaguch 	struct vioif_rxqueue *rxq = xrxq;
   1691  1.55  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1692  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1693  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1694  1.55  yamaguch 	u_int limit;
   1695  1.55  yamaguch 	bool more;
   1696  1.55  yamaguch 
   1697  1.55  yamaguch 	limit = sc->sc_rx_process_limit;
   1698  1.55  yamaguch 
   1699  1.55  yamaguch 	mutex_enter(rxq->rxq_lock);
   1700   1.1   hannken 
   1701  1.55  yamaguch 	if (!rxq->rxq_stopping) {
   1702  1.55  yamaguch 		more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
   1703  1.55  yamaguch 		if (more) {
   1704  1.55  yamaguch 			vioif_rx_sched_handle(sc, rxq);
   1705  1.55  yamaguch 		} else {
   1706  1.55  yamaguch 			atomic_store_relaxed(&rxq->rxq_active, false);
   1707  1.55  yamaguch 			virtio_start_vq_intr(vsc, rxq->rxq_vq);
   1708  1.55  yamaguch 		}
   1709  1.55  yamaguch 	}
   1710   1.1   hannken 
   1711  1.46  yamaguch 	mutex_exit(rxq->rxq_lock);
   1712  1.55  yamaguch }
   1713  1.55  yamaguch 
   1714  1.55  yamaguch static void
   1715  1.55  yamaguch vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
   1716  1.55  yamaguch {
   1717  1.55  yamaguch 
   1718  1.55  yamaguch 	if (rxq->rxq_workqueue)
   1719  1.55  yamaguch 		vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work);
   1720  1.55  yamaguch 	else
   1721  1.55  yamaguch 		softint_schedule(rxq->rxq_handle_si);
   1722   1.1   hannken }
   1723   1.1   hannken 
   1724  1.39  dholland /* softint: enqueue receive requests for new incoming packets */
   1725   1.1   hannken static void
   1726   1.1   hannken vioif_rx_softint(void *arg)
   1727   1.1   hannken {
   1728  1.46  yamaguch 	struct vioif_rxqueue *rxq = arg;
   1729   1.1   hannken 
   1730  1.46  yamaguch 	vioif_populate_rx_mbufs(rxq);
   1731   1.1   hannken }
   1732   1.1   hannken 
   1733   1.1   hannken /* free all the mbufs; called from if_stop(disable) */
   1734   1.1   hannken static void
   1735  1.46  yamaguch vioif_rx_drain(struct vioif_rxqueue *rxq)
   1736   1.1   hannken {
   1737  1.43  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1738   1.1   hannken 	int i;
   1739   1.1   hannken 
   1740   1.1   hannken 	for (i = 0; i < vq->vq_num; i++) {
   1741  1.43  yamaguch 		if (rxq->rxq_mbufs[i] == NULL)
   1742   1.1   hannken 			continue;
   1743  1.46  yamaguch 		vioif_free_rx_mbuf(rxq, i);
   1744   1.1   hannken 	}
   1745   1.1   hannken }
   1746   1.1   hannken 
   1747   1.1   hannken /*
   1748   1.1   hannken  * Transmition implementation
   1749   1.1   hannken  */
   1750   1.1   hannken /* actual transmission is done in if_start */
   1751   1.1   hannken /* tx interrupt; dequeue and free mbufs */
   1752   1.1   hannken /*
   1753   1.1   hannken  * tx interrupt is actually disabled; this should be called upon
   1754   1.1   hannken  * tx vq full and watchdog
   1755   1.1   hannken  */
   1756  1.55  yamaguch 
   1757   1.1   hannken static int
   1758  1.54  yamaguch vioif_tx_intr(void *arg)
   1759   1.1   hannken {
   1760  1.54  yamaguch 	struct vioif_txqueue *txq = arg;
   1761  1.54  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1762   1.1   hannken 	struct virtio_softc *vsc = vq->vq_owner;
   1763  1.32  jdolecek 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1764  1.23     pooka 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1765  1.55  yamaguch 	bool more;
   1766  1.55  yamaguch 	u_int limit;
   1767  1.55  yamaguch 
   1768  1.55  yamaguch 	limit = sc->sc_tx_intr_process_limit;
   1769  1.55  yamaguch 
   1770  1.55  yamaguch 	if (atomic_load_relaxed(&txq->txq_active) == true)
   1771  1.55  yamaguch 		return 1;
   1772   1.7     ozaki 
   1773  1.46  yamaguch 	mutex_enter(txq->txq_lock);
   1774   1.7     ozaki 
   1775  1.55  yamaguch 	if (!txq->txq_stopping) {
   1776  1.55  yamaguch 		txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl;
   1777  1.55  yamaguch 
   1778  1.55  yamaguch 		virtio_stop_vq_intr(vsc, vq);
   1779  1.55  yamaguch 		atomic_store_relaxed(&txq->txq_active, true);
   1780  1.55  yamaguch 
   1781  1.55  yamaguch 		more = vioif_tx_deq_locked(sc, vsc, txq, limit);
   1782  1.55  yamaguch 		if (more) {
   1783  1.55  yamaguch 			vioif_tx_sched_handle(sc, txq);
   1784  1.55  yamaguch 		} else {
   1785  1.55  yamaguch 			atomic_store_relaxed(&txq->txq_active, false);
   1786   1.7     ozaki 
   1787  1.55  yamaguch 			/* for ALTQ */
   1788  1.55  yamaguch 			if (txq == &sc->sc_txq[0]) {
   1789  1.55  yamaguch 				if_schedule_deferred_start(ifp);
   1790  1.55  yamaguch 				ifp->if_flags &= ~IFF_OACTIVE;
   1791  1.55  yamaguch 			}
   1792  1.55  yamaguch 			softint_schedule(txq->txq_deferred_transmit);
   1793  1.55  yamaguch 
   1794  1.55  yamaguch 			virtio_start_vq_intr(vsc, vq);
   1795  1.55  yamaguch 		}
   1796  1.55  yamaguch 	}
   1797   1.7     ozaki 
   1798  1.46  yamaguch 	mutex_exit(txq->txq_lock);
   1799  1.46  yamaguch 
   1800  1.55  yamaguch 	return 1;
   1801  1.55  yamaguch }
   1802  1.55  yamaguch 
   1803  1.55  yamaguch static void
   1804  1.55  yamaguch vioif_tx_handle(void *xtxq)
   1805  1.55  yamaguch {
   1806  1.55  yamaguch 	struct vioif_txqueue *txq = xtxq;
   1807  1.55  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1808  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1809  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1810  1.55  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1811  1.55  yamaguch 	u_int limit;
   1812  1.55  yamaguch 	bool more;
   1813  1.55  yamaguch 
   1814  1.55  yamaguch 	limit = sc->sc_tx_process_limit;
   1815  1.55  yamaguch 
   1816  1.55  yamaguch 	mutex_enter(txq->txq_lock);
   1817  1.55  yamaguch 
   1818  1.55  yamaguch 	if (!txq->txq_stopping) {
   1819  1.55  yamaguch 		more = vioif_tx_deq_locked(sc, vsc, txq, limit);
   1820  1.55  yamaguch 		if (more) {
   1821  1.55  yamaguch 			vioif_tx_sched_handle(sc, txq);
   1822  1.55  yamaguch 		} else {
   1823  1.55  yamaguch 			atomic_store_relaxed(&txq->txq_active, false);
   1824  1.55  yamaguch 
   1825  1.55  yamaguch 			/* for ALTQ */
   1826  1.55  yamaguch 			if (txq == &sc->sc_txq[0]) {
   1827  1.55  yamaguch 				if_schedule_deferred_start(ifp);
   1828  1.55  yamaguch 				ifp->if_flags &= ~IFF_OACTIVE;
   1829  1.55  yamaguch 			}
   1830  1.55  yamaguch 			softint_schedule(txq->txq_deferred_transmit);
   1831  1.55  yamaguch 
   1832  1.55  yamaguch 			virtio_start_vq_intr(vsc, txq->txq_vq);
   1833  1.55  yamaguch 		}
   1834  1.46  yamaguch 	}
   1835  1.55  yamaguch 
   1836  1.55  yamaguch 	mutex_exit(txq->txq_lock);
   1837  1.55  yamaguch }
   1838  1.55  yamaguch 
   1839  1.55  yamaguch static void
   1840  1.55  yamaguch vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq)
   1841  1.55  yamaguch {
   1842  1.55  yamaguch 
   1843  1.55  yamaguch 	if (txq->txq_workqueue)
   1844  1.55  yamaguch 		vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work);
   1845  1.55  yamaguch 	else
   1846  1.55  yamaguch 		softint_schedule(txq->txq_handle_si);
   1847   1.7     ozaki }
   1848   1.7     ozaki 
   1849  1.55  yamaguch static void
   1850  1.55  yamaguch vioif_tx_queue_clear(struct vioif_txqueue *txq)
   1851   1.7     ozaki {
   1852  1.55  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1853   1.7     ozaki 	struct virtio_softc *vsc = vq->vq_owner;
   1854  1.32  jdolecek 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1855  1.55  yamaguch 	u_int limit = UINT_MAX;
   1856  1.55  yamaguch 	bool more;
   1857  1.55  yamaguch 
   1858  1.55  yamaguch 	mutex_enter(txq->txq_lock);
   1859  1.55  yamaguch 	for (;;) {
   1860  1.55  yamaguch 		more = vioif_tx_deq_locked(sc, vsc, txq, limit);
   1861  1.55  yamaguch 		if (more == false)
   1862  1.55  yamaguch 			break;
   1863  1.55  yamaguch 	}
   1864  1.55  yamaguch 	mutex_exit(txq->txq_lock);
   1865  1.55  yamaguch }
   1866  1.55  yamaguch 
   1867  1.55  yamaguch static bool
   1868  1.55  yamaguch vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   1869  1.55  yamaguch     struct vioif_txqueue *txq, u_int limit)
   1870  1.55  yamaguch {
   1871  1.55  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1872   1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1873   1.1   hannken 	struct mbuf *m;
   1874   1.1   hannken 	int slot, len;
   1875  1.55  yamaguch 	bool more = false;
   1876   1.1   hannken 
   1877  1.46  yamaguch 	KASSERT(mutex_owned(txq->txq_lock));
   1878   1.7     ozaki 
   1879  1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   1880  1.55  yamaguch 		return false;
   1881  1.55  yamaguch 
   1882  1.55  yamaguch 	for (;;) {
   1883  1.55  yamaguch 		if (limit-- == 0) {
   1884  1.55  yamaguch 			more = true;
   1885  1.55  yamaguch 			break;
   1886  1.55  yamaguch 		}
   1887  1.55  yamaguch 
   1888  1.55  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   1889  1.55  yamaguch 			break;
   1890  1.54  yamaguch 
   1891  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
   1892  1.50  christos 		    0, sizeof(struct virtio_net_hdr), BUS_DMASYNC_POSTWRITE);
   1893  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
   1894  1.50  christos 		    0, txq->txq_dmamaps[slot]->dm_mapsize,
   1895  1.50  christos 		    BUS_DMASYNC_POSTWRITE);
   1896  1.43  yamaguch 		m = txq->txq_mbufs[slot];
   1897  1.43  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]);
   1898  1.46  yamaguch 		txq->txq_mbufs[slot] = NULL;
   1899   1.1   hannken 		virtio_dequeue_commit(vsc, vq, slot);
   1900  1.52   thorpej 		if_statinc(ifp, if_opackets);
   1901   1.1   hannken 		m_freem(m);
   1902   1.1   hannken 	}
   1903   1.1   hannken 
   1904  1.55  yamaguch 	return more;
   1905   1.1   hannken }
   1906   1.1   hannken 
   1907   1.1   hannken /* free all the mbufs already put on vq; called from if_stop(disable) */
   1908   1.1   hannken static void
   1909  1.46  yamaguch vioif_tx_drain(struct vioif_txqueue *txq)
   1910   1.1   hannken {
   1911  1.43  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1912  1.46  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1913   1.1   hannken 	int i;
   1914   1.1   hannken 
   1915  1.43  yamaguch 	KASSERT(txq->txq_stopping);
   1916   1.7     ozaki 
   1917   1.1   hannken 	for (i = 0; i < vq->vq_num; i++) {
   1918  1.43  yamaguch 		if (txq->txq_mbufs[i] == NULL)
   1919   1.1   hannken 			continue;
   1920  1.43  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]);
   1921  1.43  yamaguch 		m_freem(txq->txq_mbufs[i]);
   1922  1.43  yamaguch 		txq->txq_mbufs[i] = NULL;
   1923   1.1   hannken 	}
   1924   1.1   hannken }
   1925   1.1   hannken 
   1926   1.1   hannken /*
   1927   1.1   hannken  * Control vq
   1928   1.1   hannken  */
   1929   1.1   hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
   1930  1.44  yamaguch static void
   1931  1.44  yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
   1932   1.1   hannken {
   1933  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1934   1.1   hannken 
   1935  1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   1936  1.43  yamaguch 	while (ctrlq->ctrlq_inuse != FREE)
   1937  1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   1938  1.43  yamaguch 	ctrlq->ctrlq_inuse = INUSE;
   1939  1.44  yamaguch 	ctrlq->ctrlq_owner = curlwp;
   1940  1.44  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   1941  1.44  yamaguch }
   1942  1.44  yamaguch 
   1943  1.44  yamaguch static void
   1944  1.44  yamaguch vioif_ctrl_release(struct vioif_softc *sc)
   1945  1.44  yamaguch {
   1946  1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1947  1.44  yamaguch 
   1948  1.44  yamaguch 	KASSERT(ctrlq->ctrlq_inuse != FREE);
   1949  1.44  yamaguch 	KASSERT(ctrlq->ctrlq_owner == curlwp);
   1950  1.44  yamaguch 
   1951  1.44  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   1952  1.44  yamaguch 	ctrlq->ctrlq_inuse = FREE;
   1953  1.46  yamaguch 	ctrlq->ctrlq_owner = NULL;
   1954  1.44  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   1955  1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   1956  1.44  yamaguch }
   1957  1.44  yamaguch 
   1958  1.44  yamaguch static int
   1959  1.44  yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
   1960  1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   1961  1.44  yamaguch {
   1962  1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1963  1.44  yamaguch 	int i, r, loaded;
   1964  1.44  yamaguch 
   1965  1.44  yamaguch 	loaded = 0;
   1966  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   1967  1.44  yamaguch 		r = bus_dmamap_load(virtio_dmat(vsc),
   1968  1.44  yamaguch 		    specs[i].dmamap, specs[i].buf, specs[i].bufsize,
   1969  1.48   msaitoh 		    NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   1970  1.44  yamaguch 		if (r) {
   1971  1.50  christos 			aprint_error_dev(sc->sc_dev, "control command dmamap"
   1972  1.50  christos 			    " load failed, error code %d\n", r);
   1973  1.44  yamaguch 			goto err;
   1974  1.44  yamaguch 		}
   1975  1.44  yamaguch 		loaded++;
   1976  1.44  yamaguch 
   1977  1.44  yamaguch 	}
   1978  1.44  yamaguch 
   1979  1.44  yamaguch 	return r;
   1980  1.44  yamaguch 
   1981  1.44  yamaguch err:
   1982  1.44  yamaguch 	for (i = 0; i < loaded; i++) {
   1983  1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   1984  1.44  yamaguch 	}
   1985  1.44  yamaguch 
   1986  1.44  yamaguch 	return r;
   1987  1.44  yamaguch }
   1988  1.44  yamaguch 
   1989  1.44  yamaguch static void
   1990  1.44  yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
   1991  1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   1992  1.44  yamaguch {
   1993  1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1994  1.44  yamaguch 	int i;
   1995  1.44  yamaguch 
   1996  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   1997  1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   1998  1.44  yamaguch 	}
   1999  1.44  yamaguch }
   2000  1.44  yamaguch 
   2001  1.44  yamaguch static int
   2002  1.44  yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
   2003  1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2004  1.44  yamaguch {
   2005  1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2006  1.44  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2007  1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2008  1.44  yamaguch 	int i, r, slot;
   2009  1.43  yamaguch 
   2010  1.44  yamaguch 	ctrlq->ctrlq_cmd->class = class;
   2011  1.43  yamaguch 	ctrlq->ctrlq_cmd->command = cmd;
   2012   1.1   hannken 
   2013  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
   2014  1.50  christos 	    0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
   2015  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2016  1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
   2017  1.50  christos 		    0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
   2018  1.44  yamaguch 	}
   2019  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
   2020  1.50  christos 	    0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
   2021   1.1   hannken 
   2022   1.1   hannken 	r = virtio_enqueue_prep(vsc, vq, &slot);
   2023   1.1   hannken 	if (r != 0)
   2024   1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2025  1.44  yamaguch 	r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
   2026   1.1   hannken 	if (r != 0)
   2027   1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2028  1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
   2029  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2030  1.44  yamaguch 		virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
   2031  1.44  yamaguch 	}
   2032  1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
   2033   1.1   hannken 	virtio_enqueue_commit(vsc, vq, slot, true);
   2034   1.1   hannken 
   2035   1.1   hannken 	/* wait for done */
   2036  1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2037  1.43  yamaguch 	while (ctrlq->ctrlq_inuse != DONE)
   2038  1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   2039  1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2040   1.1   hannken 	/* already dequeueued */
   2041   1.1   hannken 
   2042  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
   2043  1.50  christos 	    sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
   2044  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2045  1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
   2046  1.50  christos 		    specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
   2047  1.44  yamaguch 	}
   2048  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
   2049  1.50  christos 	    sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
   2050   1.1   hannken 
   2051  1.43  yamaguch 	if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
   2052   1.1   hannken 		r = 0;
   2053   1.1   hannken 	else {
   2054  1.50  christos 		aprint_error_dev(sc->sc_dev, "failed setting rx mode\n");
   2055   1.1   hannken 		r = EIO;
   2056   1.1   hannken 	}
   2057   1.1   hannken 
   2058  1.44  yamaguch 	return r;
   2059  1.44  yamaguch }
   2060  1.44  yamaguch 
   2061  1.44  yamaguch static int
   2062  1.44  yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
   2063  1.44  yamaguch {
   2064  1.44  yamaguch 	struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
   2065  1.44  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2066  1.44  yamaguch 	int r;
   2067  1.44  yamaguch 
   2068  1.44  yamaguch 	if (!sc->sc_has_ctrl)
   2069  1.44  yamaguch 		return ENOTSUP;
   2070  1.44  yamaguch 
   2071  1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2072  1.44  yamaguch 
   2073  1.44  yamaguch 	rx->onoff = onoff;
   2074  1.44  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
   2075  1.44  yamaguch 	specs[0].buf = rx;
   2076  1.44  yamaguch 	specs[0].bufsize = sizeof(*rx);
   2077  1.44  yamaguch 
   2078  1.44  yamaguch 	r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
   2079  1.44  yamaguch 	    specs, __arraycount(specs));
   2080   1.3  christos 
   2081  1.44  yamaguch 	vioif_ctrl_release(sc);
   2082   1.1   hannken 	return r;
   2083   1.1   hannken }
   2084   1.1   hannken 
   2085   1.1   hannken static int
   2086   1.1   hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
   2087   1.1   hannken {
   2088  1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
   2089   1.1   hannken }
   2090   1.1   hannken 
   2091   1.1   hannken static int
   2092   1.1   hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
   2093   1.1   hannken {
   2094  1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
   2095   1.1   hannken }
   2096   1.1   hannken 
   2097   1.1   hannken /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
   2098   1.1   hannken static int
   2099   1.1   hannken vioif_set_rx_filter(struct vioif_softc *sc)
   2100   1.1   hannken {
   2101  1.43  yamaguch 	/* filter already set in ctrlq->ctrlq_mac_tbl */
   2102  1.44  yamaguch 	struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
   2103  1.44  yamaguch 	struct vioif_ctrl_cmdspec specs[2];
   2104  1.44  yamaguch 	int nspecs = __arraycount(specs);
   2105  1.44  yamaguch 	int r;
   2106  1.44  yamaguch 
   2107  1.44  yamaguch 	mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
   2108  1.44  yamaguch 	mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
   2109   1.1   hannken 
   2110  1.32  jdolecek 	if (!sc->sc_has_ctrl)
   2111   1.1   hannken 		return ENOTSUP;
   2112   1.1   hannken 
   2113  1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2114   1.1   hannken 
   2115  1.44  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
   2116  1.44  yamaguch 	specs[0].buf = mac_tbl_uc;
   2117  1.44  yamaguch 	specs[0].bufsize = sizeof(*mac_tbl_uc)
   2118  1.44  yamaguch 	    + (ETHER_ADDR_LEN * mac_tbl_uc->nentries);
   2119  1.44  yamaguch 
   2120  1.44  yamaguch 	specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
   2121  1.44  yamaguch 	specs[1].buf = mac_tbl_mc;
   2122  1.44  yamaguch 	specs[1].bufsize = sizeof(*mac_tbl_mc)
   2123  1.44  yamaguch 	    + (ETHER_ADDR_LEN * mac_tbl_mc->nentries);
   2124   1.1   hannken 
   2125  1.44  yamaguch 	r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
   2126  1.44  yamaguch 	if (r != 0)
   2127   1.1   hannken 		goto out;
   2128   1.1   hannken 
   2129  1.44  yamaguch 	r = vioif_ctrl_send_command(sc,
   2130  1.44  yamaguch 	    VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
   2131  1.44  yamaguch 	    specs, nspecs);
   2132   1.1   hannken 
   2133  1.44  yamaguch 	vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
   2134   1.1   hannken 
   2135   1.1   hannken out:
   2136  1.44  yamaguch 	vioif_ctrl_release(sc);
   2137   1.3  christos 
   2138   1.1   hannken 	return r;
   2139   1.1   hannken }
   2140   1.1   hannken 
   2141  1.46  yamaguch static int
   2142  1.46  yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
   2143  1.46  yamaguch {
   2144  1.46  yamaguch 	struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
   2145  1.46  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2146  1.46  yamaguch 	int r;
   2147  1.46  yamaguch 
   2148  1.46  yamaguch 	if (!sc->sc_has_ctrl)
   2149  1.46  yamaguch 		return ENOTSUP;
   2150  1.46  yamaguch 
   2151  1.46  yamaguch 	if (nvq_pairs <= 1)
   2152  1.46  yamaguch 		return EINVAL;
   2153  1.46  yamaguch 
   2154  1.46  yamaguch 	vioif_ctrl_acquire(sc);
   2155  1.46  yamaguch 
   2156  1.46  yamaguch 	mq->virtqueue_pairs = nvq_pairs;
   2157  1.46  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
   2158  1.46  yamaguch 	specs[0].buf = mq;
   2159  1.46  yamaguch 	specs[0].bufsize = sizeof(*mq);
   2160  1.46  yamaguch 
   2161  1.46  yamaguch 	r = vioif_ctrl_send_command(sc,
   2162  1.46  yamaguch 	    VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
   2163  1.46  yamaguch 	    specs, __arraycount(specs));
   2164  1.46  yamaguch 
   2165  1.46  yamaguch 	vioif_ctrl_release(sc);
   2166  1.46  yamaguch 
   2167  1.46  yamaguch 	return r;
   2168  1.46  yamaguch }
   2169  1.46  yamaguch 
   2170   1.1   hannken /* ctrl vq interrupt; wake up the command issuer */
   2171   1.1   hannken static int
   2172  1.54  yamaguch vioif_ctrl_intr(void *arg)
   2173   1.1   hannken {
   2174  1.54  yamaguch 	struct vioif_ctrlqueue *ctrlq = arg;
   2175  1.54  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2176   1.1   hannken 	struct virtio_softc *vsc = vq->vq_owner;
   2177   1.1   hannken 	int r, slot;
   2178   1.1   hannken 
   2179  1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   2180  1.54  yamaguch 		return 0;
   2181  1.54  yamaguch 
   2182   1.1   hannken 	r = virtio_dequeue(vsc, vq, &slot, NULL);
   2183   1.1   hannken 	if (r == ENOENT)
   2184   1.1   hannken 		return 0;
   2185   1.1   hannken 	virtio_dequeue_commit(vsc, vq, slot);
   2186   1.1   hannken 
   2187  1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2188  1.43  yamaguch 	ctrlq->ctrlq_inuse = DONE;
   2189  1.43  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   2190  1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2191   1.1   hannken 
   2192   1.1   hannken 	return 1;
   2193   1.1   hannken }
   2194   1.1   hannken 
   2195   1.1   hannken /*
   2196   1.1   hannken  * If IFF_PROMISC requested,  set promiscuous
   2197   1.1   hannken  * If multicast filter small enough (<=MAXENTRIES) set rx filter
   2198   1.1   hannken  * If large multicast filter exist use ALLMULTI
   2199   1.1   hannken  */
   2200   1.1   hannken /*
   2201   1.1   hannken  * If setting rx filter fails fall back to ALLMULTI
   2202   1.1   hannken  * If ALLMULTI fails fall back to PROMISC
   2203   1.1   hannken  */
   2204   1.1   hannken static int
   2205   1.1   hannken vioif_rx_filter(struct vioif_softc *sc)
   2206   1.1   hannken {
   2207  1.48   msaitoh 	struct ethercom *ec = &sc->sc_ethercom;
   2208  1.48   msaitoh 	struct ifnet *ifp = &ec->ec_if;
   2209   1.1   hannken 	struct ether_multi *enm;
   2210   1.1   hannken 	struct ether_multistep step;
   2211  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2212   1.1   hannken 	int nentries;
   2213   1.1   hannken 	int promisc = 0, allmulti = 0, rxfilter = 0;
   2214   1.1   hannken 	int r;
   2215   1.1   hannken 
   2216  1.32  jdolecek 	if (!sc->sc_has_ctrl) {	/* no ctrl vq; always promisc */
   2217   1.1   hannken 		ifp->if_flags |= IFF_PROMISC;
   2218   1.1   hannken 		return 0;
   2219   1.1   hannken 	}
   2220   1.1   hannken 
   2221   1.1   hannken 	if (ifp->if_flags & IFF_PROMISC) {
   2222   1.1   hannken 		promisc = 1;
   2223   1.1   hannken 		goto set;
   2224   1.1   hannken 	}
   2225   1.1   hannken 
   2226   1.1   hannken 	nentries = -1;
   2227  1.48   msaitoh 	ETHER_LOCK(ec);
   2228  1.48   msaitoh 	ETHER_FIRST_MULTI(step, ec, enm);
   2229   1.1   hannken 	while (nentries++, enm != NULL) {
   2230   1.1   hannken 		if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
   2231   1.1   hannken 			allmulti = 1;
   2232  1.31     ozaki 			goto set_unlock;
   2233   1.1   hannken 		}
   2234  1.50  christos 		if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
   2235   1.1   hannken 			allmulti = 1;
   2236  1.31     ozaki 			goto set_unlock;
   2237   1.1   hannken 		}
   2238  1.43  yamaguch 		memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
   2239  1.50  christos 		    enm->enm_addrlo, ETHER_ADDR_LEN);
   2240   1.1   hannken 		ETHER_NEXT_MULTI(step, enm);
   2241   1.1   hannken 	}
   2242   1.1   hannken 	rxfilter = 1;
   2243   1.1   hannken 
   2244  1.31     ozaki set_unlock:
   2245  1.48   msaitoh 	ETHER_UNLOCK(ec);
   2246  1.30     ozaki 
   2247  1.31     ozaki set:
   2248   1.1   hannken 	if (rxfilter) {
   2249  1.43  yamaguch 		ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
   2250  1.43  yamaguch 		ctrlq->ctrlq_mac_tbl_mc->nentries = nentries;
   2251   1.1   hannken 		r = vioif_set_rx_filter(sc);
   2252   1.1   hannken 		if (r != 0) {
   2253   1.1   hannken 			rxfilter = 0;
   2254   1.1   hannken 			allmulti = 1; /* fallback */
   2255   1.1   hannken 		}
   2256   1.1   hannken 	} else {
   2257   1.1   hannken 		/* remove rx filter */
   2258  1.43  yamaguch 		ctrlq->ctrlq_mac_tbl_uc->nentries = 0;
   2259  1.43  yamaguch 		ctrlq->ctrlq_mac_tbl_mc->nentries = 0;
   2260   1.1   hannken 		r = vioif_set_rx_filter(sc);
   2261   1.1   hannken 		/* what to do on failure? */
   2262   1.1   hannken 	}
   2263   1.1   hannken 	if (allmulti) {
   2264   1.1   hannken 		r = vioif_set_allmulti(sc, true);
   2265   1.1   hannken 		if (r != 0) {
   2266   1.1   hannken 			allmulti = 0;
   2267   1.1   hannken 			promisc = 1; /* fallback */
   2268   1.1   hannken 		}
   2269   1.1   hannken 	} else {
   2270   1.1   hannken 		r = vioif_set_allmulti(sc, false);
   2271   1.1   hannken 		/* what to do on failure? */
   2272   1.1   hannken 	}
   2273   1.1   hannken 	if (promisc) {
   2274   1.1   hannken 		r = vioif_set_promisc(sc, true);
   2275   1.1   hannken 	} else {
   2276   1.1   hannken 		r = vioif_set_promisc(sc, false);
   2277   1.1   hannken 	}
   2278   1.1   hannken 
   2279   1.1   hannken 	return r;
   2280   1.1   hannken }
   2281   1.1   hannken 
   2282  1.34     ozaki static bool
   2283  1.34     ozaki vioif_is_link_up(struct vioif_softc *sc)
   2284  1.34     ozaki {
   2285  1.34     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   2286  1.34     ozaki 	uint16_t status;
   2287  1.34     ozaki 
   2288  1.34     ozaki 	if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
   2289  1.34     ozaki 		status = virtio_read_device_config_2(vsc,
   2290  1.34     ozaki 		    VIRTIO_NET_CONFIG_STATUS);
   2291  1.34     ozaki 	else
   2292  1.34     ozaki 		status = VIRTIO_NET_S_LINK_UP;
   2293  1.34     ozaki 
   2294  1.34     ozaki 	return ((status & VIRTIO_NET_S_LINK_UP) != 0);
   2295  1.34     ozaki }
   2296  1.34     ozaki 
   2297  1.34     ozaki /* change link status */
   2298  1.34     ozaki static void
   2299  1.34     ozaki vioif_update_link_status(struct vioif_softc *sc)
   2300  1.34     ozaki {
   2301  1.34     ozaki 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2302  1.46  yamaguch 	struct vioif_txqueue *txq;
   2303  1.34     ozaki 	bool active, changed;
   2304  1.46  yamaguch 	int link, i;
   2305  1.34     ozaki 
   2306  1.34     ozaki 	active = vioif_is_link_up(sc);
   2307  1.34     ozaki 	changed = false;
   2308  1.34     ozaki 
   2309  1.34     ozaki 	if (active) {
   2310  1.34     ozaki 		if (!sc->sc_link_active)
   2311  1.34     ozaki 			changed = true;
   2312  1.34     ozaki 
   2313  1.34     ozaki 		link = LINK_STATE_UP;
   2314  1.34     ozaki 		sc->sc_link_active = true;
   2315  1.34     ozaki 	} else {
   2316  1.34     ozaki 		if (sc->sc_link_active)
   2317  1.34     ozaki 			changed = true;
   2318  1.34     ozaki 
   2319  1.34     ozaki 		link = LINK_STATE_DOWN;
   2320  1.34     ozaki 		sc->sc_link_active = false;
   2321  1.34     ozaki 	}
   2322  1.34     ozaki 
   2323  1.43  yamaguch 	if (changed) {
   2324  1.46  yamaguch 		for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   2325  1.46  yamaguch 			txq = &sc->sc_txq[i];
   2326  1.46  yamaguch 
   2327  1.46  yamaguch 			mutex_enter(txq->txq_lock);
   2328  1.46  yamaguch 			txq->txq_link_active = sc->sc_link_active;
   2329  1.46  yamaguch 			mutex_exit(txq->txq_lock);
   2330  1.46  yamaguch 		}
   2331  1.43  yamaguch 
   2332  1.34     ozaki 		if_link_state_change(ifp, link);
   2333  1.43  yamaguch 	}
   2334  1.34     ozaki }
   2335  1.34     ozaki 
   2336  1.34     ozaki static int
   2337  1.34     ozaki vioif_config_change(struct virtio_softc *vsc)
   2338  1.34     ozaki {
   2339  1.34     ozaki 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2340  1.34     ozaki 
   2341  1.34     ozaki 	softint_schedule(sc->sc_ctl_softint);
   2342  1.34     ozaki 	return 0;
   2343  1.34     ozaki }
   2344  1.34     ozaki 
   2345  1.34     ozaki static void
   2346  1.34     ozaki vioif_ctl_softint(void *arg)
   2347  1.34     ozaki {
   2348  1.34     ozaki 	struct vioif_softc *sc = arg;
   2349  1.34     ozaki 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2350  1.34     ozaki 
   2351  1.34     ozaki 	vioif_update_link_status(sc);
   2352  1.34     ozaki 	vioif_start(ifp);
   2353  1.34     ozaki }
   2354  1.34     ozaki 
   2355  1.55  yamaguch static struct workqueue *
   2356  1.55  yamaguch vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
   2357  1.55  yamaguch {
   2358  1.55  yamaguch 	struct workqueue *wq;
   2359  1.55  yamaguch 	int error;
   2360  1.55  yamaguch 
   2361  1.55  yamaguch 	error = workqueue_create(&wq, name, vioif_workq_work, NULL,
   2362  1.55  yamaguch 	    prio, ipl, flags);
   2363  1.55  yamaguch 
   2364  1.55  yamaguch 	if (error)
   2365  1.55  yamaguch 		return NULL;
   2366  1.55  yamaguch 
   2367  1.55  yamaguch 	return wq;
   2368  1.55  yamaguch }
   2369  1.55  yamaguch 
   2370  1.55  yamaguch static void
   2371  1.55  yamaguch vioif_workq_destroy(struct workqueue *wq)
   2372  1.55  yamaguch {
   2373  1.55  yamaguch 
   2374  1.55  yamaguch 	workqueue_destroy(wq);
   2375  1.55  yamaguch }
   2376  1.55  yamaguch 
   2377  1.55  yamaguch static void
   2378  1.55  yamaguch vioif_workq_work(struct work *wk, void *context)
   2379  1.55  yamaguch {
   2380  1.55  yamaguch 	struct vioif_work *work;
   2381  1.55  yamaguch 
   2382  1.55  yamaguch 	work = container_of(wk, struct vioif_work, cookie);
   2383  1.55  yamaguch 
   2384  1.55  yamaguch 	atomic_store_relaxed(&work->added, 0);
   2385  1.55  yamaguch 	work->func(work->arg);
   2386  1.55  yamaguch }
   2387  1.55  yamaguch 
   2388  1.55  yamaguch static void
   2389  1.55  yamaguch vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
   2390  1.55  yamaguch {
   2391  1.55  yamaguch 
   2392  1.55  yamaguch 	memset(work, 0, sizeof(*work));
   2393  1.55  yamaguch 	work->func = func;
   2394  1.55  yamaguch 	work->arg = arg;
   2395  1.55  yamaguch }
   2396  1.55  yamaguch 
   2397  1.55  yamaguch static void
   2398  1.55  yamaguch vioif_work_add(struct workqueue *wq, struct vioif_work *work)
   2399  1.55  yamaguch {
   2400  1.55  yamaguch 
   2401  1.55  yamaguch 	if (atomic_load_relaxed(&work->added) != 0)
   2402  1.55  yamaguch 		return;
   2403  1.55  yamaguch 
   2404  1.55  yamaguch 	atomic_store_relaxed(&work->added, 1);
   2405  1.55  yamaguch 	kpreempt_disable();
   2406  1.55  yamaguch 	workqueue_enqueue(wq, &work->cookie, NULL);
   2407  1.55  yamaguch 	kpreempt_enable();
   2408  1.55  yamaguch }
   2409  1.55  yamaguch 
   2410  1.55  yamaguch static void
   2411  1.55  yamaguch vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
   2412  1.55  yamaguch {
   2413  1.55  yamaguch 
   2414  1.55  yamaguch 	workqueue_wait(wq, &work->cookie);
   2415  1.55  yamaguch }
   2416  1.55  yamaguch 
   2417  1.55  yamaguch static int
   2418  1.55  yamaguch vioif_setup_sysctl(struct vioif_softc *sc)
   2419  1.55  yamaguch {
   2420  1.55  yamaguch 	const char *devname;
   2421  1.55  yamaguch 	struct sysctllog **log;
   2422  1.55  yamaguch 	const struct sysctlnode *rnode, *rxnode, *txnode;
   2423  1.55  yamaguch 	int error;
   2424  1.55  yamaguch 
   2425  1.55  yamaguch 	log = &sc->sc_sysctllog;
   2426  1.55  yamaguch 	devname = device_xname(sc->sc_dev);
   2427  1.55  yamaguch 
   2428  1.55  yamaguch 	error = sysctl_createv(log, 0, NULL, &rnode,
   2429  1.55  yamaguch 	    0, CTLTYPE_NODE, devname,
   2430  1.55  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings"),
   2431  1.55  yamaguch 	    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
   2432  1.55  yamaguch 	if (error)
   2433  1.55  yamaguch 		goto out;
   2434  1.55  yamaguch 
   2435  1.55  yamaguch 	error = sysctl_createv(log, 0, &rnode, NULL,
   2436  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
   2437  1.55  yamaguch 	    SYSCTL_DESCR("Use workqueue for packet processing"),
   2438  1.55  yamaguch 	    NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
   2439  1.55  yamaguch 	if (error)
   2440  1.55  yamaguch 		goto out;
   2441  1.55  yamaguch 
   2442  1.55  yamaguch 	error = sysctl_createv(log, 0, &rnode, &rxnode,
   2443  1.55  yamaguch 	    0, CTLTYPE_NODE, "rx",
   2444  1.55  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Rx"),
   2445  1.55  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
   2446  1.55  yamaguch 	if (error)
   2447  1.55  yamaguch 		goto out;
   2448  1.55  yamaguch 
   2449  1.55  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
   2450  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
   2451  1.55  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
   2452  1.55  yamaguch 	    NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
   2453  1.55  yamaguch 	if (error)
   2454  1.55  yamaguch 		goto out;
   2455  1.55  yamaguch 
   2456  1.55  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
   2457  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
   2458  1.55  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
   2459  1.55  yamaguch 	    NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
   2460  1.55  yamaguch 	if (error)
   2461  1.55  yamaguch 		goto out;
   2462  1.55  yamaguch 
   2463  1.55  yamaguch 	error = sysctl_createv(log, 0, &rnode, &txnode,
   2464  1.55  yamaguch 	    0, CTLTYPE_NODE, "tx",
   2465  1.55  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Tx"),
   2466  1.55  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
   2467  1.55  yamaguch 	if (error)
   2468  1.55  yamaguch 		goto out;
   2469  1.55  yamaguch 
   2470  1.55  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
   2471  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
   2472  1.55  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
   2473  1.55  yamaguch 	    NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
   2474  1.55  yamaguch 	if (error)
   2475  1.55  yamaguch 		goto out;
   2476  1.55  yamaguch 
   2477  1.55  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
   2478  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
   2479  1.55  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
   2480  1.55  yamaguch 	    NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
   2481  1.55  yamaguch 
   2482  1.55  yamaguch out:
   2483  1.55  yamaguch 	if (error)
   2484  1.55  yamaguch 		sysctl_teardown(log);
   2485  1.55  yamaguch 
   2486  1.55  yamaguch 	return error;
   2487  1.55  yamaguch }
   2488  1.55  yamaguch 
   2489  1.26  pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
   2490  1.48   msaitoh 
   2491  1.26  pgoyette #ifdef _MODULE
   2492  1.26  pgoyette #include "ioconf.c"
   2493  1.26  pgoyette #endif
   2494  1.48   msaitoh 
   2495  1.48   msaitoh static int
   2496  1.26  pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
   2497  1.26  pgoyette {
   2498  1.26  pgoyette 	int error = 0;
   2499  1.48   msaitoh 
   2500  1.26  pgoyette #ifdef _MODULE
   2501  1.26  pgoyette 	switch (cmd) {
   2502  1.26  pgoyette 	case MODULE_CMD_INIT:
   2503  1.48   msaitoh 		error = config_init_component(cfdriver_ioconf_if_vioif,
   2504  1.48   msaitoh 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2505  1.26  pgoyette 		break;
   2506  1.26  pgoyette 	case MODULE_CMD_FINI:
   2507  1.26  pgoyette 		error = config_fini_component(cfdriver_ioconf_if_vioif,
   2508  1.26  pgoyette 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2509  1.26  pgoyette 		break;
   2510  1.26  pgoyette 	default:
   2511  1.26  pgoyette 		error = ENOTTY;
   2512  1.48   msaitoh 		break;
   2513  1.26  pgoyette 	}
   2514  1.26  pgoyette #endif
   2515  1.48   msaitoh 
   2516  1.26  pgoyette 	return error;
   2517  1.26  pgoyette }
   2518