Home | History | Annotate | Line # | Download | only in pci
if_vioif.c revision 1.79
      1  1.79       uwe /*	$NetBSD: if_vioif.c,v 1.79 2022/04/13 10:42:12 uwe Exp $	*/
      2   1.1   hannken 
      3   1.1   hannken /*
      4  1.66   reinoud  * Copyright (c) 2020 The NetBSD Foundation, Inc.
      5   1.1   hannken  * Copyright (c) 2010 Minoura Makoto.
      6   1.1   hannken  * All rights reserved.
      7   1.1   hannken  *
      8   1.1   hannken  * Redistribution and use in source and binary forms, with or without
      9   1.1   hannken  * modification, are permitted provided that the following conditions
     10   1.1   hannken  * are met:
     11   1.1   hannken  * 1. Redistributions of source code must retain the above copyright
     12   1.1   hannken  *    notice, this list of conditions and the following disclaimer.
     13   1.1   hannken  * 2. Redistributions in binary form must reproduce the above copyright
     14   1.1   hannken  *    notice, this list of conditions and the following disclaimer in the
     15   1.1   hannken  *    documentation and/or other materials provided with the distribution.
     16   1.1   hannken  *
     17   1.1   hannken  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     18   1.1   hannken  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     19   1.1   hannken  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     20   1.1   hannken  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     21   1.1   hannken  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     22   1.1   hannken  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23   1.1   hannken  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24   1.1   hannken  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25   1.1   hannken  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     26   1.1   hannken  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27   1.1   hannken  */
     28   1.1   hannken 
     29   1.1   hannken #include <sys/cdefs.h>
     30  1.79       uwe __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.79 2022/04/13 10:42:12 uwe Exp $");
     31  1.15     ozaki 
     32  1.15     ozaki #ifdef _KERNEL_OPT
     33  1.15     ozaki #include "opt_net_mpsafe.h"
     34  1.15     ozaki #endif
     35   1.1   hannken 
     36   1.1   hannken #include <sys/param.h>
     37   1.1   hannken #include <sys/systm.h>
     38   1.1   hannken #include <sys/kernel.h>
     39  1.55  yamaguch #include <sys/atomic.h>
     40   1.1   hannken #include <sys/bus.h>
     41   1.1   hannken #include <sys/condvar.h>
     42   1.1   hannken #include <sys/device.h>
     43  1.63  yamaguch #include <sys/evcnt.h>
     44   1.1   hannken #include <sys/intr.h>
     45   1.1   hannken #include <sys/kmem.h>
     46   1.1   hannken #include <sys/mbuf.h>
     47   1.1   hannken #include <sys/mutex.h>
     48   1.1   hannken #include <sys/sockio.h>
     49  1.71  yamaguch #include <sys/syslog.h>
     50  1.12     ozaki #include <sys/cpu.h>
     51  1.26  pgoyette #include <sys/module.h>
     52  1.46  yamaguch #include <sys/pcq.h>
     53  1.55  yamaguch #include <sys/workqueue.h>
     54   1.1   hannken 
     55   1.1   hannken #include <dev/pci/virtioreg.h>
     56   1.1   hannken #include <dev/pci/virtiovar.h>
     57   1.1   hannken 
     58   1.1   hannken #include <net/if.h>
     59  1.74  yamaguch #include <net/if_dl.h>
     60   1.1   hannken #include <net/if_media.h>
     61   1.1   hannken #include <net/if_ether.h>
     62   1.1   hannken 
     63   1.1   hannken #include <net/bpf.h>
     64   1.1   hannken 
     65  1.26  pgoyette #include "ioconf.h"
     66   1.1   hannken 
     67   1.7     ozaki #ifdef NET_MPSAFE
     68   1.7     ozaki #define VIOIF_MPSAFE	1
     69  1.46  yamaguch #define VIOIF_MULTIQ	1
     70   1.7     ozaki #endif
     71   1.7     ozaki 
     72   1.1   hannken /*
     73   1.1   hannken  * if_vioifreg.h:
     74   1.1   hannken  */
     75   1.1   hannken /* Configuration registers */
     76  1.66   reinoud #define VIRTIO_NET_CONFIG_MAC		 0 /* 8bit x 6byte */
     77  1.66   reinoud #define VIRTIO_NET_CONFIG_STATUS	 6 /* 16bit */
     78  1.66   reinoud #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS	 8 /* 16bit */
     79  1.66   reinoud #define VIRTIO_NET_CONFIG_MTU		10 /* 16bit */
     80   1.1   hannken 
     81   1.1   hannken /* Feature bits */
     82  1.46  yamaguch #define VIRTIO_NET_F_CSUM		__BIT(0)
     83  1.46  yamaguch #define VIRTIO_NET_F_GUEST_CSUM		__BIT(1)
     84  1.46  yamaguch #define VIRTIO_NET_F_MAC		__BIT(5)
     85  1.46  yamaguch #define VIRTIO_NET_F_GSO		__BIT(6)
     86  1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO4		__BIT(7)
     87  1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO6		__BIT(8)
     88  1.46  yamaguch #define VIRTIO_NET_F_GUEST_ECN		__BIT(9)
     89  1.46  yamaguch #define VIRTIO_NET_F_GUEST_UFO		__BIT(10)
     90  1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO4		__BIT(11)
     91  1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO6		__BIT(12)
     92  1.46  yamaguch #define VIRTIO_NET_F_HOST_ECN		__BIT(13)
     93  1.46  yamaguch #define VIRTIO_NET_F_HOST_UFO		__BIT(14)
     94  1.46  yamaguch #define VIRTIO_NET_F_MRG_RXBUF		__BIT(15)
     95  1.46  yamaguch #define VIRTIO_NET_F_STATUS		__BIT(16)
     96  1.46  yamaguch #define VIRTIO_NET_F_CTRL_VQ		__BIT(17)
     97  1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX		__BIT(18)
     98  1.46  yamaguch #define VIRTIO_NET_F_CTRL_VLAN		__BIT(19)
     99  1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA	__BIT(20)
    100  1.46  yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE	__BIT(21)
    101  1.46  yamaguch #define VIRTIO_NET_F_MQ			__BIT(22)
    102  1.78  yamaguch #define VIRTIO_NET_F_CTRL_MAC_ADDR 	__BIT(23)
    103   1.1   hannken 
    104  1.79       uwe #define VIRTIO_NET_FLAG_BITS			\
    105  1.79       uwe 	VIRTIO_COMMON_FLAG_BITS			\
    106  1.79       uwe 	"b\x17" "CTRL_MAC\0"			\
    107  1.79       uwe 	"b\x16" "MQ\0"				\
    108  1.79       uwe 	"b\x15" "GUEST_ANNOUNCE\0"		\
    109  1.79       uwe 	"b\x14" "CTRL_RX_EXTRA\0"		\
    110  1.79       uwe 	"b\x13" "CTRL_VLAN\0"			\
    111  1.79       uwe 	"b\x12" "CTRL_RX\0"			\
    112  1.79       uwe 	"b\x11" "CTRL_VQ\0"			\
    113  1.79       uwe 	"b\x10" "STATUS\0"			\
    114  1.79       uwe 	"b\x0f" "MRG_RXBUF\0"			\
    115  1.79       uwe 	"b\x0e" "HOST_UFO\0"			\
    116  1.79       uwe 	"b\x0d" "HOST_ECN\0"			\
    117  1.79       uwe 	"b\x0c" "HOST_TSO6\0"			\
    118  1.79       uwe 	"b\x0b" "HOST_TSO4\0"			\
    119  1.79       uwe 	"b\x0a" "GUEST_UFO\0"			\
    120  1.79       uwe 	"b\x09" "GUEST_ECN\0"			\
    121  1.79       uwe 	"b\x08" "GUEST_TSO6\0"			\
    122  1.79       uwe 	"b\x07" "GUEST_TSO4\0"			\
    123  1.79       uwe 	"b\x06" "GSO\0"				\
    124  1.79       uwe 	"b\x05" "MAC\0"				\
    125  1.79       uwe 	"b\x01" "GUEST_CSUM\0"			\
    126  1.79       uwe 	"b\x00" "CSUM\0"
    127  1.18  christos 
    128   1.1   hannken /* Status */
    129   1.1   hannken #define VIRTIO_NET_S_LINK_UP	1
    130   1.1   hannken 
    131   1.1   hannken /* Packet header structure */
    132   1.1   hannken struct virtio_net_hdr {
    133   1.1   hannken 	uint8_t		flags;
    134   1.1   hannken 	uint8_t		gso_type;
    135   1.1   hannken 	uint16_t	hdr_len;
    136   1.1   hannken 	uint16_t	gso_size;
    137   1.1   hannken 	uint16_t	csum_start;
    138   1.1   hannken 	uint16_t	csum_offset;
    139  1.66   reinoud 
    140  1.66   reinoud 	uint16_t	num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */
    141   1.1   hannken } __packed;
    142   1.1   hannken 
    143   1.1   hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM	1 /* flags */
    144   1.1   hannken #define VIRTIO_NET_HDR_GSO_NONE		0 /* gso_type */
    145   1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV4	1 /* gso_type */
    146   1.1   hannken #define VIRTIO_NET_HDR_GSO_UDP		3 /* gso_type */
    147   1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV6	4 /* gso_type */
    148   1.1   hannken #define VIRTIO_NET_HDR_GSO_ECN		0x80 /* gso_type, |'ed */
    149   1.1   hannken 
    150   1.1   hannken #define VIRTIO_NET_MAX_GSO_LEN		(65536+ETHER_HDR_LEN)
    151   1.1   hannken 
    152   1.1   hannken /* Control virtqueue */
    153   1.1   hannken struct virtio_net_ctrl_cmd {
    154   1.1   hannken 	uint8_t	class;
    155   1.1   hannken 	uint8_t	command;
    156   1.1   hannken } __packed;
    157   1.1   hannken #define VIRTIO_NET_CTRL_RX		0
    158   1.1   hannken # define VIRTIO_NET_CTRL_RX_PROMISC	0
    159   1.1   hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI	1
    160   1.1   hannken 
    161   1.1   hannken #define VIRTIO_NET_CTRL_MAC		1
    162   1.1   hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET	0
    163  1.74  yamaguch # define  VIRTIO_NET_CTRL_MAC_ADDR_SET	1
    164   1.1   hannken 
    165   1.1   hannken #define VIRTIO_NET_CTRL_VLAN		2
    166   1.1   hannken # define VIRTIO_NET_CTRL_VLAN_ADD	0
    167   1.1   hannken # define VIRTIO_NET_CTRL_VLAN_DEL	1
    168   1.1   hannken 
    169  1.46  yamaguch #define VIRTIO_NET_CTRL_MQ			4
    170  1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET	0
    171  1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN	1
    172  1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX	0x8000
    173  1.46  yamaguch 
    174   1.1   hannken struct virtio_net_ctrl_status {
    175   1.1   hannken 	uint8_t	ack;
    176   1.1   hannken } __packed;
    177   1.1   hannken #define VIRTIO_NET_OK			0
    178   1.1   hannken #define VIRTIO_NET_ERR			1
    179   1.1   hannken 
    180   1.1   hannken struct virtio_net_ctrl_rx {
    181   1.1   hannken 	uint8_t	onoff;
    182   1.1   hannken } __packed;
    183   1.1   hannken 
    184   1.1   hannken struct virtio_net_ctrl_mac_tbl {
    185   1.1   hannken 	uint32_t nentries;
    186   1.1   hannken 	uint8_t macs[][ETHER_ADDR_LEN];
    187   1.1   hannken } __packed;
    188   1.1   hannken 
    189  1.74  yamaguch struct virtio_net_ctrl_mac_addr {
    190  1.74  yamaguch 	uint8_t mac[ETHER_ADDR_LEN];
    191  1.74  yamaguch } __packed;
    192  1.74  yamaguch 
    193   1.1   hannken struct virtio_net_ctrl_vlan {
    194   1.1   hannken 	uint16_t id;
    195   1.1   hannken } __packed;
    196   1.1   hannken 
    197  1.46  yamaguch struct virtio_net_ctrl_mq {
    198  1.46  yamaguch 	uint16_t virtqueue_pairs;
    199  1.46  yamaguch } __packed;
    200  1.46  yamaguch 
    201   1.1   hannken /*
    202   1.1   hannken  * if_vioifvar.h:
    203   1.1   hannken  */
    204  1.43  yamaguch 
    205  1.43  yamaguch /*
    206  1.43  yamaguch  * Locking notes:
    207  1.43  yamaguch  * + a field in vioif_txqueue is protected by txq_lock (a spin mutex), and
    208  1.57  yamaguch  *   a field in vioif_rxqueue is protected by rxq_lock (a spin mutex).
    209  1.43  yamaguch  *      - more than one lock cannot be held at onece
    210  1.43  yamaguch  * + ctrlq_inuse is protected by ctrlq_wait_lock.
    211  1.43  yamaguch  *      - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
    212  1.43  yamaguch  *      - txq_lock or rxq_lock cannot be held along with ctrlq_wait_lock
    213  1.62  yamaguch  * + fields in vioif_softc except queues are protected by
    214  1.62  yamaguch  *   sc->sc_lock(an adaptive mutex)
    215  1.62  yamaguch  *      - the lock is held before acquisition of other locks
    216  1.43  yamaguch  */
    217  1.43  yamaguch 
    218  1.66   reinoud struct vioif_ctrl_cmdspec {
    219  1.66   reinoud 	bus_dmamap_t	dmamap;
    220  1.66   reinoud 	void		*buf;
    221  1.66   reinoud 	bus_size_t	bufsize;
    222  1.66   reinoud };
    223  1.66   reinoud 
    224  1.55  yamaguch struct vioif_work {
    225  1.55  yamaguch 	struct work	 cookie;
    226  1.55  yamaguch 	void		(*func)(void *);
    227  1.55  yamaguch 	void		*arg;
    228  1.55  yamaguch 	unsigned int	 added;
    229  1.55  yamaguch };
    230  1.55  yamaguch 
    231  1.43  yamaguch struct vioif_txqueue {
    232  1.43  yamaguch 	kmutex_t		*txq_lock;	/* lock for tx operations */
    233  1.43  yamaguch 
    234  1.43  yamaguch 	struct virtqueue	*txq_vq;
    235  1.43  yamaguch 	bool			txq_stopping;
    236  1.43  yamaguch 	bool			txq_link_active;
    237  1.46  yamaguch 	pcq_t			*txq_intrq;
    238  1.43  yamaguch 
    239  1.43  yamaguch 	struct virtio_net_hdr	*txq_hdrs;
    240  1.43  yamaguch 	bus_dmamap_t		*txq_hdr_dmamaps;
    241  1.43  yamaguch 
    242  1.43  yamaguch 	struct mbuf		**txq_mbufs;
    243  1.43  yamaguch 	bus_dmamap_t		*txq_dmamaps;
    244  1.46  yamaguch 
    245  1.46  yamaguch 	void			*txq_deferred_transmit;
    246  1.55  yamaguch 	void			*txq_handle_si;
    247  1.55  yamaguch 	struct vioif_work	 txq_work;
    248  1.55  yamaguch 	bool			 txq_workqueue;
    249  1.55  yamaguch 	bool			 txq_active;
    250  1.63  yamaguch 
    251  1.65  riastrad 	char			 txq_evgroup[16];
    252  1.63  yamaguch 	struct evcnt		 txq_defrag_failed;
    253  1.63  yamaguch 	struct evcnt		 txq_mbuf_load_failed;
    254  1.63  yamaguch 	struct evcnt		 txq_enqueue_reserve_failed;
    255  1.43  yamaguch };
    256  1.43  yamaguch 
    257  1.43  yamaguch struct vioif_rxqueue {
    258  1.43  yamaguch 	kmutex_t		*rxq_lock;	/* lock for rx operations */
    259  1.43  yamaguch 
    260  1.43  yamaguch 	struct virtqueue	*rxq_vq;
    261  1.43  yamaguch 	bool			rxq_stopping;
    262  1.43  yamaguch 
    263  1.43  yamaguch 	struct virtio_net_hdr	*rxq_hdrs;
    264  1.43  yamaguch 	bus_dmamap_t		*rxq_hdr_dmamaps;
    265  1.43  yamaguch 
    266  1.43  yamaguch 	struct mbuf		**rxq_mbufs;
    267  1.43  yamaguch 	bus_dmamap_t		*rxq_dmamaps;
    268  1.43  yamaguch 
    269  1.55  yamaguch 	void			*rxq_handle_si;
    270  1.55  yamaguch 	struct vioif_work	 rxq_work;
    271  1.55  yamaguch 	bool			 rxq_workqueue;
    272  1.55  yamaguch 	bool			 rxq_active;
    273  1.63  yamaguch 
    274  1.65  riastrad 	char			 rxq_evgroup[16];
    275  1.63  yamaguch 	struct evcnt		 rxq_mbuf_add_failed;
    276  1.43  yamaguch };
    277  1.43  yamaguch 
    278  1.43  yamaguch struct vioif_ctrlqueue {
    279  1.43  yamaguch 	struct virtqueue		*ctrlq_vq;
    280  1.43  yamaguch 	enum {
    281  1.43  yamaguch 		FREE, INUSE, DONE
    282  1.43  yamaguch 	}				ctrlq_inuse;
    283  1.43  yamaguch 	kcondvar_t			ctrlq_wait;
    284  1.43  yamaguch 	kmutex_t			ctrlq_wait_lock;
    285  1.44  yamaguch 	struct lwp			*ctrlq_owner;
    286  1.43  yamaguch 
    287  1.43  yamaguch 	struct virtio_net_ctrl_cmd	*ctrlq_cmd;
    288  1.43  yamaguch 	struct virtio_net_ctrl_status	*ctrlq_status;
    289  1.43  yamaguch 	struct virtio_net_ctrl_rx	*ctrlq_rx;
    290  1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_uc;
    291  1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_mc;
    292  1.74  yamaguch 	struct virtio_net_ctrl_mac_addr	*ctrlq_mac_addr;
    293  1.46  yamaguch 	struct virtio_net_ctrl_mq	*ctrlq_mq;
    294  1.43  yamaguch 
    295  1.43  yamaguch 	bus_dmamap_t			ctrlq_cmd_dmamap;
    296  1.43  yamaguch 	bus_dmamap_t			ctrlq_status_dmamap;
    297  1.43  yamaguch 	bus_dmamap_t			ctrlq_rx_dmamap;
    298  1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_uc_dmamap;
    299  1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_mc_dmamap;
    300  1.74  yamaguch 	bus_dmamap_t			ctrlq_mac_addr_dmamap;
    301  1.46  yamaguch 	bus_dmamap_t			ctrlq_mq_dmamap;
    302  1.63  yamaguch 
    303  1.63  yamaguch 	struct evcnt			ctrlq_cmd_load_failed;
    304  1.63  yamaguch 	struct evcnt			ctrlq_cmd_failed;
    305  1.43  yamaguch };
    306  1.43  yamaguch 
    307   1.1   hannken struct vioif_softc {
    308   1.1   hannken 	device_t		sc_dev;
    309  1.62  yamaguch 	kmutex_t		sc_lock;
    310  1.55  yamaguch 	struct sysctllog	*sc_sysctllog;
    311   1.1   hannken 
    312   1.1   hannken 	struct virtio_softc	*sc_virtio;
    313  1.46  yamaguch 	struct virtqueue	*sc_vqs;
    314  1.66   reinoud 	u_int			 sc_hdr_size;
    315  1.46  yamaguch 
    316  1.46  yamaguch 	int			sc_max_nvq_pairs;
    317  1.46  yamaguch 	int			sc_req_nvq_pairs;
    318  1.46  yamaguch 	int			sc_act_nvq_pairs;
    319   1.1   hannken 
    320   1.1   hannken 	uint8_t			sc_mac[ETHER_ADDR_LEN];
    321   1.1   hannken 	struct ethercom		sc_ethercom;
    322  1.34     ozaki 	bool			sc_link_active;
    323   1.1   hannken 
    324  1.46  yamaguch 	struct vioif_txqueue	*sc_txq;
    325  1.46  yamaguch 	struct vioif_rxqueue	*sc_rxq;
    326  1.43  yamaguch 
    327  1.43  yamaguch 	bool			sc_has_ctrl;
    328  1.43  yamaguch 	struct vioif_ctrlqueue	sc_ctrlq;
    329  1.43  yamaguch 
    330   1.1   hannken 	bus_dma_segment_t	sc_hdr_segs[1];
    331  1.43  yamaguch 	void			*sc_dmamem;
    332  1.43  yamaguch 	void			*sc_kmem;
    333   1.1   hannken 
    334  1.34     ozaki 	void			*sc_ctl_softint;
    335  1.55  yamaguch 
    336  1.55  yamaguch 	struct workqueue	*sc_txrx_workqueue;
    337  1.55  yamaguch 	bool			 sc_txrx_workqueue_sysctl;
    338  1.55  yamaguch 	u_int			 sc_tx_intr_process_limit;
    339  1.55  yamaguch 	u_int			 sc_tx_process_limit;
    340  1.55  yamaguch 	u_int			 sc_rx_intr_process_limit;
    341  1.55  yamaguch 	u_int			 sc_rx_process_limit;
    342   1.1   hannken };
    343   1.1   hannken #define VIRTIO_NET_TX_MAXNSEGS		(16) /* XXX */
    344   1.1   hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES	(64) /* XXX */
    345   1.1   hannken 
    346  1.55  yamaguch #define VIOIF_TX_INTR_PROCESS_LIMIT	256
    347  1.55  yamaguch #define VIOIF_TX_PROCESS_LIMIT		256
    348  1.55  yamaguch #define VIOIF_RX_INTR_PROCESS_LIMIT	0U
    349  1.55  yamaguch #define VIOIF_RX_PROCESS_LIMIT		256
    350  1.55  yamaguch 
    351  1.55  yamaguch #define VIOIF_WORKQUEUE_PRI		PRI_SOFTNET
    352  1.55  yamaguch 
    353   1.1   hannken /* cfattach interface functions */
    354   1.1   hannken static int	vioif_match(device_t, cfdata_t, void *);
    355   1.1   hannken static void	vioif_attach(device_t, device_t, void *);
    356  1.55  yamaguch static int	vioif_finalize_teardown(device_t);
    357   1.1   hannken 
    358   1.1   hannken /* ifnet interface functions */
    359   1.1   hannken static int	vioif_init(struct ifnet *);
    360   1.1   hannken static void	vioif_stop(struct ifnet *, int);
    361   1.1   hannken static void	vioif_start(struct ifnet *);
    362  1.46  yamaguch static void	vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
    363  1.46  yamaguch static int	vioif_transmit(struct ifnet *, struct mbuf *);
    364  1.46  yamaguch static void	vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
    365   1.1   hannken static int	vioif_ioctl(struct ifnet *, u_long, void *);
    366   1.1   hannken static void	vioif_watchdog(struct ifnet *);
    367  1.75  yamaguch static int	vioif_ifflags_cb(struct ethercom *);
    368   1.1   hannken 
    369   1.1   hannken /* rx */
    370  1.46  yamaguch static int	vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
    371  1.46  yamaguch static void	vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
    372  1.66   reinoud static void	vioif_populate_rx_mbufs_locked(struct vioif_softc *,
    373  1.66   reinoud 		    struct vioif_rxqueue *);
    374  1.55  yamaguch static void	vioif_rx_queue_clear(struct vioif_rxqueue *);
    375  1.55  yamaguch static bool	vioif_rx_deq_locked(struct vioif_softc *, struct virtio_softc *,
    376  1.55  yamaguch 		    struct vioif_rxqueue *, u_int);
    377  1.54  yamaguch static int	vioif_rx_intr(void *);
    378  1.55  yamaguch static void	vioif_rx_handle(void *);
    379  1.55  yamaguch static void	vioif_rx_sched_handle(struct vioif_softc *,
    380  1.55  yamaguch 		    struct vioif_rxqueue *);
    381  1.46  yamaguch static void	vioif_rx_drain(struct vioif_rxqueue *);
    382   1.1   hannken 
    383   1.1   hannken /* tx */
    384  1.54  yamaguch static int	vioif_tx_intr(void *);
    385  1.55  yamaguch static void	vioif_tx_handle(void *);
    386  1.55  yamaguch static void	vioif_tx_sched_handle(struct vioif_softc *,
    387  1.55  yamaguch 		    struct vioif_txqueue *);
    388  1.55  yamaguch static void	vioif_tx_queue_clear(struct vioif_txqueue *);
    389  1.55  yamaguch static bool	vioif_tx_deq_locked(struct vioif_softc *, struct virtio_softc *,
    390  1.55  yamaguch 		    struct vioif_txqueue *, u_int);
    391  1.46  yamaguch static void	vioif_tx_drain(struct vioif_txqueue *);
    392  1.46  yamaguch static void	vioif_deferred_transmit(void *);
    393   1.1   hannken 
    394  1.55  yamaguch /* workqueue */
    395  1.55  yamaguch static struct workqueue*
    396  1.55  yamaguch 		vioif_workq_create(const char *, pri_t, int, int);
    397  1.55  yamaguch static void	vioif_workq_destroy(struct workqueue *);
    398  1.55  yamaguch static void	vioif_workq_work(struct work *, void *);
    399  1.55  yamaguch static void	vioif_work_set(struct vioif_work *, void(*)(void *), void *);
    400  1.55  yamaguch static void	vioif_work_add(struct workqueue *, struct vioif_work *);
    401  1.55  yamaguch static void	vioif_work_wait(struct workqueue *, struct vioif_work *);
    402  1.55  yamaguch 
    403   1.1   hannken /* other control */
    404  1.34     ozaki static bool	vioif_is_link_up(struct vioif_softc *);
    405  1.34     ozaki static void	vioif_update_link_status(struct vioif_softc *);
    406   1.1   hannken static int	vioif_ctrl_rx(struct vioif_softc *, int, bool);
    407   1.1   hannken static int	vioif_set_promisc(struct vioif_softc *, bool);
    408   1.1   hannken static int	vioif_set_allmulti(struct vioif_softc *, bool);
    409   1.1   hannken static int	vioif_set_rx_filter(struct vioif_softc *);
    410   1.1   hannken static int	vioif_rx_filter(struct vioif_softc *);
    411  1.74  yamaguch static int	vioif_set_mac_addr(struct vioif_softc *);
    412  1.54  yamaguch static int	vioif_ctrl_intr(void *);
    413  1.34     ozaki static int	vioif_config_change(struct virtio_softc *);
    414  1.34     ozaki static void	vioif_ctl_softint(void *);
    415  1.46  yamaguch static int	vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
    416  1.46  yamaguch static void	vioif_enable_interrupt_vqpairs(struct vioif_softc *);
    417  1.46  yamaguch static void	vioif_disable_interrupt_vqpairs(struct vioif_softc *);
    418  1.55  yamaguch static int	vioif_setup_sysctl(struct vioif_softc *);
    419  1.63  yamaguch static void	vioif_setup_stats(struct vioif_softc *);
    420  1.75  yamaguch static int	vioif_ifflags(struct vioif_softc *);
    421   1.1   hannken 
    422   1.1   hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
    423   1.1   hannken 		  vioif_match, vioif_attach, NULL, NULL);
    424   1.1   hannken 
    425   1.1   hannken static int
    426   1.1   hannken vioif_match(device_t parent, cfdata_t match, void *aux)
    427   1.1   hannken {
    428  1.32  jdolecek 	struct virtio_attach_args *va = aux;
    429   1.1   hannken 
    430  1.66   reinoud 	if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK)
    431   1.1   hannken 		return 1;
    432   1.1   hannken 
    433   1.1   hannken 	return 0;
    434   1.1   hannken }
    435   1.1   hannken 
    436  1.58  yamaguch static int
    437  1.58  yamaguch vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
    438  1.58  yamaguch     bus_size_t size, int nsegs, const char *usage)
    439  1.58  yamaguch {
    440  1.58  yamaguch 	int r;
    441  1.58  yamaguch 
    442  1.58  yamaguch 	r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
    443  1.58  yamaguch 	    nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
    444  1.58  yamaguch 
    445  1.58  yamaguch 	if (r != 0) {
    446  1.58  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
    447  1.58  yamaguch 		    "error code %d\n", usage, r);
    448  1.58  yamaguch 	}
    449  1.58  yamaguch 
    450  1.58  yamaguch 	return r;
    451  1.58  yamaguch }
    452  1.58  yamaguch 
    453  1.58  yamaguch static void
    454  1.58  yamaguch vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
    455  1.58  yamaguch {
    456  1.58  yamaguch 
    457  1.58  yamaguch 	if (*map) {
    458  1.58  yamaguch 		bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
    459  1.58  yamaguch 		*map = NULL;
    460  1.58  yamaguch 	}
    461  1.58  yamaguch }
    462  1.58  yamaguch 
    463  1.58  yamaguch static int
    464  1.58  yamaguch vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
    465  1.58  yamaguch     void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
    466  1.58  yamaguch {
    467  1.58  yamaguch 	int r;
    468  1.58  yamaguch 
    469  1.58  yamaguch 	r = vioif_dmamap_create(sc, map, size, nsegs, usage);
    470  1.58  yamaguch 	if (r != 0)
    471  1.58  yamaguch 		return 1;
    472  1.58  yamaguch 
    473  1.58  yamaguch 	r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
    474  1.58  yamaguch 	    size, NULL, rw | BUS_DMA_NOWAIT);
    475  1.58  yamaguch 	if (r != 0) {
    476  1.58  yamaguch 		vioif_dmamap_destroy(sc, map);
    477  1.58  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
    478  1.58  yamaguch 		    "error code %d\n", usage, r);
    479  1.58  yamaguch 	}
    480  1.58  yamaguch 
    481  1.58  yamaguch 	return r;
    482  1.58  yamaguch }
    483  1.58  yamaguch 
    484  1.58  yamaguch static void *
    485  1.58  yamaguch vioif_assign_mem(intptr_t *p, size_t size)
    486  1.58  yamaguch {
    487  1.58  yamaguch 	intptr_t rv;
    488  1.58  yamaguch 
    489  1.58  yamaguch 	rv = *p;
    490  1.58  yamaguch 	*p += size;
    491  1.58  yamaguch 
    492  1.58  yamaguch 	return (void *)rv;
    493  1.58  yamaguch }
    494  1.58  yamaguch 
    495  1.51       chs static void
    496  1.46  yamaguch vioif_alloc_queues(struct vioif_softc *sc)
    497  1.46  yamaguch {
    498  1.46  yamaguch 	int nvq_pairs = sc->sc_max_nvq_pairs;
    499  1.46  yamaguch 	int nvqs = nvq_pairs * 2;
    500  1.46  yamaguch 	int i;
    501  1.46  yamaguch 
    502  1.46  yamaguch 	KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
    503  1.46  yamaguch 
    504  1.46  yamaguch 	sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
    505  1.51       chs 	    KM_SLEEP);
    506  1.46  yamaguch 	sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
    507  1.51       chs 	    KM_SLEEP);
    508  1.46  yamaguch 
    509  1.46  yamaguch 	if (sc->sc_has_ctrl)
    510  1.46  yamaguch 		nvqs++;
    511  1.46  yamaguch 
    512  1.51       chs 	sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
    513  1.46  yamaguch 	nvqs = 0;
    514  1.46  yamaguch 	for (i = 0; i < nvq_pairs; i++) {
    515  1.46  yamaguch 		sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
    516  1.46  yamaguch 		sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
    517  1.46  yamaguch 	}
    518  1.46  yamaguch 
    519  1.46  yamaguch 	if (sc->sc_has_ctrl)
    520  1.46  yamaguch 		sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
    521  1.46  yamaguch }
    522  1.46  yamaguch 
    523  1.46  yamaguch static void
    524  1.46  yamaguch vioif_free_queues(struct vioif_softc *sc)
    525  1.46  yamaguch {
    526  1.46  yamaguch 	int nvq_pairs = sc->sc_max_nvq_pairs;
    527  1.46  yamaguch 	int nvqs = nvq_pairs * 2;
    528  1.46  yamaguch 
    529  1.46  yamaguch 	if (sc->sc_ctrlq.ctrlq_vq)
    530  1.46  yamaguch 		nvqs++;
    531  1.46  yamaguch 
    532  1.46  yamaguch 	if (sc->sc_txq) {
    533  1.46  yamaguch 		kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
    534  1.46  yamaguch 		sc->sc_txq = NULL;
    535  1.46  yamaguch 	}
    536  1.46  yamaguch 
    537  1.46  yamaguch 	if (sc->sc_rxq) {
    538  1.46  yamaguch 		kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
    539  1.46  yamaguch 		sc->sc_rxq = NULL;
    540  1.46  yamaguch 	}
    541  1.46  yamaguch 
    542  1.46  yamaguch 	if (sc->sc_vqs) {
    543  1.46  yamaguch 		kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
    544  1.46  yamaguch 		sc->sc_vqs = NULL;
    545  1.46  yamaguch 	}
    546  1.46  yamaguch }
    547  1.46  yamaguch 
    548   1.1   hannken /* allocate memory */
    549   1.1   hannken /*
    550   1.1   hannken  * dma memory is used for:
    551  1.43  yamaguch  *   rxq_hdrs[slot]:	 metadata array for received frames (READ)
    552  1.43  yamaguch  *   txq_hdrs[slot]:	 metadata array for frames to be sent (WRITE)
    553  1.43  yamaguch  *   ctrlq_cmd:		 command to be sent via ctrl vq (WRITE)
    554  1.43  yamaguch  *   ctrlq_status:	 return value for a command via ctrl vq (READ)
    555  1.43  yamaguch  *   ctrlq_rx:		 parameter for a VIRTIO_NET_CTRL_RX class command
    556   1.1   hannken  *			 (WRITE)
    557  1.43  yamaguch  *   ctrlq_mac_tbl_uc:	 unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
    558   1.1   hannken  *			 class command (WRITE)
    559  1.43  yamaguch  *   ctrlq_mac_tbl_mc:	 multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
    560   1.1   hannken  *			 class command (WRITE)
    561  1.43  yamaguch  * ctrlq_* structures are allocated only one each; they are protected by
    562  1.43  yamaguch  * ctrlq_inuse variable and ctrlq_wait condvar.
    563   1.1   hannken  */
    564   1.1   hannken /*
    565   1.1   hannken  * dynamically allocated memory is used for:
    566  1.43  yamaguch  *   rxq_hdr_dmamaps[slot]:	bus_dmamap_t array for sc_rx_hdrs[slot]
    567  1.43  yamaguch  *   txq_hdr_dmamaps[slot]:	bus_dmamap_t array for sc_tx_hdrs[slot]
    568  1.43  yamaguch  *   rxq_dmamaps[slot]:		bus_dmamap_t array for received payload
    569  1.43  yamaguch  *   txq_dmamaps[slot]:		bus_dmamap_t array for sent payload
    570  1.43  yamaguch  *   rxq_mbufs[slot]:		mbuf pointer array for received frames
    571  1.43  yamaguch  *   txq_mbufs[slot]:		mbuf pointer array for sent frames
    572   1.1   hannken  */
    573   1.1   hannken static int
    574   1.1   hannken vioif_alloc_mems(struct vioif_softc *sc)
    575   1.1   hannken {
    576   1.1   hannken 	struct virtio_softc *vsc = sc->sc_virtio;
    577  1.46  yamaguch 	struct vioif_txqueue *txq;
    578  1.46  yamaguch 	struct vioif_rxqueue *rxq;
    579  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    580  1.46  yamaguch 	int allocsize, allocsize2, r, rsegs, i, qid;
    581   1.1   hannken 	void *vaddr;
    582   1.1   hannken 	intptr_t p;
    583   1.1   hannken 
    584  1.46  yamaguch 	allocsize = 0;
    585  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    586  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    587  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    588  1.46  yamaguch 
    589  1.69   reinoud 		allocsize += sizeof(struct virtio_net_hdr) *
    590  1.68   reinoud 			(rxq->rxq_vq->vq_num + txq->txq_vq->vq_num);
    591  1.46  yamaguch 	}
    592  1.32  jdolecek 	if (sc->sc_has_ctrl) {
    593  1.74  yamaguch 		allocsize += sizeof(struct virtio_net_ctrl_cmd);
    594  1.74  yamaguch 		allocsize += sizeof(struct virtio_net_ctrl_status);
    595  1.74  yamaguch 		allocsize += sizeof(struct virtio_net_ctrl_rx);
    596  1.74  yamaguch 		allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
    597  1.74  yamaguch 		    + ETHER_ADDR_LEN;
    598   1.1   hannken 		allocsize += sizeof(struct virtio_net_ctrl_mac_tbl)
    599  1.50  christos 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
    600  1.74  yamaguch 		allocsize += sizeof(struct virtio_net_ctrl_mac_addr);
    601  1.74  yamaguch 		allocsize += sizeof(struct virtio_net_ctrl_mq);
    602   1.1   hannken 	}
    603  1.32  jdolecek 	r = bus_dmamem_alloc(virtio_dmat(vsc), allocsize, 0, 0,
    604  1.50  christos 	    &sc->sc_hdr_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
    605   1.1   hannken 	if (r != 0) {
    606   1.1   hannken 		aprint_error_dev(sc->sc_dev,
    607  1.50  christos 		    "DMA memory allocation failed, size %d, "
    608  1.50  christos 		    "error code %d\n", allocsize, r);
    609   1.1   hannken 		goto err_none;
    610   1.1   hannken 	}
    611  1.32  jdolecek 	r = bus_dmamem_map(virtio_dmat(vsc),
    612  1.50  christos 	    &sc->sc_hdr_segs[0], 1, allocsize, &vaddr, BUS_DMA_NOWAIT);
    613   1.1   hannken 	if (r != 0) {
    614   1.1   hannken 		aprint_error_dev(sc->sc_dev,
    615  1.50  christos 		    "DMA memory map failed, error code %d\n", r);
    616   1.1   hannken 		goto err_dmamem_alloc;
    617   1.1   hannken 	}
    618  1.42  yamaguch 
    619   1.1   hannken 	memset(vaddr, 0, allocsize);
    620  1.43  yamaguch 	sc->sc_dmamem = vaddr;
    621   1.1   hannken 	p = (intptr_t) vaddr;
    622  1.42  yamaguch 
    623  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    624  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    625  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    626  1.46  yamaguch 
    627  1.58  yamaguch 		rxq->rxq_hdrs = vioif_assign_mem(&p,
    628  1.67   reinoud 		    sizeof(struct virtio_net_hdr) * rxq->rxq_vq->vq_num);
    629  1.58  yamaguch 		txq->txq_hdrs = vioif_assign_mem(&p,
    630  1.67   reinoud 		    sizeof(struct virtio_net_hdr) * txq->txq_vq->vq_num);
    631  1.46  yamaguch 	}
    632  1.32  jdolecek 	if (sc->sc_has_ctrl) {
    633  1.58  yamaguch 		ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
    634  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_cmd));
    635  1.58  yamaguch 		ctrlq->ctrlq_status = vioif_assign_mem(&p,
    636  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_status));
    637  1.58  yamaguch 		ctrlq->ctrlq_rx = vioif_assign_mem(&p,
    638  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_rx));
    639  1.58  yamaguch 		ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
    640  1.74  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc)
    641  1.74  yamaguch 		    + ETHER_ADDR_LEN);
    642  1.58  yamaguch 		ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
    643  1.58  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
    644  1.50  christos 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
    645  1.74  yamaguch 		ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p,
    646  1.74  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_addr));
    647  1.58  yamaguch 		ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
    648   1.1   hannken 	}
    649   1.1   hannken 
    650  1.46  yamaguch 	allocsize2 = 0;
    651  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    652  1.46  yamaguch 		int rxqsize, txqsize;
    653  1.46  yamaguch 
    654  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    655  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    656  1.46  yamaguch 		rxqsize = rxq->rxq_vq->vq_num;
    657  1.46  yamaguch 		txqsize = txq->txq_vq->vq_num;
    658  1.46  yamaguch 
    659  1.46  yamaguch 		allocsize2 += sizeof(rxq->rxq_dmamaps[0]) * rxqsize;
    660  1.46  yamaguch 		allocsize2 += sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize;
    661  1.46  yamaguch 		allocsize2 += sizeof(rxq->rxq_mbufs[0]) * rxqsize;
    662  1.46  yamaguch 
    663  1.46  yamaguch 		allocsize2 += sizeof(txq->txq_dmamaps[0]) * txqsize;
    664  1.46  yamaguch 		allocsize2 += sizeof(txq->txq_hdr_dmamaps[0]) * txqsize;
    665  1.46  yamaguch 		allocsize2 += sizeof(txq->txq_mbufs[0]) * txqsize;
    666  1.46  yamaguch 	}
    667  1.42  yamaguch 	vaddr = kmem_zalloc(allocsize2, KM_SLEEP);
    668  1.43  yamaguch 	sc->sc_kmem = vaddr;
    669  1.42  yamaguch 	p = (intptr_t) vaddr;
    670  1.42  yamaguch 
    671  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    672  1.46  yamaguch 		int rxqsize, txqsize;
    673  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    674  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    675  1.46  yamaguch 		rxqsize = rxq->rxq_vq->vq_num;
    676  1.46  yamaguch 		txqsize = txq->txq_vq->vq_num;
    677  1.46  yamaguch 
    678  1.58  yamaguch 		rxq->rxq_hdr_dmamaps = vioif_assign_mem(&p,
    679  1.48   msaitoh 		    sizeof(rxq->rxq_hdr_dmamaps[0]) * rxqsize);
    680  1.58  yamaguch 		txq->txq_hdr_dmamaps = vioif_assign_mem(&p,
    681  1.48   msaitoh 		    sizeof(txq->txq_hdr_dmamaps[0]) * txqsize);
    682  1.58  yamaguch 		rxq->rxq_dmamaps = vioif_assign_mem(&p,
    683  1.58  yamaguch 		    sizeof(rxq->rxq_dmamaps[0]) * rxqsize);
    684  1.58  yamaguch 		txq->txq_dmamaps = vioif_assign_mem(&p,
    685  1.58  yamaguch 		    sizeof(txq->txq_dmamaps[0]) * txqsize);
    686  1.58  yamaguch 		rxq->rxq_mbufs = vioif_assign_mem(&p,
    687  1.58  yamaguch 		    sizeof(rxq->rxq_mbufs[0]) * rxqsize);
    688  1.58  yamaguch 		txq->txq_mbufs = vioif_assign_mem(&p,
    689  1.58  yamaguch 		    sizeof(txq->txq_mbufs[0]) * txqsize);
    690  1.58  yamaguch 	}
    691   1.1   hannken 
    692  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    693  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    694  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    695  1.46  yamaguch 
    696  1.46  yamaguch 		for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
    697  1.58  yamaguch 			r = vioif_dmamap_create_load(sc, &rxq->rxq_hdr_dmamaps[i],
    698  1.66   reinoud 			    &rxq->rxq_hdrs[i], sc->sc_hdr_size, 1,
    699  1.46  yamaguch 			    BUS_DMA_READ, "rx header");
    700  1.58  yamaguch 			if (r != 0)
    701  1.58  yamaguch 				goto err_reqs;
    702  1.58  yamaguch 
    703  1.58  yamaguch 			r = vioif_dmamap_create(sc, &rxq->rxq_dmamaps[i],
    704  1.58  yamaguch 			    MCLBYTES, 1, "rx payload");
    705  1.58  yamaguch 			if (r != 0)
    706  1.58  yamaguch 				goto err_reqs;
    707  1.46  yamaguch 		}
    708  1.46  yamaguch 
    709  1.46  yamaguch 		for (i = 0; i < txq->txq_vq->vq_num; i++) {
    710  1.58  yamaguch 			r = vioif_dmamap_create_load(sc, &txq->txq_hdr_dmamaps[i],
    711  1.66   reinoud 			    &txq->txq_hdrs[i], sc->sc_hdr_size, 1,
    712  1.46  yamaguch 			    BUS_DMA_READ, "tx header");
    713  1.58  yamaguch 			if (r != 0)
    714  1.58  yamaguch 				goto err_reqs;
    715  1.58  yamaguch 
    716  1.58  yamaguch 			r = vioif_dmamap_create(sc, &txq->txq_dmamaps[i], ETHER_MAX_LEN,
    717  1.46  yamaguch 			    VIRTIO_NET_TX_MAXNSEGS, "tx payload");
    718  1.58  yamaguch 			if (r != 0)
    719  1.58  yamaguch 				goto err_reqs;
    720  1.46  yamaguch 		}
    721   1.1   hannken 	}
    722   1.1   hannken 
    723  1.32  jdolecek 	if (sc->sc_has_ctrl) {
    724   1.1   hannken 		/* control vq class & command */
    725  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
    726  1.43  yamaguch 		    ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
    727  1.42  yamaguch 		    BUS_DMA_WRITE, "control command");
    728  1.58  yamaguch 		if (r != 0)
    729  1.58  yamaguch 			goto err_reqs;
    730  1.58  yamaguch 
    731  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
    732  1.43  yamaguch 		    ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
    733  1.42  yamaguch 		    BUS_DMA_READ, "control status");
    734  1.58  yamaguch 		if (r != 0)
    735  1.58  yamaguch 			goto err_reqs;
    736   1.1   hannken 
    737   1.1   hannken 		/* control vq rx mode command parameter */
    738  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
    739  1.43  yamaguch 		    ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
    740  1.42  yamaguch 		    BUS_DMA_WRITE, "rx mode control command");
    741  1.58  yamaguch 		if (r != 0)
    742  1.58  yamaguch 			goto err_reqs;
    743   1.1   hannken 
    744  1.46  yamaguch 		/* multiqueue set command */
    745  1.58  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
    746  1.46  yamaguch 		    ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
    747  1.46  yamaguch 		    BUS_DMA_WRITE, "multiqueue set command");
    748  1.58  yamaguch 		if (r != 0)
    749  1.58  yamaguch 			goto err_reqs;
    750  1.46  yamaguch 
    751   1.1   hannken 		/* control vq MAC filter table for unicast */
    752   1.1   hannken 		/* do not load now since its length is variable */
    753  1.58  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
    754  1.74  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc)
    755  1.74  yamaguch 		    + ETHER_ADDR_LEN, 1,
    756  1.42  yamaguch 		    "unicast MAC address filter command");
    757  1.58  yamaguch 		if (r != 0)
    758  1.58  yamaguch 			goto err_reqs;
    759   1.1   hannken 
    760   1.1   hannken 		/* control vq MAC filter table for multicast */
    761  1.58  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
    762  1.43  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
    763  1.42  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
    764  1.42  yamaguch 		    "multicast MAC address filter command");
    765  1.73  yamaguch 		if (r != 0)
    766  1.73  yamaguch 			goto err_reqs;
    767  1.74  yamaguch 
    768  1.74  yamaguch 		/* control vq MAC address set command */
    769  1.74  yamaguch 		r = vioif_dmamap_create_load(sc,
    770  1.74  yamaguch 		    &ctrlq->ctrlq_mac_addr_dmamap,
    771  1.74  yamaguch 		    ctrlq->ctrlq_mac_addr,
    772  1.74  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_addr), 1,
    773  1.74  yamaguch 		    BUS_DMA_WRITE, "mac addr set command");
    774  1.74  yamaguch 		if (r != 0)
    775  1.74  yamaguch 			goto err_reqs;
    776   1.1   hannken 	}
    777   1.1   hannken 
    778   1.1   hannken 	return 0;
    779   1.1   hannken 
    780   1.1   hannken err_reqs:
    781  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
    782  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
    783  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
    784  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
    785  1.58  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
    786  1.74  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap);
    787  1.46  yamaguch 	for (qid = 0; qid < sc->sc_max_nvq_pairs; qid++) {
    788  1.46  yamaguch 		rxq = &sc->sc_rxq[qid];
    789  1.46  yamaguch 		txq = &sc->sc_txq[qid];
    790  1.46  yamaguch 
    791  1.46  yamaguch 		for (i = 0; i < txq->txq_vq->vq_num; i++) {
    792  1.58  yamaguch 			vioif_dmamap_destroy(sc, &txq->txq_dmamaps[i]);
    793  1.58  yamaguch 			vioif_dmamap_destroy(sc, &txq->txq_hdr_dmamaps[i]);
    794  1.46  yamaguch 		}
    795  1.46  yamaguch 		for (i = 0; i < rxq->rxq_vq->vq_num; i++) {
    796  1.58  yamaguch 			vioif_dmamap_destroy(sc, &rxq->rxq_dmamaps[i]);
    797  1.58  yamaguch 			vioif_dmamap_destroy(sc, &rxq->rxq_hdr_dmamaps[i]);
    798  1.46  yamaguch 		}
    799   1.1   hannken 	}
    800  1.43  yamaguch 	if (sc->sc_kmem) {
    801  1.43  yamaguch 		kmem_free(sc->sc_kmem, allocsize2);
    802  1.43  yamaguch 		sc->sc_kmem = NULL;
    803   1.1   hannken 	}
    804  1.43  yamaguch 	bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, allocsize);
    805   1.1   hannken err_dmamem_alloc:
    806  1.32  jdolecek 	bus_dmamem_free(virtio_dmat(vsc), &sc->sc_hdr_segs[0], 1);
    807   1.1   hannken err_none:
    808   1.1   hannken 	return -1;
    809   1.1   hannken }
    810   1.1   hannken 
    811   1.1   hannken static void
    812   1.1   hannken vioif_attach(device_t parent, device_t self, void *aux)
    813   1.1   hannken {
    814   1.1   hannken 	struct vioif_softc *sc = device_private(self);
    815   1.1   hannken 	struct virtio_softc *vsc = device_private(parent);
    816  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    817  1.46  yamaguch 	struct vioif_txqueue *txq;
    818  1.46  yamaguch 	struct vioif_rxqueue *rxq;
    819  1.66   reinoud 	uint64_t features, req_features;
    820   1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    821  1.43  yamaguch 	u_int softint_flags;
    822  1.46  yamaguch 	int r, i, nvqs=0, req_flags;
    823  1.55  yamaguch 	char xnamebuf[MAXCOMLEN];
    824   1.1   hannken 
    825  1.32  jdolecek 	if (virtio_child(vsc) != NULL) {
    826   1.1   hannken 		aprint_normal(": child already attached for %s; "
    827  1.50  christos 		    "something wrong...\n", device_xname(parent));
    828   1.1   hannken 		return;
    829   1.1   hannken 	}
    830   1.1   hannken 
    831   1.1   hannken 	sc->sc_dev = self;
    832   1.1   hannken 	sc->sc_virtio = vsc;
    833  1.34     ozaki 	sc->sc_link_active = false;
    834   1.1   hannken 
    835  1.46  yamaguch 	sc->sc_max_nvq_pairs = 1;
    836  1.46  yamaguch 	sc->sc_req_nvq_pairs = 1;
    837  1.46  yamaguch 	sc->sc_act_nvq_pairs = 1;
    838  1.55  yamaguch 	sc->sc_txrx_workqueue_sysctl = true;
    839  1.55  yamaguch 	sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
    840  1.55  yamaguch 	sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
    841  1.55  yamaguch 	sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
    842  1.55  yamaguch 	sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
    843  1.55  yamaguch 
    844  1.62  yamaguch 	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE);
    845  1.62  yamaguch 
    846  1.55  yamaguch 	snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
    847  1.55  yamaguch 	sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
    848  1.55  yamaguch 	    IPL_NET, WQ_PERCPU | WQ_MPSAFE);
    849  1.55  yamaguch 	if (sc->sc_txrx_workqueue == NULL)
    850  1.55  yamaguch 		goto err;
    851  1.46  yamaguch 
    852  1.32  jdolecek 	req_flags = 0;
    853   1.1   hannken 
    854   1.7     ozaki #ifdef VIOIF_MPSAFE
    855  1.66   reinoud 	req_flags |= VIRTIO_F_INTR_MPSAFE;
    856   1.7     ozaki #endif
    857  1.66   reinoud 	req_flags |= VIRTIO_F_INTR_MSIX;
    858  1.32  jdolecek 
    859  1.46  yamaguch 	req_features =
    860  1.46  yamaguch 	    VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
    861  1.46  yamaguch 	    VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
    862  1.66   reinoud 	req_features |= VIRTIO_F_RING_EVENT_IDX;
    863  1.78  yamaguch 	req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR;
    864  1.46  yamaguch #ifdef VIOIF_MULTIQ
    865  1.46  yamaguch 	req_features |= VIRTIO_NET_F_MQ;
    866  1.46  yamaguch #endif
    867  1.46  yamaguch 	virtio_child_attach_start(vsc, self, IPL_NET, NULL,
    868  1.54  yamaguch 	    vioif_config_change, virtio_vq_intrhand, req_flags,
    869  1.46  yamaguch 	    req_features, VIRTIO_NET_FLAG_BITS);
    870  1.32  jdolecek 
    871  1.32  jdolecek 	features = virtio_features(vsc);
    872  1.66   reinoud 	if (features == 0)
    873  1.66   reinoud 		goto err;
    874   1.7     ozaki 
    875   1.1   hannken 	if (features & VIRTIO_NET_F_MAC) {
    876  1.50  christos 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    877  1.50  christos 			sc->sc_mac[i] = virtio_read_device_config_1(vsc,
    878  1.50  christos 			    VIRTIO_NET_CONFIG_MAC + i);
    879  1.50  christos 		}
    880   1.1   hannken 	} else {
    881   1.1   hannken 		/* code stolen from sys/net/if_tap.c */
    882   1.1   hannken 		struct timeval tv;
    883   1.1   hannken 		uint32_t ui;
    884   1.1   hannken 		getmicrouptime(&tv);
    885   1.1   hannken 		ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
    886   1.1   hannken 		memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
    887  1.50  christos 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    888  1.50  christos 			virtio_write_device_config_1(vsc,
    889  1.50  christos 			    VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
    890  1.50  christos 		}
    891   1.1   hannken 	}
    892  1.32  jdolecek 
    893  1.66   reinoud 	/* 'Ethernet' with capital follows other ethernet driver attachment */
    894  1.50  christos 	aprint_normal_dev(self, "Ethernet address %s\n",
    895  1.50  christos 	    ether_sprintf(sc->sc_mac));
    896   1.1   hannken 
    897  1.66   reinoud 	if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) {
    898  1.66   reinoud 		sc->sc_hdr_size = sizeof(struct virtio_net_hdr);
    899  1.66   reinoud 	} else {
    900  1.66   reinoud 		sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers);
    901  1.66   reinoud 	}
    902  1.66   reinoud 
    903  1.46  yamaguch 	if ((features & VIRTIO_NET_F_CTRL_VQ) &&
    904  1.46  yamaguch 	    (features & VIRTIO_NET_F_CTRL_RX)) {
    905  1.46  yamaguch 		sc->sc_has_ctrl = true;
    906  1.46  yamaguch 
    907  1.46  yamaguch 		cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
    908  1.46  yamaguch 		mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
    909  1.46  yamaguch 		ctrlq->ctrlq_inuse = FREE;
    910  1.46  yamaguch 	} else {
    911  1.46  yamaguch 		sc->sc_has_ctrl = false;
    912  1.46  yamaguch 	}
    913  1.46  yamaguch 
    914  1.46  yamaguch 	if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
    915  1.46  yamaguch 		sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
    916  1.46  yamaguch 		    VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
    917  1.46  yamaguch 
    918  1.46  yamaguch 		if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
    919  1.46  yamaguch 			goto err;
    920  1.46  yamaguch 
    921  1.46  yamaguch 		/* Limit the number of queue pairs to use */
    922  1.46  yamaguch 		sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
    923  1.46  yamaguch 	}
    924  1.46  yamaguch 
    925  1.51       chs 	vioif_alloc_queues(sc);
    926  1.46  yamaguch 	virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
    927  1.46  yamaguch 
    928  1.43  yamaguch #ifdef VIOIF_MPSAFE
    929  1.43  yamaguch 	softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
    930  1.43  yamaguch #else
    931  1.43  yamaguch 	softint_flags = SOFTINT_NET;
    932  1.43  yamaguch #endif
    933   1.7     ozaki 
    934  1.17     ozaki 	/*
    935  1.57  yamaguch 	 * Allocating virtqueues
    936  1.17     ozaki 	 */
    937  1.46  yamaguch 	for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
    938  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
    939  1.46  yamaguch 		txq = &sc->sc_txq[i];
    940  1.46  yamaguch 		char qname[32];
    941  1.46  yamaguch 
    942  1.46  yamaguch 		rxq->rxq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
    943  1.46  yamaguch 
    944  1.55  yamaguch 		rxq->rxq_handle_si = softint_establish(softint_flags,
    945  1.55  yamaguch 		    vioif_rx_handle, rxq);
    946  1.55  yamaguch 		if (rxq->rxq_handle_si == NULL) {
    947  1.55  yamaguch 			aprint_error_dev(self, "cannot establish rx softint\n");
    948  1.55  yamaguch 			goto err;
    949  1.55  yamaguch 		}
    950  1.55  yamaguch 
    951  1.46  yamaguch 		snprintf(qname, sizeof(qname), "rx%d", i);
    952  1.46  yamaguch 		r = virtio_alloc_vq(vsc, rxq->rxq_vq, nvqs,
    953  1.66   reinoud 		    MCLBYTES + sc->sc_hdr_size, 2, qname);
    954  1.46  yamaguch 		if (r != 0)
    955  1.46  yamaguch 			goto err;
    956  1.46  yamaguch 		nvqs++;
    957  1.54  yamaguch 		rxq->rxq_vq->vq_intrhand = vioif_rx_intr;
    958  1.54  yamaguch 		rxq->rxq_vq->vq_intrhand_arg = (void *)rxq;
    959  1.46  yamaguch 		rxq->rxq_stopping = true;
    960  1.55  yamaguch 		vioif_work_set(&rxq->rxq_work, vioif_rx_handle, rxq);
    961  1.46  yamaguch 
    962  1.46  yamaguch 		txq->txq_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NET);
    963  1.55  yamaguch 
    964  1.46  yamaguch 		txq->txq_deferred_transmit = softint_establish(softint_flags,
    965  1.46  yamaguch 		    vioif_deferred_transmit, txq);
    966  1.46  yamaguch 		if (txq->txq_deferred_transmit == NULL) {
    967  1.46  yamaguch 			aprint_error_dev(self, "cannot establish tx softint\n");
    968  1.46  yamaguch 			goto err;
    969  1.46  yamaguch 		}
    970  1.55  yamaguch 		txq->txq_handle_si = softint_establish(softint_flags,
    971  1.55  yamaguch 		    vioif_tx_handle, txq);
    972  1.55  yamaguch 		if (txq->txq_handle_si == NULL) {
    973  1.55  yamaguch 			aprint_error_dev(self, "cannot establish tx softint\n");
    974  1.55  yamaguch 			goto err;
    975  1.55  yamaguch 		}
    976  1.55  yamaguch 
    977  1.46  yamaguch 		snprintf(qname, sizeof(qname), "tx%d", i);
    978  1.46  yamaguch 		r = virtio_alloc_vq(vsc, txq->txq_vq, nvqs,
    979  1.66   reinoud 		    sc->sc_hdr_size + (ETHER_MAX_LEN - ETHER_HDR_LEN),
    980  1.46  yamaguch 		    VIRTIO_NET_TX_MAXNSEGS + 1, qname);
    981  1.46  yamaguch 		if (r != 0)
    982  1.46  yamaguch 			goto err;
    983  1.46  yamaguch 		nvqs++;
    984  1.54  yamaguch 		txq->txq_vq->vq_intrhand = vioif_tx_intr;
    985  1.54  yamaguch 		txq->txq_vq->vq_intrhand_arg = (void *)txq;
    986  1.46  yamaguch 		txq->txq_link_active = sc->sc_link_active;
    987  1.46  yamaguch 		txq->txq_stopping = false;
    988  1.51       chs 		txq->txq_intrq = pcq_create(txq->txq_vq->vq_num, KM_SLEEP);
    989  1.55  yamaguch 		vioif_work_set(&txq->txq_work, vioif_tx_handle, txq);
    990  1.43  yamaguch 	}
    991  1.17     ozaki 
    992  1.46  yamaguch 	if (sc->sc_has_ctrl) {
    993  1.17     ozaki 		/*
    994  1.17     ozaki 		 * Allocating a virtqueue for control channel
    995  1.17     ozaki 		 */
    996  1.46  yamaguch 		r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, nvqs,
    997  1.17     ozaki 		    NBPG, 1, "control");
    998  1.17     ozaki 		if (r != 0) {
    999  1.17     ozaki 			aprint_error_dev(self, "failed to allocate "
   1000  1.50  christos 			    "a virtqueue for control channel, error code %d\n",
   1001  1.50  christos 			    r);
   1002  1.46  yamaguch 
   1003  1.46  yamaguch 			sc->sc_has_ctrl = false;
   1004  1.46  yamaguch 			cv_destroy(&ctrlq->ctrlq_wait);
   1005  1.46  yamaguch 			mutex_destroy(&ctrlq->ctrlq_wait_lock);
   1006  1.46  yamaguch 		} else {
   1007  1.46  yamaguch 			nvqs++;
   1008  1.54  yamaguch 			ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
   1009  1.54  yamaguch 			ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
   1010   1.1   hannken 		}
   1011   1.1   hannken 	}
   1012  1.34     ozaki 
   1013  1.48   msaitoh 	sc->sc_ctl_softint = softint_establish(softint_flags,
   1014  1.48   msaitoh 	    vioif_ctl_softint, sc);
   1015  1.34     ozaki 	if (sc->sc_ctl_softint == NULL) {
   1016  1.34     ozaki 		aprint_error_dev(self, "cannot establish ctl softint\n");
   1017   1.1   hannken 		goto err;
   1018   1.1   hannken 	}
   1019   1.1   hannken 
   1020   1.1   hannken 	if (vioif_alloc_mems(sc) < 0)
   1021   1.1   hannken 		goto err;
   1022   1.1   hannken 
   1023  1.32  jdolecek 	if (virtio_child_attach_finish(vsc) != 0)
   1024  1.32  jdolecek 		goto err;
   1025  1.32  jdolecek 
   1026  1.55  yamaguch 	if (vioif_setup_sysctl(sc) != 0) {
   1027  1.55  yamaguch 		aprint_error_dev(self, "unable to create sysctl node\n");
   1028  1.55  yamaguch 		/* continue */
   1029  1.55  yamaguch 	}
   1030  1.55  yamaguch 
   1031  1.63  yamaguch 	vioif_setup_stats(sc);
   1032  1.63  yamaguch 
   1033   1.1   hannken 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
   1034   1.1   hannken 	ifp->if_softc = sc;
   1035   1.1   hannken 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
   1036  1.45  yamaguch #ifdef VIOIF_MPSAFE
   1037  1.45  yamaguch 	ifp->if_extflags = IFEF_MPSAFE;
   1038  1.45  yamaguch #endif
   1039   1.1   hannken 	ifp->if_start = vioif_start;
   1040  1.46  yamaguch 	if (sc->sc_req_nvq_pairs > 1)
   1041  1.46  yamaguch 		ifp->if_transmit = vioif_transmit;
   1042   1.1   hannken 	ifp->if_ioctl = vioif_ioctl;
   1043   1.1   hannken 	ifp->if_init = vioif_init;
   1044   1.1   hannken 	ifp->if_stop = vioif_stop;
   1045   1.1   hannken 	ifp->if_capabilities = 0;
   1046   1.1   hannken 	ifp->if_watchdog = vioif_watchdog;
   1047  1.46  yamaguch 	txq = &sc->sc_txq[0];
   1048  1.43  yamaguch 	IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq->txq_vq->vq_num, IFQ_MAXLEN));
   1049  1.36  jdolecek 	IFQ_SET_READY(&ifp->if_snd);
   1050   1.1   hannken 
   1051  1.11     ozaki 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
   1052  1.11     ozaki 
   1053   1.1   hannken 	if_attach(ifp);
   1054  1.28     ozaki 	if_deferred_start_init(ifp, NULL);
   1055   1.1   hannken 	ether_ifattach(ifp, sc->sc_mac);
   1056  1.75  yamaguch 	ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb);
   1057   1.1   hannken 
   1058   1.1   hannken 	return;
   1059   1.1   hannken 
   1060   1.1   hannken err:
   1061  1.46  yamaguch 	for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
   1062  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1063  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1064  1.46  yamaguch 
   1065  1.46  yamaguch 		if (rxq->rxq_lock) {
   1066  1.46  yamaguch 			mutex_obj_free(rxq->rxq_lock);
   1067  1.46  yamaguch 			rxq->rxq_lock = NULL;
   1068  1.46  yamaguch 		}
   1069  1.46  yamaguch 
   1070  1.55  yamaguch 		if (rxq->rxq_handle_si) {
   1071  1.55  yamaguch 			softint_disestablish(rxq->rxq_handle_si);
   1072  1.55  yamaguch 			rxq->rxq_handle_si = NULL;
   1073  1.55  yamaguch 		}
   1074  1.55  yamaguch 
   1075  1.46  yamaguch 		if (txq->txq_lock) {
   1076  1.46  yamaguch 			mutex_obj_free(txq->txq_lock);
   1077  1.46  yamaguch 			txq->txq_lock = NULL;
   1078  1.46  yamaguch 		}
   1079  1.43  yamaguch 
   1080  1.55  yamaguch 		if (txq->txq_handle_si) {
   1081  1.55  yamaguch 			softint_disestablish(txq->txq_handle_si);
   1082  1.55  yamaguch 			txq->txq_handle_si = NULL;
   1083  1.55  yamaguch 		}
   1084  1.55  yamaguch 
   1085  1.46  yamaguch 		if (txq->txq_deferred_transmit) {
   1086  1.46  yamaguch 			softint_disestablish(txq->txq_deferred_transmit);
   1087  1.46  yamaguch 			txq->txq_deferred_transmit = NULL;
   1088  1.46  yamaguch 		}
   1089  1.43  yamaguch 
   1090  1.46  yamaguch 		if (txq->txq_intrq) {
   1091  1.46  yamaguch 			pcq_destroy(txq->txq_intrq);
   1092  1.46  yamaguch 			txq->txq_intrq = NULL;
   1093  1.46  yamaguch 		}
   1094  1.43  yamaguch 	}
   1095   1.7     ozaki 
   1096  1.32  jdolecek 	if (sc->sc_has_ctrl) {
   1097  1.43  yamaguch 		cv_destroy(&ctrlq->ctrlq_wait);
   1098  1.43  yamaguch 		mutex_destroy(&ctrlq->ctrlq_wait_lock);
   1099   1.1   hannken 	}
   1100  1.20  christos 
   1101  1.32  jdolecek 	while (nvqs > 0)
   1102  1.46  yamaguch 		virtio_free_vq(vsc, &sc->sc_vqs[--nvqs]);
   1103  1.46  yamaguch 
   1104  1.46  yamaguch 	vioif_free_queues(sc);
   1105  1.62  yamaguch 	mutex_destroy(&sc->sc_lock);
   1106  1.55  yamaguch 	virtio_child_attach_failed(vsc);
   1107  1.55  yamaguch 	config_finalize_register(self, vioif_finalize_teardown);
   1108  1.20  christos 
   1109   1.1   hannken 	return;
   1110   1.1   hannken }
   1111   1.1   hannken 
   1112  1.55  yamaguch static int
   1113  1.55  yamaguch vioif_finalize_teardown(device_t self)
   1114  1.55  yamaguch {
   1115  1.55  yamaguch 	struct vioif_softc *sc = device_private(self);
   1116  1.55  yamaguch 
   1117  1.55  yamaguch 	if (sc->sc_txrx_workqueue != NULL) {
   1118  1.55  yamaguch 		vioif_workq_destroy(sc->sc_txrx_workqueue);
   1119  1.55  yamaguch 		sc->sc_txrx_workqueue = NULL;
   1120  1.55  yamaguch 	}
   1121  1.55  yamaguch 
   1122  1.55  yamaguch 	return 0;
   1123  1.55  yamaguch }
   1124  1.55  yamaguch 
   1125  1.46  yamaguch static void
   1126  1.46  yamaguch vioif_enable_interrupt_vqpairs(struct vioif_softc *sc)
   1127  1.46  yamaguch {
   1128  1.46  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1129  1.46  yamaguch 	struct vioif_txqueue *txq;
   1130  1.46  yamaguch 	struct vioif_rxqueue *rxq;
   1131  1.46  yamaguch 	int i;
   1132  1.46  yamaguch 
   1133  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1134  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1135  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1136  1.46  yamaguch 
   1137  1.46  yamaguch 		virtio_start_vq_intr(vsc, txq->txq_vq);
   1138  1.46  yamaguch 		virtio_start_vq_intr(vsc, rxq->rxq_vq);
   1139  1.46  yamaguch 	}
   1140  1.46  yamaguch }
   1141  1.46  yamaguch 
   1142  1.46  yamaguch static void
   1143  1.46  yamaguch vioif_disable_interrupt_vqpairs(struct vioif_softc *sc)
   1144  1.46  yamaguch {
   1145  1.46  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1146  1.46  yamaguch 	struct vioif_txqueue *txq;
   1147  1.46  yamaguch 	struct vioif_rxqueue *rxq;
   1148  1.46  yamaguch 	int i;
   1149  1.46  yamaguch 
   1150  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1151  1.64  yamaguch 		rxq = &sc->sc_rxq[i];
   1152  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1153  1.46  yamaguch 
   1154  1.64  yamaguch 		virtio_stop_vq_intr(vsc, rxq->rxq_vq);
   1155  1.46  yamaguch 		virtio_stop_vq_intr(vsc, txq->txq_vq);
   1156  1.46  yamaguch 	}
   1157  1.46  yamaguch }
   1158  1.46  yamaguch 
   1159   1.1   hannken /*
   1160   1.1   hannken  * Interface functions for ifnet
   1161   1.1   hannken  */
   1162   1.1   hannken static int
   1163   1.1   hannken vioif_init(struct ifnet *ifp)
   1164   1.1   hannken {
   1165   1.1   hannken 	struct vioif_softc *sc = ifp->if_softc;
   1166  1.33     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   1167  1.46  yamaguch 	struct vioif_rxqueue *rxq;
   1168  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1169  1.46  yamaguch 	int r, i;
   1170   1.1   hannken 
   1171   1.1   hannken 	vioif_stop(ifp, 0);
   1172   1.7     ozaki 
   1173  1.71  yamaguch 	r = virtio_reinit_start(vsc);
   1174  1.71  yamaguch 	if (r != 0) {
   1175  1.71  yamaguch 		log(LOG_ERR, "%s: reset failed\n", ifp->if_xname);
   1176  1.71  yamaguch 		return EIO;
   1177  1.71  yamaguch 	}
   1178  1.71  yamaguch 
   1179  1.33     ozaki 	virtio_negotiate_features(vsc, virtio_features(vsc));
   1180  1.46  yamaguch 
   1181  1.46  yamaguch 	for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
   1182  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1183  1.46  yamaguch 
   1184  1.46  yamaguch 		/* Have to set false before vioif_populate_rx_mbufs */
   1185  1.61  yamaguch 		mutex_enter(rxq->rxq_lock);
   1186  1.46  yamaguch 		rxq->rxq_stopping = false;
   1187  1.66   reinoud 		vioif_populate_rx_mbufs_locked(sc, rxq);
   1188  1.61  yamaguch 		mutex_exit(rxq->rxq_lock);
   1189  1.61  yamaguch 
   1190  1.46  yamaguch 	}
   1191  1.46  yamaguch 
   1192  1.46  yamaguch 	virtio_reinit_end(vsc);
   1193  1.47  yamaguch 
   1194  1.47  yamaguch 	if (sc->sc_has_ctrl)
   1195  1.43  yamaguch 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
   1196  1.46  yamaguch 
   1197  1.46  yamaguch 	r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
   1198  1.46  yamaguch 	if (r == 0)
   1199  1.46  yamaguch 		sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
   1200  1.46  yamaguch 	else
   1201  1.46  yamaguch 		sc->sc_act_nvq_pairs = 1;
   1202  1.46  yamaguch 
   1203  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++)
   1204  1.46  yamaguch 		sc->sc_txq[i].txq_stopping = false;
   1205  1.46  yamaguch 
   1206  1.46  yamaguch 	vioif_enable_interrupt_vqpairs(sc);
   1207  1.33     ozaki 
   1208  1.34     ozaki 	vioif_update_link_status(sc);
   1209   1.1   hannken 	ifp->if_flags |= IFF_RUNNING;
   1210   1.1   hannken 	ifp->if_flags &= ~IFF_OACTIVE;
   1211  1.75  yamaguch 	r = vioif_rx_filter(sc);
   1212   1.1   hannken 
   1213  1.75  yamaguch 	return r;
   1214   1.1   hannken }
   1215   1.1   hannken 
   1216   1.1   hannken static void
   1217  1.77  yamaguch vioif_stop(struct ifnet *ifp, int disable)
   1218  1.76  yamaguch {
   1219  1.77  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1220  1.77  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1221  1.76  yamaguch 	struct vioif_txqueue *txq;
   1222  1.76  yamaguch 	struct vioif_rxqueue *rxq;
   1223  1.77  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1224  1.76  yamaguch 	int i;
   1225  1.76  yamaguch 
   1226  1.77  yamaguch 	/* disable interrupts */
   1227  1.77  yamaguch 	vioif_disable_interrupt_vqpairs(sc);
   1228  1.77  yamaguch 	if (sc->sc_has_ctrl)
   1229  1.77  yamaguch 		virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
   1230  1.77  yamaguch 
   1231  1.76  yamaguch 	/*
   1232  1.76  yamaguch 	 * stop all packet processing:
   1233  1.77  yamaguch 	 * 1. stop interrupt handlers by rxq_stopping and txq_stopping
   1234  1.77  yamaguch 	 * 2. wait for stoping workqueue for packet processing
   1235  1.76  yamaguch 	 */
   1236  1.76  yamaguch 	for (i =0; i < sc->sc_act_nvq_pairs; i++) {
   1237  1.76  yamaguch 		txq = &sc->sc_txq[i];
   1238  1.76  yamaguch 		rxq = &sc->sc_rxq[i];
   1239  1.76  yamaguch 
   1240  1.76  yamaguch 		mutex_enter(rxq->rxq_lock);
   1241  1.77  yamaguch 		rxq->rxq_stopping = true;
   1242  1.76  yamaguch 		mutex_exit(rxq->rxq_lock);
   1243  1.76  yamaguch 		vioif_work_wait(sc->sc_txrx_workqueue, &rxq->rxq_work);
   1244  1.76  yamaguch 
   1245  1.76  yamaguch 		mutex_enter(txq->txq_lock);
   1246  1.77  yamaguch 		txq->txq_stopping = true;
   1247  1.76  yamaguch 		mutex_exit(txq->txq_lock);
   1248  1.76  yamaguch 		vioif_work_wait(sc->sc_txrx_workqueue, &txq->txq_work);
   1249  1.76  yamaguch 	}
   1250  1.33     ozaki 
   1251   1.1   hannken 	/* only way to stop I/O and DMA is resetting... */
   1252   1.1   hannken 	virtio_reset(vsc);
   1253  1.55  yamaguch 
   1254  1.55  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1255  1.55  yamaguch 		vioif_rx_queue_clear(&sc->sc_rxq[i]);
   1256  1.55  yamaguch 		vioif_tx_queue_clear(&sc->sc_txq[i]);
   1257  1.55  yamaguch 	}
   1258  1.46  yamaguch 
   1259   1.1   hannken 	ifp->if_flags &= ~(IFF_RUNNING | IFF_OACTIVE);
   1260  1.34     ozaki 	sc->sc_link_active = false;
   1261   1.1   hannken 
   1262  1.46  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1263  1.46  yamaguch 		txq = &sc->sc_txq[i];
   1264  1.46  yamaguch 		rxq = &sc->sc_rxq[i];
   1265  1.46  yamaguch 
   1266  1.46  yamaguch 		txq->txq_link_active = false;
   1267  1.46  yamaguch 
   1268  1.46  yamaguch 		if (disable)
   1269  1.46  yamaguch 			vioif_rx_drain(rxq);
   1270  1.46  yamaguch 
   1271  1.46  yamaguch 		vioif_tx_drain(txq);
   1272  1.46  yamaguch 	}
   1273   1.1   hannken }
   1274   1.1   hannken 
   1275   1.1   hannken static void
   1276  1.48   msaitoh vioif_send_common_locked(struct ifnet *ifp, struct vioif_txqueue *txq,
   1277  1.48   msaitoh     bool is_transmit)
   1278   1.1   hannken {
   1279   1.1   hannken 	struct vioif_softc *sc = ifp->if_softc;
   1280   1.1   hannken 	struct virtio_softc *vsc = sc->sc_virtio;
   1281  1.43  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1282  1.66   reinoud 	struct virtio_net_hdr *hdr;
   1283   1.1   hannken 	struct mbuf *m;
   1284  1.36  jdolecek 	int queued = 0;
   1285   1.1   hannken 
   1286  1.46  yamaguch 	KASSERT(mutex_owned(txq->txq_lock));
   1287  1.46  yamaguch 
   1288  1.46  yamaguch 	if ((ifp->if_flags & IFF_RUNNING) == 0)
   1289  1.46  yamaguch 		return;
   1290   1.7     ozaki 
   1291  1.46  yamaguch 	if (!txq->txq_link_active || txq->txq_stopping)
   1292  1.46  yamaguch 		return;
   1293   1.7     ozaki 
   1294  1.46  yamaguch 	if ((ifp->if_flags & IFF_OACTIVE) != 0 && !is_transmit)
   1295  1.46  yamaguch 		return;
   1296   1.1   hannken 
   1297   1.2  jmcneill 	for (;;) {
   1298   1.1   hannken 		int slot, r;
   1299   1.1   hannken 
   1300  1.46  yamaguch 		if (is_transmit)
   1301  1.46  yamaguch 			m = pcq_get(txq->txq_intrq);
   1302  1.46  yamaguch 		else
   1303  1.46  yamaguch 			IFQ_DEQUEUE(&ifp->if_snd, m);
   1304  1.46  yamaguch 
   1305   1.2  jmcneill 		if (m == NULL)
   1306   1.2  jmcneill 			break;
   1307   1.2  jmcneill 
   1308   1.1   hannken 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1309   1.1   hannken 		if (r == EAGAIN) {
   1310   1.1   hannken 			ifp->if_flags |= IFF_OACTIVE;
   1311  1.36  jdolecek 			m_freem(m);
   1312  1.36  jdolecek 			break;
   1313   1.1   hannken 		}
   1314   1.1   hannken 		if (r != 0)
   1315   1.1   hannken 			panic("enqueue_prep for a tx buffer");
   1316  1.36  jdolecek 
   1317  1.32  jdolecek 		r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1318  1.50  christos 		    txq->txq_dmamaps[slot], m, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   1319   1.1   hannken 		if (r != 0) {
   1320  1.36  jdolecek 			/* maybe just too fragmented */
   1321  1.36  jdolecek 			struct mbuf *newm;
   1322  1.36  jdolecek 
   1323  1.36  jdolecek 			newm = m_defrag(m, M_NOWAIT);
   1324  1.36  jdolecek 			if (newm == NULL) {
   1325  1.63  yamaguch 				txq->txq_defrag_failed.ev_count++;
   1326  1.36  jdolecek 				goto skip;
   1327  1.36  jdolecek 			}
   1328  1.36  jdolecek 
   1329  1.37  jdolecek 			m = newm;
   1330  1.36  jdolecek 			r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1331  1.50  christos 			    txq->txq_dmamaps[slot], m,
   1332  1.50  christos 			    BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   1333  1.36  jdolecek 			if (r != 0) {
   1334  1.63  yamaguch 				txq->txq_mbuf_load_failed.ev_count++;
   1335  1.36  jdolecek skip:
   1336  1.37  jdolecek 				m_freem(m);
   1337  1.36  jdolecek 				virtio_enqueue_abort(vsc, vq, slot);
   1338  1.36  jdolecek 				continue;
   1339  1.36  jdolecek 			}
   1340   1.1   hannken 		}
   1341  1.36  jdolecek 
   1342  1.36  jdolecek 		/* This should actually never fail */
   1343   1.1   hannken 		r = virtio_enqueue_reserve(vsc, vq, slot,
   1344  1.63  yamaguch 		    txq->txq_dmamaps[slot]->dm_nsegs + 1);
   1345   1.1   hannken 		if (r != 0) {
   1346  1.63  yamaguch 			txq->txq_enqueue_reserve_failed.ev_count++;
   1347  1.32  jdolecek 			bus_dmamap_unload(virtio_dmat(vsc),
   1348  1.63  yamaguch 			     txq->txq_dmamaps[slot]);
   1349  1.36  jdolecek 			/* slot already freed by virtio_enqueue_reserve */
   1350  1.37  jdolecek 			m_freem(m);
   1351  1.36  jdolecek 			continue;
   1352   1.1   hannken 		}
   1353   1.7     ozaki 
   1354  1.43  yamaguch 		txq->txq_mbufs[slot] = m;
   1355   1.1   hannken 
   1356  1.66   reinoud 		hdr = &txq->txq_hdrs[slot];
   1357  1.66   reinoud 		memset(hdr, 0, sc->sc_hdr_size);
   1358  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
   1359  1.50  christos 		    0, txq->txq_dmamaps[slot]->dm_mapsize,
   1360  1.50  christos 		    BUS_DMASYNC_PREWRITE);
   1361  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
   1362  1.50  christos 		    0, txq->txq_hdr_dmamaps[slot]->dm_mapsize,
   1363  1.50  christos 		    BUS_DMASYNC_PREWRITE);
   1364  1.43  yamaguch 		virtio_enqueue(vsc, vq, slot, txq->txq_hdr_dmamaps[slot], true);
   1365  1.43  yamaguch 		virtio_enqueue(vsc, vq, slot, txq->txq_dmamaps[slot], true);
   1366   1.1   hannken 		virtio_enqueue_commit(vsc, vq, slot, false);
   1367  1.36  jdolecek 
   1368   1.1   hannken 		queued++;
   1369  1.41   msaitoh 		bpf_mtap(ifp, m, BPF_D_OUT);
   1370   1.1   hannken 	}
   1371   1.1   hannken 
   1372   1.1   hannken 	if (queued > 0) {
   1373   1.1   hannken 		virtio_enqueue_commit(vsc, vq, -1, true);
   1374   1.1   hannken 		ifp->if_timer = 5;
   1375   1.1   hannken 	}
   1376  1.46  yamaguch }
   1377  1.46  yamaguch 
   1378  1.46  yamaguch static void
   1379  1.46  yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
   1380  1.46  yamaguch {
   1381  1.46  yamaguch 
   1382  1.46  yamaguch 	/*
   1383  1.46  yamaguch 	 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
   1384  1.46  yamaguch 	 */
   1385  1.46  yamaguch 	vioif_send_common_locked(ifp, txq, false);
   1386  1.46  yamaguch 
   1387  1.46  yamaguch }
   1388  1.46  yamaguch 
   1389  1.46  yamaguch static void
   1390  1.46  yamaguch vioif_start(struct ifnet *ifp)
   1391  1.46  yamaguch {
   1392  1.46  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1393  1.46  yamaguch 	struct vioif_txqueue *txq = &sc->sc_txq[0];
   1394  1.46  yamaguch 
   1395  1.46  yamaguch #ifdef VIOIF_MPSAFE
   1396  1.46  yamaguch 	KASSERT(if_is_mpsafe(ifp));
   1397  1.46  yamaguch #endif
   1398  1.46  yamaguch 
   1399  1.46  yamaguch 	mutex_enter(txq->txq_lock);
   1400  1.59  yamaguch 	vioif_start_locked(ifp, txq);
   1401  1.46  yamaguch 	mutex_exit(txq->txq_lock);
   1402  1.46  yamaguch }
   1403  1.46  yamaguch 
   1404  1.46  yamaguch static inline int
   1405  1.46  yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
   1406  1.46  yamaguch {
   1407  1.46  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1408  1.46  yamaguch 	u_int cpuid = cpu_index(curcpu());
   1409  1.46  yamaguch 
   1410  1.46  yamaguch 	return cpuid % sc->sc_act_nvq_pairs;
   1411  1.46  yamaguch }
   1412  1.46  yamaguch 
   1413  1.46  yamaguch static void
   1414  1.46  yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_txqueue *txq)
   1415  1.46  yamaguch {
   1416  1.46  yamaguch 
   1417  1.46  yamaguch 	vioif_send_common_locked(ifp, txq, true);
   1418  1.46  yamaguch }
   1419  1.46  yamaguch 
   1420  1.46  yamaguch static int
   1421  1.46  yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
   1422  1.46  yamaguch {
   1423  1.46  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1424  1.46  yamaguch 	struct vioif_txqueue *txq;
   1425  1.46  yamaguch 	int qid;
   1426  1.46  yamaguch 
   1427  1.46  yamaguch 	qid = vioif_select_txqueue(ifp, m);
   1428  1.46  yamaguch 	txq = &sc->sc_txq[qid];
   1429  1.46  yamaguch 
   1430  1.46  yamaguch 	if (__predict_false(!pcq_put(txq->txq_intrq, m))) {
   1431  1.46  yamaguch 		m_freem(m);
   1432  1.46  yamaguch 		return ENOBUFS;
   1433  1.46  yamaguch 	}
   1434  1.46  yamaguch 
   1435  1.52   thorpej 	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
   1436  1.52   thorpej 	if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
   1437  1.46  yamaguch 	if (m->m_flags & M_MCAST)
   1438  1.52   thorpej 		if_statinc_ref(nsr, if_omcasts);
   1439  1.52   thorpej 	IF_STAT_PUTREF(ifp);
   1440  1.46  yamaguch 
   1441  1.46  yamaguch 	if (mutex_tryenter(txq->txq_lock)) {
   1442  1.59  yamaguch 		vioif_transmit_locked(ifp, txq);
   1443  1.46  yamaguch 		mutex_exit(txq->txq_lock);
   1444  1.46  yamaguch 	}
   1445  1.46  yamaguch 
   1446  1.46  yamaguch 	return 0;
   1447  1.46  yamaguch }
   1448  1.46  yamaguch 
   1449  1.46  yamaguch static void
   1450  1.46  yamaguch vioif_deferred_transmit(void *arg)
   1451  1.46  yamaguch {
   1452  1.46  yamaguch 	struct vioif_txqueue *txq = arg;
   1453  1.46  yamaguch 	struct virtio_softc *vsc = txq->txq_vq->vq_owner;
   1454  1.46  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1455  1.46  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1456   1.7     ozaki 
   1457  1.60  yamaguch 	mutex_enter(txq->txq_lock);
   1458  1.60  yamaguch 	vioif_send_common_locked(ifp, txq, true);
   1459  1.60  yamaguch 	mutex_exit(txq->txq_lock);
   1460   1.1   hannken }
   1461   1.1   hannken 
   1462   1.1   hannken static int
   1463   1.1   hannken vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
   1464   1.1   hannken {
   1465   1.1   hannken 	int s, r;
   1466   1.1   hannken 
   1467   1.1   hannken 	s = splnet();
   1468   1.1   hannken 
   1469   1.1   hannken 	r = ether_ioctl(ifp, cmd, data);
   1470  1.75  yamaguch 	if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) {
   1471  1.75  yamaguch 		if (ifp->if_flags & IFF_RUNNING) {
   1472   1.1   hannken 			r = vioif_rx_filter(ifp->if_softc);
   1473  1.75  yamaguch 		} else {
   1474   1.1   hannken 			r = 0;
   1475  1.75  yamaguch 		}
   1476   1.1   hannken 	}
   1477   1.1   hannken 
   1478   1.1   hannken 	splx(s);
   1479   1.1   hannken 
   1480   1.1   hannken 	return r;
   1481   1.1   hannken }
   1482   1.1   hannken 
   1483   1.1   hannken void
   1484   1.1   hannken vioif_watchdog(struct ifnet *ifp)
   1485   1.1   hannken {
   1486   1.1   hannken 	struct vioif_softc *sc = ifp->if_softc;
   1487  1.46  yamaguch 	int i;
   1488   1.1   hannken 
   1489  1.46  yamaguch 	if (ifp->if_flags & IFF_RUNNING) {
   1490  1.55  yamaguch 		for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   1491  1.55  yamaguch 			vioif_tx_queue_clear(&sc->sc_txq[i]);
   1492  1.55  yamaguch 		}
   1493  1.46  yamaguch 	}
   1494   1.1   hannken }
   1495   1.1   hannken 
   1496   1.1   hannken /*
   1497  1.39  dholland  * Receive implementation
   1498   1.1   hannken  */
   1499  1.39  dholland /* allocate and initialize a mbuf for receive */
   1500   1.1   hannken static int
   1501  1.46  yamaguch vioif_add_rx_mbuf(struct vioif_rxqueue *rxq, int i)
   1502   1.1   hannken {
   1503  1.46  yamaguch 	struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
   1504   1.1   hannken 	struct mbuf *m;
   1505   1.1   hannken 	int r;
   1506   1.1   hannken 
   1507   1.1   hannken 	MGETHDR(m, M_DONTWAIT, MT_DATA);
   1508   1.1   hannken 	if (m == NULL)
   1509   1.1   hannken 		return ENOBUFS;
   1510   1.1   hannken 	MCLGET(m, M_DONTWAIT);
   1511   1.1   hannken 	if ((m->m_flags & M_EXT) == 0) {
   1512   1.1   hannken 		m_freem(m);
   1513   1.1   hannken 		return ENOBUFS;
   1514   1.1   hannken 	}
   1515  1.43  yamaguch 	rxq->rxq_mbufs[i] = m;
   1516   1.1   hannken 	m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
   1517  1.46  yamaguch 	r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1518  1.50  christos 	    rxq->rxq_dmamaps[i], m, BUS_DMA_READ | BUS_DMA_NOWAIT);
   1519   1.1   hannken 	if (r) {
   1520   1.1   hannken 		m_freem(m);
   1521  1.46  yamaguch 		rxq->rxq_mbufs[i] = NULL;
   1522   1.1   hannken 		return r;
   1523   1.1   hannken 	}
   1524   1.1   hannken 
   1525   1.1   hannken 	return 0;
   1526   1.1   hannken }
   1527   1.1   hannken 
   1528  1.39  dholland /* free a mbuf for receive */
   1529   1.1   hannken static void
   1530  1.46  yamaguch vioif_free_rx_mbuf(struct vioif_rxqueue *rxq, int i)
   1531   1.1   hannken {
   1532  1.46  yamaguch 	struct virtio_softc *vsc = rxq->rxq_vq->vq_owner;
   1533  1.43  yamaguch 
   1534  1.46  yamaguch 	bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[i]);
   1535  1.43  yamaguch 	m_freem(rxq->rxq_mbufs[i]);
   1536  1.43  yamaguch 	rxq->rxq_mbufs[i] = NULL;
   1537   1.1   hannken }
   1538   1.1   hannken 
   1539  1.39  dholland /* add mbufs for all the empty receive slots */
   1540   1.1   hannken static void
   1541  1.66   reinoud vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
   1542  1.12     ozaki {
   1543  1.46  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1544  1.46  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1545   1.1   hannken 	int i, r, ndone = 0;
   1546   1.1   hannken 
   1547  1.46  yamaguch 	KASSERT(mutex_owned(rxq->rxq_lock));
   1548   1.7     ozaki 
   1549  1.43  yamaguch 	if (rxq->rxq_stopping)
   1550  1.12     ozaki 		return;
   1551   1.7     ozaki 
   1552   1.1   hannken 	for (i = 0; i < vq->vq_num; i++) {
   1553   1.1   hannken 		int slot;
   1554   1.1   hannken 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1555   1.1   hannken 		if (r == EAGAIN)
   1556   1.1   hannken 			break;
   1557   1.1   hannken 		if (r != 0)
   1558   1.1   hannken 			panic("enqueue_prep for rx buffers");
   1559  1.43  yamaguch 		if (rxq->rxq_mbufs[slot] == NULL) {
   1560  1.46  yamaguch 			r = vioif_add_rx_mbuf(rxq, slot);
   1561   1.1   hannken 			if (r != 0) {
   1562  1.63  yamaguch 				rxq->rxq_mbuf_add_failed.ev_count++;
   1563   1.1   hannken 				break;
   1564   1.1   hannken 			}
   1565   1.1   hannken 		}
   1566   1.1   hannken 		r = virtio_enqueue_reserve(vsc, vq, slot,
   1567  1.50  christos 		    rxq->rxq_dmamaps[slot]->dm_nsegs + 1);
   1568   1.1   hannken 		if (r != 0) {
   1569  1.46  yamaguch 			vioif_free_rx_mbuf(rxq, slot);
   1570   1.1   hannken 			break;
   1571   1.1   hannken 		}
   1572  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
   1573  1.66   reinoud 		    0, sc->sc_hdr_size, BUS_DMASYNC_PREREAD);
   1574  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
   1575  1.50  christos 		    0, MCLBYTES, BUS_DMASYNC_PREREAD);
   1576  1.50  christos 		virtio_enqueue(vsc, vq, slot, rxq->rxq_hdr_dmamaps[slot],
   1577  1.50  christos 		    false);
   1578  1.43  yamaguch 		virtio_enqueue(vsc, vq, slot, rxq->rxq_dmamaps[slot], false);
   1579   1.1   hannken 		virtio_enqueue_commit(vsc, vq, slot, false);
   1580   1.1   hannken 		ndone++;
   1581   1.1   hannken 	}
   1582   1.1   hannken 	if (ndone > 0)
   1583   1.1   hannken 		virtio_enqueue_commit(vsc, vq, -1, true);
   1584   1.1   hannken }
   1585   1.1   hannken 
   1586  1.55  yamaguch static void
   1587  1.55  yamaguch vioif_rx_queue_clear(struct vioif_rxqueue *rxq)
   1588   1.1   hannken {
   1589  1.55  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1590  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1591  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1592  1.55  yamaguch 	u_int limit = UINT_MAX;
   1593  1.55  yamaguch 	bool more;
   1594   1.7     ozaki 
   1595  1.43  yamaguch 	KASSERT(rxq->rxq_stopping);
   1596   1.7     ozaki 
   1597  1.46  yamaguch 	mutex_enter(rxq->rxq_lock);
   1598  1.55  yamaguch 	for (;;) {
   1599  1.55  yamaguch 		more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
   1600  1.55  yamaguch 		if (more == false)
   1601  1.55  yamaguch 			break;
   1602  1.55  yamaguch 	}
   1603  1.46  yamaguch 	mutex_exit(rxq->rxq_lock);
   1604   1.7     ozaki }
   1605   1.7     ozaki 
   1606  1.39  dholland /* dequeue received packets */
   1607  1.55  yamaguch static bool
   1608  1.55  yamaguch vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   1609  1.55  yamaguch     struct vioif_rxqueue *rxq, u_int limit)
   1610   1.7     ozaki {
   1611  1.43  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1612   1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1613   1.1   hannken 	struct mbuf *m;
   1614   1.1   hannken 	int slot, len;
   1615  1.55  yamaguch 	bool more = false, dequeued = false;
   1616   1.1   hannken 
   1617  1.46  yamaguch 	KASSERT(mutex_owned(rxq->rxq_lock));
   1618   1.7     ozaki 
   1619  1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   1620  1.55  yamaguch 		return false;
   1621  1.55  yamaguch 
   1622  1.55  yamaguch 	for (;;) {
   1623  1.55  yamaguch 		if (limit-- == 0) {
   1624  1.55  yamaguch 			more = true;
   1625  1.55  yamaguch 			break;
   1626  1.55  yamaguch 		}
   1627  1.55  yamaguch 
   1628  1.55  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   1629  1.55  yamaguch 			break;
   1630  1.55  yamaguch 
   1631  1.55  yamaguch 		dequeued = true;
   1632  1.54  yamaguch 
   1633  1.66   reinoud 		len -= sc->sc_hdr_size;
   1634  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_hdr_dmamaps[slot],
   1635  1.66   reinoud 		    0, sc->sc_hdr_size, BUS_DMASYNC_POSTREAD);
   1636  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), rxq->rxq_dmamaps[slot],
   1637  1.50  christos 		    0, MCLBYTES, BUS_DMASYNC_POSTREAD);
   1638  1.43  yamaguch 		m = rxq->rxq_mbufs[slot];
   1639   1.1   hannken 		KASSERT(m != NULL);
   1640  1.43  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), rxq->rxq_dmamaps[slot]);
   1641  1.46  yamaguch 		rxq->rxq_mbufs[slot] = NULL;
   1642   1.1   hannken 		virtio_dequeue_commit(vsc, vq, slot);
   1643  1.24     ozaki 		m_set_rcvif(m, ifp);
   1644   1.1   hannken 		m->m_len = m->m_pkthdr.len = len;
   1645   1.7     ozaki 
   1646  1.46  yamaguch 		mutex_exit(rxq->rxq_lock);
   1647  1.22     ozaki 		if_percpuq_enqueue(ifp->if_percpuq, m);
   1648  1.46  yamaguch 		mutex_enter(rxq->rxq_lock);
   1649   1.7     ozaki 
   1650  1.43  yamaguch 		if (rxq->rxq_stopping)
   1651   1.7     ozaki 			break;
   1652   1.1   hannken 	}
   1653   1.3  christos 
   1654  1.55  yamaguch 	if (dequeued)
   1655  1.66   reinoud 		vioif_populate_rx_mbufs_locked(sc, rxq);
   1656  1.55  yamaguch 
   1657  1.55  yamaguch 	return more;
   1658   1.1   hannken }
   1659   1.1   hannken 
   1660   1.1   hannken /* rx interrupt; call _dequeue above and schedule a softint */
   1661  1.66   reinoud 
   1662  1.66   reinoud static void
   1663  1.66   reinoud vioif_rx_handle_locked(void *xrxq, u_int limit)
   1664  1.66   reinoud {
   1665  1.66   reinoud 	struct vioif_rxqueue *rxq = xrxq;
   1666  1.66   reinoud 	struct virtqueue *vq = rxq->rxq_vq;
   1667  1.66   reinoud 	struct virtio_softc *vsc = vq->vq_owner;
   1668  1.66   reinoud 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1669  1.66   reinoud 	bool more;
   1670  1.66   reinoud 
   1671  1.66   reinoud 	KASSERT(!rxq->rxq_stopping);
   1672  1.66   reinoud 
   1673  1.66   reinoud 	more = vioif_rx_deq_locked(sc, vsc, rxq, limit);
   1674  1.66   reinoud 	if (more) {
   1675  1.66   reinoud 		vioif_rx_sched_handle(sc, rxq);
   1676  1.66   reinoud 		return;
   1677  1.70     skrll 	}
   1678  1.66   reinoud 	more = virtio_start_vq_intr(vsc, rxq->rxq_vq);
   1679  1.66   reinoud 	if (more) {
   1680  1.66   reinoud 		vioif_rx_sched_handle(sc, rxq);
   1681  1.66   reinoud 		return;
   1682  1.70     skrll 	}
   1683  1.66   reinoud 	atomic_store_relaxed(&rxq->rxq_active, false);
   1684  1.66   reinoud }
   1685  1.66   reinoud 
   1686   1.1   hannken static int
   1687  1.54  yamaguch vioif_rx_intr(void *arg)
   1688   1.1   hannken {
   1689  1.54  yamaguch 	struct vioif_rxqueue *rxq = arg;
   1690  1.55  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1691  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1692  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1693  1.55  yamaguch 	u_int limit;
   1694  1.55  yamaguch 
   1695  1.55  yamaguch 	limit = sc->sc_rx_intr_process_limit;
   1696  1.55  yamaguch 
   1697  1.55  yamaguch 	if (atomic_load_relaxed(&rxq->rxq_active) == true)
   1698  1.55  yamaguch 		return 1;
   1699   1.7     ozaki 
   1700  1.46  yamaguch 	mutex_enter(rxq->rxq_lock);
   1701   1.7     ozaki 
   1702  1.55  yamaguch 	if (!rxq->rxq_stopping) {
   1703  1.55  yamaguch 		rxq->rxq_workqueue = sc->sc_txrx_workqueue_sysctl;
   1704  1.55  yamaguch 
   1705  1.55  yamaguch 		virtio_stop_vq_intr(vsc, vq);
   1706  1.55  yamaguch 		atomic_store_relaxed(&rxq->rxq_active, true);
   1707  1.55  yamaguch 
   1708  1.66   reinoud 		vioif_rx_handle_locked(rxq, limit);
   1709  1.55  yamaguch 	}
   1710  1.55  yamaguch 
   1711  1.55  yamaguch 	mutex_exit(rxq->rxq_lock);
   1712  1.55  yamaguch 	return 1;
   1713  1.55  yamaguch }
   1714  1.55  yamaguch 
   1715  1.55  yamaguch static void
   1716  1.55  yamaguch vioif_rx_handle(void *xrxq)
   1717  1.55  yamaguch {
   1718  1.55  yamaguch 	struct vioif_rxqueue *rxq = xrxq;
   1719  1.55  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1720  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1721  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1722  1.55  yamaguch 	u_int limit;
   1723  1.55  yamaguch 
   1724  1.55  yamaguch 	limit = sc->sc_rx_process_limit;
   1725  1.55  yamaguch 
   1726  1.55  yamaguch 	mutex_enter(rxq->rxq_lock);
   1727   1.1   hannken 
   1728  1.66   reinoud 	if (!rxq->rxq_stopping)
   1729  1.66   reinoud 		vioif_rx_handle_locked(rxq, limit);
   1730   1.1   hannken 
   1731  1.46  yamaguch 	mutex_exit(rxq->rxq_lock);
   1732  1.55  yamaguch }
   1733  1.55  yamaguch 
   1734  1.55  yamaguch static void
   1735  1.55  yamaguch vioif_rx_sched_handle(struct vioif_softc *sc, struct vioif_rxqueue *rxq)
   1736  1.55  yamaguch {
   1737  1.55  yamaguch 
   1738  1.72  yamaguch 	KASSERT(mutex_owned(rxq->rxq_lock));
   1739  1.72  yamaguch 
   1740  1.72  yamaguch 	if (rxq->rxq_stopping)
   1741  1.72  yamaguch 		return;
   1742  1.72  yamaguch 
   1743  1.55  yamaguch 	if (rxq->rxq_workqueue)
   1744  1.55  yamaguch 		vioif_work_add(sc->sc_txrx_workqueue, &rxq->rxq_work);
   1745  1.55  yamaguch 	else
   1746  1.55  yamaguch 		softint_schedule(rxq->rxq_handle_si);
   1747   1.1   hannken }
   1748   1.1   hannken 
   1749   1.1   hannken /* free all the mbufs; called from if_stop(disable) */
   1750   1.1   hannken static void
   1751  1.46  yamaguch vioif_rx_drain(struct vioif_rxqueue *rxq)
   1752   1.1   hannken {
   1753  1.43  yamaguch 	struct virtqueue *vq = rxq->rxq_vq;
   1754   1.1   hannken 	int i;
   1755   1.1   hannken 
   1756   1.1   hannken 	for (i = 0; i < vq->vq_num; i++) {
   1757  1.43  yamaguch 		if (rxq->rxq_mbufs[i] == NULL)
   1758   1.1   hannken 			continue;
   1759  1.46  yamaguch 		vioif_free_rx_mbuf(rxq, i);
   1760   1.1   hannken 	}
   1761   1.1   hannken }
   1762   1.1   hannken 
   1763   1.1   hannken /*
   1764   1.1   hannken  * Transmition implementation
   1765   1.1   hannken  */
   1766   1.1   hannken /* actual transmission is done in if_start */
   1767   1.1   hannken /* tx interrupt; dequeue and free mbufs */
   1768   1.1   hannken /*
   1769   1.1   hannken  * tx interrupt is actually disabled; this should be called upon
   1770   1.1   hannken  * tx vq full and watchdog
   1771   1.1   hannken  */
   1772  1.55  yamaguch 
   1773  1.66   reinoud static void
   1774  1.66   reinoud vioif_tx_handle_locked(struct vioif_txqueue *txq, u_int limit)
   1775  1.66   reinoud {
   1776  1.66   reinoud 	struct virtqueue *vq = txq->txq_vq;
   1777  1.66   reinoud 	struct virtio_softc *vsc = vq->vq_owner;
   1778  1.66   reinoud 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1779  1.66   reinoud 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1780  1.66   reinoud 	bool more;
   1781  1.66   reinoud 
   1782  1.66   reinoud 	KASSERT(!txq->txq_stopping);
   1783  1.66   reinoud 
   1784  1.66   reinoud 	more = vioif_tx_deq_locked(sc, vsc, txq, limit);
   1785  1.66   reinoud 	if (more) {
   1786  1.66   reinoud 		vioif_tx_sched_handle(sc, txq);
   1787  1.66   reinoud 		return;
   1788  1.66   reinoud 	}
   1789  1.66   reinoud 
   1790  1.70     skrll 	if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
   1791  1.66   reinoud 		more = virtio_postpone_intr_smart(vsc, vq);
   1792  1.66   reinoud 	else
   1793  1.66   reinoud 		more = virtio_start_vq_intr(vsc, vq);
   1794  1.66   reinoud 	if (more) {
   1795  1.66   reinoud 		vioif_tx_sched_handle(sc, txq);
   1796  1.66   reinoud 		return;
   1797  1.66   reinoud 	}
   1798  1.66   reinoud 
   1799  1.66   reinoud 	atomic_store_relaxed(&txq->txq_active, false);
   1800  1.66   reinoud 	/* for ALTQ */
   1801  1.66   reinoud 	if (txq == &sc->sc_txq[0]) {
   1802  1.66   reinoud 		if_schedule_deferred_start(ifp);
   1803  1.66   reinoud 		ifp->if_flags &= ~IFF_OACTIVE;
   1804  1.66   reinoud 	}
   1805  1.66   reinoud 	softint_schedule(txq->txq_deferred_transmit);
   1806  1.66   reinoud }
   1807  1.66   reinoud 
   1808  1.66   reinoud 
   1809   1.1   hannken static int
   1810  1.54  yamaguch vioif_tx_intr(void *arg)
   1811   1.1   hannken {
   1812  1.54  yamaguch 	struct vioif_txqueue *txq = arg;
   1813  1.54  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1814   1.1   hannken 	struct virtio_softc *vsc = vq->vq_owner;
   1815  1.32  jdolecek 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1816  1.55  yamaguch 	u_int limit;
   1817  1.55  yamaguch 
   1818  1.55  yamaguch 	limit = sc->sc_tx_intr_process_limit;
   1819  1.55  yamaguch 
   1820  1.55  yamaguch 	if (atomic_load_relaxed(&txq->txq_active) == true)
   1821  1.55  yamaguch 		return 1;
   1822   1.7     ozaki 
   1823  1.46  yamaguch 	mutex_enter(txq->txq_lock);
   1824   1.7     ozaki 
   1825  1.55  yamaguch 	if (!txq->txq_stopping) {
   1826  1.55  yamaguch 		txq->txq_workqueue = sc->sc_txrx_workqueue_sysctl;
   1827  1.55  yamaguch 
   1828  1.55  yamaguch 		virtio_stop_vq_intr(vsc, vq);
   1829  1.55  yamaguch 		atomic_store_relaxed(&txq->txq_active, true);
   1830  1.55  yamaguch 
   1831  1.66   reinoud 		vioif_tx_handle_locked(txq, limit);
   1832  1.55  yamaguch 	}
   1833   1.7     ozaki 
   1834  1.46  yamaguch 	mutex_exit(txq->txq_lock);
   1835  1.46  yamaguch 
   1836  1.55  yamaguch 	return 1;
   1837  1.55  yamaguch }
   1838  1.55  yamaguch 
   1839  1.55  yamaguch static void
   1840  1.55  yamaguch vioif_tx_handle(void *xtxq)
   1841  1.55  yamaguch {
   1842  1.55  yamaguch 	struct vioif_txqueue *txq = xtxq;
   1843  1.55  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1844  1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1845  1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1846  1.55  yamaguch 	u_int limit;
   1847  1.55  yamaguch 
   1848  1.55  yamaguch 	limit = sc->sc_tx_process_limit;
   1849  1.55  yamaguch 
   1850  1.55  yamaguch 	mutex_enter(txq->txq_lock);
   1851  1.66   reinoud 	if (!txq->txq_stopping)
   1852  1.66   reinoud 		vioif_tx_handle_locked(txq, limit);
   1853  1.55  yamaguch 	mutex_exit(txq->txq_lock);
   1854  1.55  yamaguch }
   1855  1.55  yamaguch 
   1856  1.55  yamaguch static void
   1857  1.55  yamaguch vioif_tx_sched_handle(struct vioif_softc *sc, struct vioif_txqueue *txq)
   1858  1.55  yamaguch {
   1859  1.55  yamaguch 
   1860  1.72  yamaguch 	KASSERT(mutex_owned(txq->txq_lock));
   1861  1.72  yamaguch 
   1862  1.72  yamaguch 	if (txq->txq_stopping)
   1863  1.72  yamaguch 		return;
   1864  1.72  yamaguch 
   1865  1.55  yamaguch 	if (txq->txq_workqueue)
   1866  1.55  yamaguch 		vioif_work_add(sc->sc_txrx_workqueue, &txq->txq_work);
   1867  1.55  yamaguch 	else
   1868  1.55  yamaguch 		softint_schedule(txq->txq_handle_si);
   1869   1.7     ozaki }
   1870   1.7     ozaki 
   1871  1.55  yamaguch static void
   1872  1.55  yamaguch vioif_tx_queue_clear(struct vioif_txqueue *txq)
   1873   1.7     ozaki {
   1874  1.55  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1875   1.7     ozaki 	struct virtio_softc *vsc = vq->vq_owner;
   1876  1.32  jdolecek 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1877  1.55  yamaguch 	u_int limit = UINT_MAX;
   1878  1.55  yamaguch 	bool more;
   1879  1.55  yamaguch 
   1880  1.55  yamaguch 	mutex_enter(txq->txq_lock);
   1881  1.55  yamaguch 	for (;;) {
   1882  1.55  yamaguch 		more = vioif_tx_deq_locked(sc, vsc, txq, limit);
   1883  1.55  yamaguch 		if (more == false)
   1884  1.55  yamaguch 			break;
   1885  1.55  yamaguch 	}
   1886  1.55  yamaguch 	mutex_exit(txq->txq_lock);
   1887  1.55  yamaguch }
   1888  1.55  yamaguch 
   1889  1.55  yamaguch static bool
   1890  1.55  yamaguch vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   1891  1.55  yamaguch     struct vioif_txqueue *txq, u_int limit)
   1892  1.55  yamaguch {
   1893  1.55  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1894   1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1895   1.1   hannken 	struct mbuf *m;
   1896   1.1   hannken 	int slot, len;
   1897  1.55  yamaguch 	bool more = false;
   1898   1.1   hannken 
   1899  1.46  yamaguch 	KASSERT(mutex_owned(txq->txq_lock));
   1900   1.7     ozaki 
   1901  1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   1902  1.55  yamaguch 		return false;
   1903  1.55  yamaguch 
   1904  1.55  yamaguch 	for (;;) {
   1905  1.55  yamaguch 		if (limit-- == 0) {
   1906  1.55  yamaguch 			more = true;
   1907  1.55  yamaguch 			break;
   1908  1.55  yamaguch 		}
   1909  1.55  yamaguch 
   1910  1.55  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   1911  1.55  yamaguch 			break;
   1912  1.54  yamaguch 
   1913  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_hdr_dmamaps[slot],
   1914  1.66   reinoud 		    0, sc->sc_hdr_size, BUS_DMASYNC_POSTWRITE);
   1915  1.43  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), txq->txq_dmamaps[slot],
   1916  1.50  christos 		    0, txq->txq_dmamaps[slot]->dm_mapsize,
   1917  1.50  christos 		    BUS_DMASYNC_POSTWRITE);
   1918  1.43  yamaguch 		m = txq->txq_mbufs[slot];
   1919  1.43  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[slot]);
   1920  1.46  yamaguch 		txq->txq_mbufs[slot] = NULL;
   1921   1.1   hannken 		virtio_dequeue_commit(vsc, vq, slot);
   1922  1.52   thorpej 		if_statinc(ifp, if_opackets);
   1923   1.1   hannken 		m_freem(m);
   1924   1.1   hannken 	}
   1925   1.1   hannken 
   1926  1.55  yamaguch 	return more;
   1927   1.1   hannken }
   1928   1.1   hannken 
   1929   1.1   hannken /* free all the mbufs already put on vq; called from if_stop(disable) */
   1930   1.1   hannken static void
   1931  1.46  yamaguch vioif_tx_drain(struct vioif_txqueue *txq)
   1932   1.1   hannken {
   1933  1.43  yamaguch 	struct virtqueue *vq = txq->txq_vq;
   1934  1.46  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1935   1.1   hannken 	int i;
   1936   1.1   hannken 
   1937  1.43  yamaguch 	KASSERT(txq->txq_stopping);
   1938   1.7     ozaki 
   1939   1.1   hannken 	for (i = 0; i < vq->vq_num; i++) {
   1940  1.43  yamaguch 		if (txq->txq_mbufs[i] == NULL)
   1941   1.1   hannken 			continue;
   1942  1.43  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), txq->txq_dmamaps[i]);
   1943  1.43  yamaguch 		m_freem(txq->txq_mbufs[i]);
   1944  1.43  yamaguch 		txq->txq_mbufs[i] = NULL;
   1945   1.1   hannken 	}
   1946   1.1   hannken }
   1947   1.1   hannken 
   1948   1.1   hannken /*
   1949   1.1   hannken  * Control vq
   1950   1.1   hannken  */
   1951   1.1   hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
   1952  1.44  yamaguch static void
   1953  1.44  yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
   1954   1.1   hannken {
   1955  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1956   1.1   hannken 
   1957  1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   1958  1.43  yamaguch 	while (ctrlq->ctrlq_inuse != FREE)
   1959  1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   1960  1.43  yamaguch 	ctrlq->ctrlq_inuse = INUSE;
   1961  1.44  yamaguch 	ctrlq->ctrlq_owner = curlwp;
   1962  1.44  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   1963  1.44  yamaguch }
   1964  1.44  yamaguch 
   1965  1.44  yamaguch static void
   1966  1.44  yamaguch vioif_ctrl_release(struct vioif_softc *sc)
   1967  1.44  yamaguch {
   1968  1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1969  1.44  yamaguch 
   1970  1.44  yamaguch 	KASSERT(ctrlq->ctrlq_inuse != FREE);
   1971  1.44  yamaguch 	KASSERT(ctrlq->ctrlq_owner == curlwp);
   1972  1.44  yamaguch 
   1973  1.44  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   1974  1.44  yamaguch 	ctrlq->ctrlq_inuse = FREE;
   1975  1.46  yamaguch 	ctrlq->ctrlq_owner = NULL;
   1976  1.44  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   1977  1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   1978  1.44  yamaguch }
   1979  1.44  yamaguch 
   1980  1.44  yamaguch static int
   1981  1.44  yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
   1982  1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   1983  1.44  yamaguch {
   1984  1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1985  1.44  yamaguch 	int i, r, loaded;
   1986  1.44  yamaguch 
   1987  1.44  yamaguch 	loaded = 0;
   1988  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   1989  1.44  yamaguch 		r = bus_dmamap_load(virtio_dmat(vsc),
   1990  1.44  yamaguch 		    specs[i].dmamap, specs[i].buf, specs[i].bufsize,
   1991  1.48   msaitoh 		    NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   1992  1.44  yamaguch 		if (r) {
   1993  1.63  yamaguch 			sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++;
   1994  1.44  yamaguch 			goto err;
   1995  1.44  yamaguch 		}
   1996  1.44  yamaguch 		loaded++;
   1997  1.44  yamaguch 
   1998  1.44  yamaguch 	}
   1999  1.44  yamaguch 
   2000  1.44  yamaguch 	return r;
   2001  1.44  yamaguch 
   2002  1.44  yamaguch err:
   2003  1.44  yamaguch 	for (i = 0; i < loaded; i++) {
   2004  1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   2005  1.44  yamaguch 	}
   2006  1.44  yamaguch 
   2007  1.44  yamaguch 	return r;
   2008  1.44  yamaguch }
   2009  1.44  yamaguch 
   2010  1.44  yamaguch static void
   2011  1.44  yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
   2012  1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2013  1.44  yamaguch {
   2014  1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2015  1.44  yamaguch 	int i;
   2016  1.44  yamaguch 
   2017  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2018  1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   2019  1.44  yamaguch 	}
   2020  1.44  yamaguch }
   2021  1.44  yamaguch 
   2022  1.44  yamaguch static int
   2023  1.44  yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
   2024  1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2025  1.44  yamaguch {
   2026  1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2027  1.44  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2028  1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2029  1.44  yamaguch 	int i, r, slot;
   2030  1.43  yamaguch 
   2031  1.44  yamaguch 	ctrlq->ctrlq_cmd->class = class;
   2032  1.43  yamaguch 	ctrlq->ctrlq_cmd->command = cmd;
   2033   1.1   hannken 
   2034  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
   2035  1.50  christos 	    0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
   2036  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2037  1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
   2038  1.50  christos 		    0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
   2039  1.44  yamaguch 	}
   2040  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
   2041  1.50  christos 	    0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
   2042   1.1   hannken 
   2043  1.66   reinoud 	/* we need to explicitly (re)start vq intr when using RING EVENT IDX */
   2044  1.66   reinoud 	if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
   2045  1.66   reinoud 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
   2046  1.66   reinoud 
   2047   1.1   hannken 	r = virtio_enqueue_prep(vsc, vq, &slot);
   2048   1.1   hannken 	if (r != 0)
   2049   1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2050  1.44  yamaguch 	r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
   2051   1.1   hannken 	if (r != 0)
   2052   1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2053  1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
   2054  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2055  1.44  yamaguch 		virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
   2056  1.44  yamaguch 	}
   2057  1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
   2058   1.1   hannken 	virtio_enqueue_commit(vsc, vq, slot, true);
   2059   1.1   hannken 
   2060   1.1   hannken 	/* wait for done */
   2061  1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2062  1.43  yamaguch 	while (ctrlq->ctrlq_inuse != DONE)
   2063  1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   2064  1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2065   1.1   hannken 	/* already dequeueued */
   2066   1.1   hannken 
   2067  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
   2068  1.50  christos 	    sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
   2069  1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2070  1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
   2071  1.50  christos 		    specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
   2072  1.44  yamaguch 	}
   2073  1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
   2074  1.50  christos 	    sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
   2075   1.1   hannken 
   2076  1.43  yamaguch 	if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
   2077   1.1   hannken 		r = 0;
   2078   1.1   hannken 	else {
   2079  1.63  yamaguch 		device_printf(sc->sc_dev, "failed setting rx mode\n");
   2080  1.63  yamaguch 		sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++;
   2081   1.1   hannken 		r = EIO;
   2082   1.1   hannken 	}
   2083   1.1   hannken 
   2084  1.44  yamaguch 	return r;
   2085  1.44  yamaguch }
   2086  1.44  yamaguch 
   2087  1.44  yamaguch static int
   2088  1.44  yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
   2089  1.44  yamaguch {
   2090  1.44  yamaguch 	struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
   2091  1.44  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2092  1.44  yamaguch 	int r;
   2093  1.44  yamaguch 
   2094  1.44  yamaguch 	if (!sc->sc_has_ctrl)
   2095  1.44  yamaguch 		return ENOTSUP;
   2096  1.44  yamaguch 
   2097  1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2098  1.44  yamaguch 
   2099  1.44  yamaguch 	rx->onoff = onoff;
   2100  1.44  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
   2101  1.44  yamaguch 	specs[0].buf = rx;
   2102  1.44  yamaguch 	specs[0].bufsize = sizeof(*rx);
   2103  1.44  yamaguch 
   2104  1.44  yamaguch 	r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
   2105  1.44  yamaguch 	    specs, __arraycount(specs));
   2106   1.3  christos 
   2107  1.44  yamaguch 	vioif_ctrl_release(sc);
   2108   1.1   hannken 	return r;
   2109   1.1   hannken }
   2110   1.1   hannken 
   2111   1.1   hannken static int
   2112   1.1   hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
   2113   1.1   hannken {
   2114  1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
   2115   1.1   hannken }
   2116   1.1   hannken 
   2117   1.1   hannken static int
   2118   1.1   hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
   2119   1.1   hannken {
   2120  1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
   2121   1.1   hannken }
   2122   1.1   hannken 
   2123   1.1   hannken /* issue VIRTIO_NET_CTRL_MAC_TABLE_SET command and wait for completion */
   2124   1.1   hannken static int
   2125   1.1   hannken vioif_set_rx_filter(struct vioif_softc *sc)
   2126   1.1   hannken {
   2127  1.43  yamaguch 	/* filter already set in ctrlq->ctrlq_mac_tbl */
   2128  1.66   reinoud 	struct virtio_softc *vsc = sc->sc_virtio;
   2129  1.44  yamaguch 	struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
   2130  1.44  yamaguch 	struct vioif_ctrl_cmdspec specs[2];
   2131  1.44  yamaguch 	int nspecs = __arraycount(specs);
   2132  1.44  yamaguch 	int r;
   2133  1.44  yamaguch 
   2134  1.44  yamaguch 	mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
   2135  1.44  yamaguch 	mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
   2136   1.1   hannken 
   2137  1.32  jdolecek 	if (!sc->sc_has_ctrl)
   2138   1.1   hannken 		return ENOTSUP;
   2139   1.1   hannken 
   2140  1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2141   1.1   hannken 
   2142  1.44  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
   2143  1.44  yamaguch 	specs[0].buf = mac_tbl_uc;
   2144  1.44  yamaguch 	specs[0].bufsize = sizeof(*mac_tbl_uc)
   2145  1.66   reinoud 	    + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries));
   2146  1.44  yamaguch 
   2147  1.44  yamaguch 	specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
   2148  1.44  yamaguch 	specs[1].buf = mac_tbl_mc;
   2149  1.44  yamaguch 	specs[1].bufsize = sizeof(*mac_tbl_mc)
   2150  1.66   reinoud 	    + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries));
   2151   1.1   hannken 
   2152  1.44  yamaguch 	r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
   2153  1.44  yamaguch 	if (r != 0)
   2154   1.1   hannken 		goto out;
   2155   1.1   hannken 
   2156  1.44  yamaguch 	r = vioif_ctrl_send_command(sc,
   2157  1.44  yamaguch 	    VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
   2158  1.44  yamaguch 	    specs, nspecs);
   2159   1.1   hannken 
   2160  1.44  yamaguch 	vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
   2161   1.1   hannken 
   2162   1.1   hannken out:
   2163  1.44  yamaguch 	vioif_ctrl_release(sc);
   2164   1.3  christos 
   2165   1.1   hannken 	return r;
   2166   1.1   hannken }
   2167   1.1   hannken 
   2168  1.46  yamaguch static int
   2169  1.74  yamaguch vioif_set_mac_addr(struct vioif_softc *sc)
   2170  1.74  yamaguch {
   2171  1.74  yamaguch 	struct virtio_net_ctrl_mac_addr *ma =
   2172  1.74  yamaguch 	    sc->sc_ctrlq.ctrlq_mac_addr;
   2173  1.74  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2174  1.74  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2175  1.74  yamaguch 	int nspecs = __arraycount(specs);
   2176  1.78  yamaguch 	uint64_t features;
   2177  1.74  yamaguch 	int r;
   2178  1.78  yamaguch 	size_t i;
   2179  1.74  yamaguch 
   2180  1.74  yamaguch 	if (!sc->sc_has_ctrl)
   2181  1.74  yamaguch 		return ENOTSUP;
   2182  1.74  yamaguch 
   2183  1.78  yamaguch 	if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac,
   2184  1.78  yamaguch 	    ETHER_ADDR_LEN) == 0) {
   2185  1.78  yamaguch 		return 0;
   2186  1.78  yamaguch 	}
   2187  1.74  yamaguch 
   2188  1.78  yamaguch 	memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
   2189  1.74  yamaguch 
   2190  1.78  yamaguch 	features = virtio_features(sc->sc_virtio);
   2191  1.78  yamaguch 	if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) {
   2192  1.78  yamaguch 		vioif_ctrl_acquire(sc);
   2193  1.78  yamaguch 
   2194  1.78  yamaguch 		memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN);
   2195  1.78  yamaguch 		specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap;
   2196  1.78  yamaguch 		specs[0].buf = ma;
   2197  1.78  yamaguch 		specs[0].bufsize = sizeof(*ma);
   2198  1.78  yamaguch 
   2199  1.78  yamaguch 		r = vioif_ctrl_send_command(sc,
   2200  1.78  yamaguch 		    VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET,
   2201  1.78  yamaguch 		    specs, nspecs);
   2202  1.74  yamaguch 
   2203  1.78  yamaguch 		vioif_ctrl_release(sc);
   2204  1.78  yamaguch 	} else {
   2205  1.78  yamaguch 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
   2206  1.78  yamaguch 			virtio_write_device_config_1(sc->sc_virtio,
   2207  1.78  yamaguch 			    VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
   2208  1.78  yamaguch 		}
   2209  1.78  yamaguch 		r = 0;
   2210  1.78  yamaguch 	}
   2211  1.74  yamaguch 
   2212  1.74  yamaguch 	return r;
   2213  1.74  yamaguch }
   2214  1.74  yamaguch 
   2215  1.74  yamaguch static int
   2216  1.46  yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
   2217  1.46  yamaguch {
   2218  1.46  yamaguch 	struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
   2219  1.46  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2220  1.46  yamaguch 	int r;
   2221  1.46  yamaguch 
   2222  1.46  yamaguch 	if (!sc->sc_has_ctrl)
   2223  1.46  yamaguch 		return ENOTSUP;
   2224  1.46  yamaguch 
   2225  1.46  yamaguch 	if (nvq_pairs <= 1)
   2226  1.46  yamaguch 		return EINVAL;
   2227  1.46  yamaguch 
   2228  1.46  yamaguch 	vioif_ctrl_acquire(sc);
   2229  1.46  yamaguch 
   2230  1.66   reinoud 	mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs);
   2231  1.46  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
   2232  1.46  yamaguch 	specs[0].buf = mq;
   2233  1.46  yamaguch 	specs[0].bufsize = sizeof(*mq);
   2234  1.46  yamaguch 
   2235  1.46  yamaguch 	r = vioif_ctrl_send_command(sc,
   2236  1.46  yamaguch 	    VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
   2237  1.46  yamaguch 	    specs, __arraycount(specs));
   2238  1.46  yamaguch 
   2239  1.46  yamaguch 	vioif_ctrl_release(sc);
   2240  1.46  yamaguch 
   2241  1.46  yamaguch 	return r;
   2242  1.46  yamaguch }
   2243  1.46  yamaguch 
   2244   1.1   hannken /* ctrl vq interrupt; wake up the command issuer */
   2245   1.1   hannken static int
   2246  1.54  yamaguch vioif_ctrl_intr(void *arg)
   2247   1.1   hannken {
   2248  1.54  yamaguch 	struct vioif_ctrlqueue *ctrlq = arg;
   2249  1.54  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2250   1.1   hannken 	struct virtio_softc *vsc = vq->vq_owner;
   2251   1.1   hannken 	int r, slot;
   2252   1.1   hannken 
   2253  1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   2254  1.54  yamaguch 		return 0;
   2255  1.54  yamaguch 
   2256   1.1   hannken 	r = virtio_dequeue(vsc, vq, &slot, NULL);
   2257   1.1   hannken 	if (r == ENOENT)
   2258   1.1   hannken 		return 0;
   2259   1.1   hannken 	virtio_dequeue_commit(vsc, vq, slot);
   2260   1.1   hannken 
   2261  1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2262  1.43  yamaguch 	ctrlq->ctrlq_inuse = DONE;
   2263  1.43  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   2264  1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2265   1.1   hannken 
   2266   1.1   hannken 	return 1;
   2267   1.1   hannken }
   2268   1.1   hannken 
   2269  1.75  yamaguch static int
   2270  1.75  yamaguch vioif_ifflags(struct vioif_softc *sc)
   2271  1.75  yamaguch {
   2272  1.75  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2273  1.75  yamaguch 	bool onoff;
   2274  1.75  yamaguch 	int r;
   2275  1.75  yamaguch 
   2276  1.75  yamaguch 	if (!sc->sc_has_ctrl) {
   2277  1.75  yamaguch 		/* no ctrl vq; always promisc and allmulti */
   2278  1.75  yamaguch 		ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI);
   2279  1.75  yamaguch 		return 0;
   2280  1.75  yamaguch 	}
   2281  1.75  yamaguch 
   2282  1.75  yamaguch 	onoff = ifp->if_flags & IFF_ALLMULTI ? true : false;
   2283  1.75  yamaguch 	r = vioif_set_allmulti(sc, onoff);
   2284  1.75  yamaguch 	if (r != 0) {
   2285  1.75  yamaguch 		log(LOG_WARNING,
   2286  1.75  yamaguch 		    "%s: couldn't %sable ALLMULTI\n",
   2287  1.75  yamaguch 		    ifp->if_xname, onoff ? "en" : "dis");
   2288  1.75  yamaguch 		if (onoff == false) {
   2289  1.75  yamaguch 			ifp->if_flags |= IFF_ALLMULTI;
   2290  1.75  yamaguch 		}
   2291  1.75  yamaguch 	}
   2292  1.75  yamaguch 
   2293  1.75  yamaguch 	onoff = ifp->if_flags & IFF_PROMISC ? true : false;
   2294  1.75  yamaguch 	r = vioif_set_promisc(sc, onoff);
   2295  1.75  yamaguch 	if (r != 0) {
   2296  1.75  yamaguch 		log(LOG_WARNING,
   2297  1.75  yamaguch 		    "%s: couldn't %sable PROMISC\n",
   2298  1.75  yamaguch 		    ifp->if_xname, onoff ? "en" : "dis");
   2299  1.75  yamaguch 		if (onoff == false) {
   2300  1.75  yamaguch 			ifp->if_flags |= IFF_PROMISC;
   2301  1.75  yamaguch 		}
   2302  1.75  yamaguch 	}
   2303  1.75  yamaguch 
   2304  1.75  yamaguch 	return 0;
   2305  1.75  yamaguch }
   2306  1.75  yamaguch 
   2307  1.75  yamaguch static int
   2308  1.75  yamaguch vioif_ifflags_cb(struct ethercom *ec)
   2309  1.75  yamaguch {
   2310  1.75  yamaguch 	struct ifnet *ifp = &ec->ec_if;
   2311  1.75  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   2312  1.75  yamaguch 
   2313  1.75  yamaguch 	return vioif_ifflags(sc);
   2314  1.75  yamaguch }
   2315  1.75  yamaguch 
   2316   1.1   hannken /*
   2317   1.1   hannken  * If multicast filter small enough (<=MAXENTRIES) set rx filter
   2318   1.1   hannken  * If large multicast filter exist use ALLMULTI
   2319   1.1   hannken  * If setting rx filter fails fall back to ALLMULTI
   2320   1.1   hannken  */
   2321   1.1   hannken static int
   2322   1.1   hannken vioif_rx_filter(struct vioif_softc *sc)
   2323   1.1   hannken {
   2324  1.66   reinoud 	struct virtio_softc *vsc = sc->sc_virtio;
   2325  1.48   msaitoh 	struct ethercom *ec = &sc->sc_ethercom;
   2326  1.48   msaitoh 	struct ifnet *ifp = &ec->ec_if;
   2327   1.1   hannken 	struct ether_multi *enm;
   2328   1.1   hannken 	struct ether_multistep step;
   2329  1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2330   1.1   hannken 	int nentries;
   2331  1.75  yamaguch 	bool allmulti = 0;
   2332   1.1   hannken 	int r;
   2333   1.1   hannken 
   2334  1.75  yamaguch 	if (!sc->sc_has_ctrl) {
   2335  1.75  yamaguch 		goto set_ifflags;
   2336   1.1   hannken 	}
   2337   1.1   hannken 
   2338  1.74  yamaguch 	memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0],
   2339  1.74  yamaguch 	    CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
   2340  1.74  yamaguch 
   2341  1.75  yamaguch 	nentries = 0;
   2342  1.75  yamaguch 	allmulti = false;
   2343  1.75  yamaguch 
   2344  1.48   msaitoh 	ETHER_LOCK(ec);
   2345  1.75  yamaguch 	for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
   2346  1.75  yamaguch 	    ETHER_NEXT_MULTI(step, enm)) {
   2347   1.1   hannken 		if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
   2348  1.75  yamaguch 			allmulti = true;
   2349  1.75  yamaguch 			break;
   2350   1.1   hannken 		}
   2351  1.50  christos 		if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
   2352  1.75  yamaguch 			allmulti = true;
   2353  1.75  yamaguch 			break;
   2354   1.1   hannken 		}
   2355  1.75  yamaguch 
   2356  1.43  yamaguch 		memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
   2357  1.50  christos 		    enm->enm_addrlo, ETHER_ADDR_LEN);
   2358  1.75  yamaguch 		nentries++;
   2359   1.1   hannken 	}
   2360  1.48   msaitoh 	ETHER_UNLOCK(ec);
   2361  1.30     ozaki 
   2362  1.74  yamaguch 	r = vioif_set_mac_addr(sc);
   2363  1.74  yamaguch 	if (r != 0) {
   2364  1.74  yamaguch 		log(LOG_WARNING, "%s: couldn't set MAC address\n",
   2365  1.74  yamaguch 		    ifp->if_xname);
   2366  1.74  yamaguch 	}
   2367  1.74  yamaguch 
   2368  1.75  yamaguch 	if (!allmulti) {
   2369  1.74  yamaguch 		ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1);
   2370  1.66   reinoud 		ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries);
   2371   1.1   hannken 		r = vioif_set_rx_filter(sc);
   2372   1.1   hannken 		if (r != 0) {
   2373  1.75  yamaguch 			allmulti = true; /* fallback */
   2374   1.1   hannken 		}
   2375  1.75  yamaguch 	}
   2376  1.75  yamaguch 
   2377  1.75  yamaguch 	if (allmulti) {
   2378  1.66   reinoud 		ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0);
   2379  1.66   reinoud 		ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0);
   2380   1.1   hannken 		r = vioif_set_rx_filter(sc);
   2381   1.1   hannken 		if (r != 0) {
   2382  1.75  yamaguch 			log(LOG_DEBUG, "%s: couldn't clear RX filter\n",
   2383  1.75  yamaguch 			    ifp->if_xname);
   2384  1.75  yamaguch 			/* what to do on failure? */
   2385   1.1   hannken 		}
   2386  1.75  yamaguch 
   2387  1.75  yamaguch 		ifp->if_flags |= IFF_ALLMULTI;
   2388   1.1   hannken 	}
   2389  1.75  yamaguch 
   2390  1.75  yamaguch set_ifflags:
   2391  1.75  yamaguch 	r = vioif_ifflags(sc);
   2392   1.1   hannken 
   2393   1.1   hannken 	return r;
   2394   1.1   hannken }
   2395   1.1   hannken 
   2396  1.34     ozaki static bool
   2397  1.34     ozaki vioif_is_link_up(struct vioif_softc *sc)
   2398  1.34     ozaki {
   2399  1.34     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   2400  1.34     ozaki 	uint16_t status;
   2401  1.34     ozaki 
   2402  1.34     ozaki 	if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
   2403  1.34     ozaki 		status = virtio_read_device_config_2(vsc,
   2404  1.34     ozaki 		    VIRTIO_NET_CONFIG_STATUS);
   2405  1.34     ozaki 	else
   2406  1.34     ozaki 		status = VIRTIO_NET_S_LINK_UP;
   2407  1.34     ozaki 
   2408  1.34     ozaki 	return ((status & VIRTIO_NET_S_LINK_UP) != 0);
   2409  1.34     ozaki }
   2410  1.34     ozaki 
   2411  1.34     ozaki /* change link status */
   2412  1.34     ozaki static void
   2413  1.34     ozaki vioif_update_link_status(struct vioif_softc *sc)
   2414  1.34     ozaki {
   2415  1.34     ozaki 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2416  1.46  yamaguch 	struct vioif_txqueue *txq;
   2417  1.34     ozaki 	bool active, changed;
   2418  1.46  yamaguch 	int link, i;
   2419  1.34     ozaki 
   2420  1.62  yamaguch 	mutex_enter(&sc->sc_lock);
   2421  1.62  yamaguch 
   2422  1.34     ozaki 	active = vioif_is_link_up(sc);
   2423  1.34     ozaki 	changed = false;
   2424  1.34     ozaki 
   2425  1.34     ozaki 	if (active) {
   2426  1.34     ozaki 		if (!sc->sc_link_active)
   2427  1.34     ozaki 			changed = true;
   2428  1.34     ozaki 
   2429  1.34     ozaki 		link = LINK_STATE_UP;
   2430  1.34     ozaki 		sc->sc_link_active = true;
   2431  1.34     ozaki 	} else {
   2432  1.34     ozaki 		if (sc->sc_link_active)
   2433  1.34     ozaki 			changed = true;
   2434  1.34     ozaki 
   2435  1.34     ozaki 		link = LINK_STATE_DOWN;
   2436  1.34     ozaki 		sc->sc_link_active = false;
   2437  1.34     ozaki 	}
   2438  1.34     ozaki 
   2439  1.43  yamaguch 	if (changed) {
   2440  1.46  yamaguch 		for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   2441  1.46  yamaguch 			txq = &sc->sc_txq[i];
   2442  1.46  yamaguch 
   2443  1.46  yamaguch 			mutex_enter(txq->txq_lock);
   2444  1.46  yamaguch 			txq->txq_link_active = sc->sc_link_active;
   2445  1.46  yamaguch 			mutex_exit(txq->txq_lock);
   2446  1.46  yamaguch 		}
   2447  1.43  yamaguch 
   2448  1.34     ozaki 		if_link_state_change(ifp, link);
   2449  1.43  yamaguch 	}
   2450  1.62  yamaguch 
   2451  1.62  yamaguch 	mutex_exit(&sc->sc_lock);
   2452  1.34     ozaki }
   2453  1.34     ozaki 
   2454  1.34     ozaki static int
   2455  1.34     ozaki vioif_config_change(struct virtio_softc *vsc)
   2456  1.34     ozaki {
   2457  1.34     ozaki 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2458  1.34     ozaki 
   2459  1.34     ozaki 	softint_schedule(sc->sc_ctl_softint);
   2460  1.34     ozaki 	return 0;
   2461  1.34     ozaki }
   2462  1.34     ozaki 
   2463  1.34     ozaki static void
   2464  1.34     ozaki vioif_ctl_softint(void *arg)
   2465  1.34     ozaki {
   2466  1.34     ozaki 	struct vioif_softc *sc = arg;
   2467  1.34     ozaki 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2468  1.34     ozaki 
   2469  1.34     ozaki 	vioif_update_link_status(sc);
   2470  1.34     ozaki 	vioif_start(ifp);
   2471  1.34     ozaki }
   2472  1.34     ozaki 
   2473  1.55  yamaguch static struct workqueue *
   2474  1.55  yamaguch vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
   2475  1.55  yamaguch {
   2476  1.55  yamaguch 	struct workqueue *wq;
   2477  1.55  yamaguch 	int error;
   2478  1.55  yamaguch 
   2479  1.55  yamaguch 	error = workqueue_create(&wq, name, vioif_workq_work, NULL,
   2480  1.55  yamaguch 	    prio, ipl, flags);
   2481  1.55  yamaguch 
   2482  1.55  yamaguch 	if (error)
   2483  1.55  yamaguch 		return NULL;
   2484  1.55  yamaguch 
   2485  1.55  yamaguch 	return wq;
   2486  1.55  yamaguch }
   2487  1.55  yamaguch 
   2488  1.55  yamaguch static void
   2489  1.55  yamaguch vioif_workq_destroy(struct workqueue *wq)
   2490  1.55  yamaguch {
   2491  1.55  yamaguch 
   2492  1.55  yamaguch 	workqueue_destroy(wq);
   2493  1.55  yamaguch }
   2494  1.55  yamaguch 
   2495  1.55  yamaguch static void
   2496  1.55  yamaguch vioif_workq_work(struct work *wk, void *context)
   2497  1.55  yamaguch {
   2498  1.55  yamaguch 	struct vioif_work *work;
   2499  1.55  yamaguch 
   2500  1.55  yamaguch 	work = container_of(wk, struct vioif_work, cookie);
   2501  1.55  yamaguch 
   2502  1.55  yamaguch 	atomic_store_relaxed(&work->added, 0);
   2503  1.55  yamaguch 	work->func(work->arg);
   2504  1.55  yamaguch }
   2505  1.55  yamaguch 
   2506  1.55  yamaguch static void
   2507  1.55  yamaguch vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
   2508  1.55  yamaguch {
   2509  1.55  yamaguch 
   2510  1.55  yamaguch 	memset(work, 0, sizeof(*work));
   2511  1.55  yamaguch 	work->func = func;
   2512  1.55  yamaguch 	work->arg = arg;
   2513  1.55  yamaguch }
   2514  1.55  yamaguch 
   2515  1.55  yamaguch static void
   2516  1.55  yamaguch vioif_work_add(struct workqueue *wq, struct vioif_work *work)
   2517  1.55  yamaguch {
   2518  1.55  yamaguch 
   2519  1.55  yamaguch 	if (atomic_load_relaxed(&work->added) != 0)
   2520  1.55  yamaguch 		return;
   2521  1.55  yamaguch 
   2522  1.55  yamaguch 	atomic_store_relaxed(&work->added, 1);
   2523  1.55  yamaguch 	kpreempt_disable();
   2524  1.55  yamaguch 	workqueue_enqueue(wq, &work->cookie, NULL);
   2525  1.55  yamaguch 	kpreempt_enable();
   2526  1.55  yamaguch }
   2527  1.55  yamaguch 
   2528  1.55  yamaguch static void
   2529  1.55  yamaguch vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
   2530  1.55  yamaguch {
   2531  1.55  yamaguch 
   2532  1.55  yamaguch 	workqueue_wait(wq, &work->cookie);
   2533  1.55  yamaguch }
   2534  1.55  yamaguch 
   2535  1.55  yamaguch static int
   2536  1.55  yamaguch vioif_setup_sysctl(struct vioif_softc *sc)
   2537  1.55  yamaguch {
   2538  1.55  yamaguch 	const char *devname;
   2539  1.55  yamaguch 	struct sysctllog **log;
   2540  1.55  yamaguch 	const struct sysctlnode *rnode, *rxnode, *txnode;
   2541  1.55  yamaguch 	int error;
   2542  1.55  yamaguch 
   2543  1.55  yamaguch 	log = &sc->sc_sysctllog;
   2544  1.55  yamaguch 	devname = device_xname(sc->sc_dev);
   2545  1.55  yamaguch 
   2546  1.55  yamaguch 	error = sysctl_createv(log, 0, NULL, &rnode,
   2547  1.55  yamaguch 	    0, CTLTYPE_NODE, devname,
   2548  1.55  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings"),
   2549  1.55  yamaguch 	    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
   2550  1.55  yamaguch 	if (error)
   2551  1.55  yamaguch 		goto out;
   2552  1.55  yamaguch 
   2553  1.55  yamaguch 	error = sysctl_createv(log, 0, &rnode, NULL,
   2554  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
   2555  1.55  yamaguch 	    SYSCTL_DESCR("Use workqueue for packet processing"),
   2556  1.55  yamaguch 	    NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
   2557  1.55  yamaguch 	if (error)
   2558  1.55  yamaguch 		goto out;
   2559  1.55  yamaguch 
   2560  1.55  yamaguch 	error = sysctl_createv(log, 0, &rnode, &rxnode,
   2561  1.55  yamaguch 	    0, CTLTYPE_NODE, "rx",
   2562  1.55  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Rx"),
   2563  1.55  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
   2564  1.55  yamaguch 	if (error)
   2565  1.55  yamaguch 		goto out;
   2566  1.55  yamaguch 
   2567  1.55  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
   2568  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
   2569  1.55  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
   2570  1.55  yamaguch 	    NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
   2571  1.55  yamaguch 	if (error)
   2572  1.55  yamaguch 		goto out;
   2573  1.55  yamaguch 
   2574  1.55  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
   2575  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
   2576  1.55  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
   2577  1.55  yamaguch 	    NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
   2578  1.55  yamaguch 	if (error)
   2579  1.55  yamaguch 		goto out;
   2580  1.55  yamaguch 
   2581  1.55  yamaguch 	error = sysctl_createv(log, 0, &rnode, &txnode,
   2582  1.55  yamaguch 	    0, CTLTYPE_NODE, "tx",
   2583  1.55  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Tx"),
   2584  1.55  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
   2585  1.55  yamaguch 	if (error)
   2586  1.55  yamaguch 		goto out;
   2587  1.55  yamaguch 
   2588  1.55  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
   2589  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
   2590  1.55  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
   2591  1.55  yamaguch 	    NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
   2592  1.55  yamaguch 	if (error)
   2593  1.55  yamaguch 		goto out;
   2594  1.55  yamaguch 
   2595  1.55  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
   2596  1.55  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
   2597  1.55  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
   2598  1.55  yamaguch 	    NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
   2599  1.55  yamaguch 
   2600  1.55  yamaguch out:
   2601  1.55  yamaguch 	if (error)
   2602  1.55  yamaguch 		sysctl_teardown(log);
   2603  1.55  yamaguch 
   2604  1.55  yamaguch 	return error;
   2605  1.55  yamaguch }
   2606  1.55  yamaguch 
   2607  1.63  yamaguch static void
   2608  1.63  yamaguch vioif_setup_stats(struct vioif_softc *sc)
   2609  1.63  yamaguch {
   2610  1.63  yamaguch 	struct vioif_rxqueue *rxq;
   2611  1.63  yamaguch 	struct vioif_txqueue *txq;
   2612  1.63  yamaguch 	int i;
   2613  1.63  yamaguch 
   2614  1.63  yamaguch 	for (i = 0; i < sc->sc_max_nvq_pairs; i++) {
   2615  1.63  yamaguch 		rxq = &sc->sc_rxq[i];
   2616  1.63  yamaguch 		txq = &sc->sc_txq[i];
   2617  1.63  yamaguch 
   2618  1.65  riastrad 		snprintf(txq->txq_evgroup, sizeof(txq->txq_evgroup), "%s-TX%d",
   2619  1.63  yamaguch 		    device_xname(sc->sc_dev), i);
   2620  1.63  yamaguch 		evcnt_attach_dynamic(&txq->txq_defrag_failed, EVCNT_TYPE_MISC,
   2621  1.65  riastrad 		    NULL, txq->txq_evgroup, "tx m_defrag() failed");
   2622  1.63  yamaguch 		evcnt_attach_dynamic(&txq->txq_mbuf_load_failed, EVCNT_TYPE_MISC,
   2623  1.65  riastrad 		    NULL, txq->txq_evgroup, "tx dmamap load failed");
   2624  1.63  yamaguch 		evcnt_attach_dynamic(&txq->txq_enqueue_reserve_failed, EVCNT_TYPE_MISC,
   2625  1.65  riastrad 		    NULL, txq->txq_evgroup, "virtio_enqueue_reserve failed");
   2626  1.63  yamaguch 
   2627  1.65  riastrad 		snprintf(rxq->rxq_evgroup, sizeof(rxq->rxq_evgroup), "%s-RX%d",
   2628  1.63  yamaguch 		    device_xname(sc->sc_dev), i);
   2629  1.63  yamaguch 		evcnt_attach_dynamic(&rxq->rxq_mbuf_add_failed, EVCNT_TYPE_MISC,
   2630  1.65  riastrad 		    NULL, rxq->rxq_evgroup, "rx mbuf allocation failed");
   2631  1.63  yamaguch 	}
   2632  1.63  yamaguch 
   2633  1.63  yamaguch 	evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC,
   2634  1.65  riastrad 	    NULL, device_xname(sc->sc_dev), "control command dmamap load failed");
   2635  1.63  yamaguch 	evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC,
   2636  1.65  riastrad 	    NULL, device_xname(sc->sc_dev), "control command failed");
   2637  1.63  yamaguch }
   2638  1.63  yamaguch 
   2639  1.26  pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
   2640  1.48   msaitoh 
   2641  1.26  pgoyette #ifdef _MODULE
   2642  1.26  pgoyette #include "ioconf.c"
   2643  1.26  pgoyette #endif
   2644  1.48   msaitoh 
   2645  1.48   msaitoh static int
   2646  1.26  pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
   2647  1.26  pgoyette {
   2648  1.26  pgoyette 	int error = 0;
   2649  1.48   msaitoh 
   2650  1.26  pgoyette #ifdef _MODULE
   2651  1.26  pgoyette 	switch (cmd) {
   2652  1.26  pgoyette 	case MODULE_CMD_INIT:
   2653  1.48   msaitoh 		error = config_init_component(cfdriver_ioconf_if_vioif,
   2654  1.48   msaitoh 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2655  1.26  pgoyette 		break;
   2656  1.26  pgoyette 	case MODULE_CMD_FINI:
   2657  1.26  pgoyette 		error = config_fini_component(cfdriver_ioconf_if_vioif,
   2658  1.26  pgoyette 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2659  1.26  pgoyette 		break;
   2660  1.26  pgoyette 	default:
   2661  1.26  pgoyette 		error = ENOTTY;
   2662  1.48   msaitoh 		break;
   2663  1.26  pgoyette 	}
   2664  1.26  pgoyette #endif
   2665  1.48   msaitoh 
   2666  1.26  pgoyette 	return error;
   2667  1.26  pgoyette }
   2668