Home | History | Annotate | Line # | Download | only in pci
if_vioif.c revision 1.101
      1  1.101  yamaguch /*	$NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $	*/
      2    1.1   hannken 
      3    1.1   hannken /*
      4   1.66   reinoud  * Copyright (c) 2020 The NetBSD Foundation, Inc.
      5    1.1   hannken  * Copyright (c) 2010 Minoura Makoto.
      6    1.1   hannken  * All rights reserved.
      7    1.1   hannken  *
      8    1.1   hannken  * Redistribution and use in source and binary forms, with or without
      9    1.1   hannken  * modification, are permitted provided that the following conditions
     10    1.1   hannken  * are met:
     11    1.1   hannken  * 1. Redistributions of source code must retain the above copyright
     12    1.1   hannken  *    notice, this list of conditions and the following disclaimer.
     13    1.1   hannken  * 2. Redistributions in binary form must reproduce the above copyright
     14    1.1   hannken  *    notice, this list of conditions and the following disclaimer in the
     15    1.1   hannken  *    documentation and/or other materials provided with the distribution.
     16    1.1   hannken  *
     17    1.1   hannken  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
     18    1.1   hannken  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
     19    1.1   hannken  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
     20    1.1   hannken  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
     21    1.1   hannken  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
     22    1.1   hannken  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
     23    1.1   hannken  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
     24    1.1   hannken  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
     25    1.1   hannken  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
     26    1.1   hannken  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
     27    1.1   hannken  */
     28    1.1   hannken 
     29    1.1   hannken #include <sys/cdefs.h>
     30  1.101  yamaguch __KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.101 2023/03/23 02:57:54 yamaguchi Exp $");
     31   1.15     ozaki 
     32   1.15     ozaki #ifdef _KERNEL_OPT
     33   1.15     ozaki #include "opt_net_mpsafe.h"
     34   1.15     ozaki #endif
     35    1.1   hannken 
     36    1.1   hannken #include <sys/param.h>
     37    1.1   hannken #include <sys/systm.h>
     38    1.1   hannken #include <sys/kernel.h>
     39   1.55  yamaguch #include <sys/atomic.h>
     40    1.1   hannken #include <sys/bus.h>
     41    1.1   hannken #include <sys/condvar.h>
     42    1.1   hannken #include <sys/device.h>
     43   1.63  yamaguch #include <sys/evcnt.h>
     44    1.1   hannken #include <sys/intr.h>
     45    1.1   hannken #include <sys/kmem.h>
     46    1.1   hannken #include <sys/mbuf.h>
     47    1.1   hannken #include <sys/mutex.h>
     48    1.1   hannken #include <sys/sockio.h>
     49   1.71  yamaguch #include <sys/syslog.h>
     50   1.12     ozaki #include <sys/cpu.h>
     51   1.26  pgoyette #include <sys/module.h>
     52   1.46  yamaguch #include <sys/pcq.h>
     53   1.55  yamaguch #include <sys/workqueue.h>
     54   1.85  yamaguch #include <sys/xcall.h>
     55    1.1   hannken 
     56    1.1   hannken #include <dev/pci/virtioreg.h>
     57    1.1   hannken #include <dev/pci/virtiovar.h>
     58    1.1   hannken 
     59    1.1   hannken #include <net/if.h>
     60   1.74  yamaguch #include <net/if_dl.h>
     61    1.1   hannken #include <net/if_media.h>
     62    1.1   hannken #include <net/if_ether.h>
     63    1.1   hannken 
     64    1.1   hannken #include <net/bpf.h>
     65    1.1   hannken 
     66   1.26  pgoyette #include "ioconf.h"
     67    1.1   hannken 
     68    1.7     ozaki #ifdef NET_MPSAFE
     69    1.7     ozaki #define VIOIF_MPSAFE	1
     70   1.46  yamaguch #define VIOIF_MULTIQ	1
     71    1.7     ozaki #endif
     72    1.7     ozaki 
     73    1.1   hannken /*
     74    1.1   hannken  * if_vioifreg.h:
     75    1.1   hannken  */
     76    1.1   hannken /* Configuration registers */
     77   1.66   reinoud #define VIRTIO_NET_CONFIG_MAC		 0 /* 8bit x 6byte */
     78   1.66   reinoud #define VIRTIO_NET_CONFIG_STATUS	 6 /* 16bit */
     79   1.66   reinoud #define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS	 8 /* 16bit */
     80   1.66   reinoud #define VIRTIO_NET_CONFIG_MTU		10 /* 16bit */
     81    1.1   hannken 
     82    1.1   hannken /* Feature bits */
     83   1.46  yamaguch #define VIRTIO_NET_F_CSUM		__BIT(0)
     84   1.46  yamaguch #define VIRTIO_NET_F_GUEST_CSUM		__BIT(1)
     85   1.46  yamaguch #define VIRTIO_NET_F_MAC		__BIT(5)
     86   1.46  yamaguch #define VIRTIO_NET_F_GSO		__BIT(6)
     87   1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO4		__BIT(7)
     88   1.46  yamaguch #define VIRTIO_NET_F_GUEST_TSO6		__BIT(8)
     89   1.46  yamaguch #define VIRTIO_NET_F_GUEST_ECN		__BIT(9)
     90   1.46  yamaguch #define VIRTIO_NET_F_GUEST_UFO		__BIT(10)
     91   1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO4		__BIT(11)
     92   1.46  yamaguch #define VIRTIO_NET_F_HOST_TSO6		__BIT(12)
     93   1.46  yamaguch #define VIRTIO_NET_F_HOST_ECN		__BIT(13)
     94   1.46  yamaguch #define VIRTIO_NET_F_HOST_UFO		__BIT(14)
     95   1.46  yamaguch #define VIRTIO_NET_F_MRG_RXBUF		__BIT(15)
     96   1.46  yamaguch #define VIRTIO_NET_F_STATUS		__BIT(16)
     97   1.46  yamaguch #define VIRTIO_NET_F_CTRL_VQ		__BIT(17)
     98   1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX		__BIT(18)
     99   1.46  yamaguch #define VIRTIO_NET_F_CTRL_VLAN		__BIT(19)
    100   1.46  yamaguch #define VIRTIO_NET_F_CTRL_RX_EXTRA	__BIT(20)
    101   1.46  yamaguch #define VIRTIO_NET_F_GUEST_ANNOUNCE	__BIT(21)
    102   1.46  yamaguch #define VIRTIO_NET_F_MQ			__BIT(22)
    103   1.78  yamaguch #define VIRTIO_NET_F_CTRL_MAC_ADDR 	__BIT(23)
    104    1.1   hannken 
    105   1.79       uwe #define VIRTIO_NET_FLAG_BITS			\
    106   1.79       uwe 	VIRTIO_COMMON_FLAG_BITS			\
    107   1.79       uwe 	"b\x17" "CTRL_MAC\0"			\
    108   1.79       uwe 	"b\x16" "MQ\0"				\
    109   1.79       uwe 	"b\x15" "GUEST_ANNOUNCE\0"		\
    110   1.79       uwe 	"b\x14" "CTRL_RX_EXTRA\0"		\
    111   1.79       uwe 	"b\x13" "CTRL_VLAN\0"			\
    112   1.79       uwe 	"b\x12" "CTRL_RX\0"			\
    113   1.79       uwe 	"b\x11" "CTRL_VQ\0"			\
    114   1.79       uwe 	"b\x10" "STATUS\0"			\
    115   1.79       uwe 	"b\x0f" "MRG_RXBUF\0"			\
    116   1.79       uwe 	"b\x0e" "HOST_UFO\0"			\
    117   1.79       uwe 	"b\x0d" "HOST_ECN\0"			\
    118   1.79       uwe 	"b\x0c" "HOST_TSO6\0"			\
    119   1.79       uwe 	"b\x0b" "HOST_TSO4\0"			\
    120   1.79       uwe 	"b\x0a" "GUEST_UFO\0"			\
    121   1.79       uwe 	"b\x09" "GUEST_ECN\0"			\
    122   1.79       uwe 	"b\x08" "GUEST_TSO6\0"			\
    123   1.79       uwe 	"b\x07" "GUEST_TSO4\0"			\
    124   1.79       uwe 	"b\x06" "GSO\0"				\
    125   1.79       uwe 	"b\x05" "MAC\0"				\
    126   1.79       uwe 	"b\x01" "GUEST_CSUM\0"			\
    127   1.79       uwe 	"b\x00" "CSUM\0"
    128   1.18  christos 
    129    1.1   hannken /* Status */
    130    1.1   hannken #define VIRTIO_NET_S_LINK_UP	1
    131    1.1   hannken 
    132    1.1   hannken /* Packet header structure */
    133    1.1   hannken struct virtio_net_hdr {
    134    1.1   hannken 	uint8_t		flags;
    135    1.1   hannken 	uint8_t		gso_type;
    136    1.1   hannken 	uint16_t	hdr_len;
    137    1.1   hannken 	uint16_t	gso_size;
    138    1.1   hannken 	uint16_t	csum_start;
    139    1.1   hannken 	uint16_t	csum_offset;
    140   1.66   reinoud 
    141   1.66   reinoud 	uint16_t	num_buffers; /* VIRTIO_NET_F_MRG_RXBUF enabled or v1 */
    142    1.1   hannken } __packed;
    143    1.1   hannken 
    144    1.1   hannken #define VIRTIO_NET_HDR_F_NEEDS_CSUM	1 /* flags */
    145    1.1   hannken #define VIRTIO_NET_HDR_GSO_NONE		0 /* gso_type */
    146    1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV4	1 /* gso_type */
    147    1.1   hannken #define VIRTIO_NET_HDR_GSO_UDP		3 /* gso_type */
    148    1.1   hannken #define VIRTIO_NET_HDR_GSO_TCPV6	4 /* gso_type */
    149    1.1   hannken #define VIRTIO_NET_HDR_GSO_ECN		0x80 /* gso_type, |'ed */
    150    1.1   hannken 
    151    1.1   hannken #define VIRTIO_NET_MAX_GSO_LEN		(65536+ETHER_HDR_LEN)
    152    1.1   hannken 
    153    1.1   hannken /* Control virtqueue */
    154    1.1   hannken struct virtio_net_ctrl_cmd {
    155    1.1   hannken 	uint8_t	class;
    156    1.1   hannken 	uint8_t	command;
    157    1.1   hannken } __packed;
    158    1.1   hannken #define VIRTIO_NET_CTRL_RX		0
    159    1.1   hannken # define VIRTIO_NET_CTRL_RX_PROMISC	0
    160    1.1   hannken # define VIRTIO_NET_CTRL_RX_ALLMULTI	1
    161    1.1   hannken 
    162    1.1   hannken #define VIRTIO_NET_CTRL_MAC		1
    163    1.1   hannken # define VIRTIO_NET_CTRL_MAC_TABLE_SET	0
    164   1.74  yamaguch # define  VIRTIO_NET_CTRL_MAC_ADDR_SET	1
    165    1.1   hannken 
    166    1.1   hannken #define VIRTIO_NET_CTRL_VLAN		2
    167    1.1   hannken # define VIRTIO_NET_CTRL_VLAN_ADD	0
    168    1.1   hannken # define VIRTIO_NET_CTRL_VLAN_DEL	1
    169    1.1   hannken 
    170   1.46  yamaguch #define VIRTIO_NET_CTRL_MQ			4
    171   1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET	0
    172   1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN	1
    173   1.46  yamaguch # define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX	0x8000
    174   1.46  yamaguch 
    175    1.1   hannken struct virtio_net_ctrl_status {
    176    1.1   hannken 	uint8_t	ack;
    177    1.1   hannken } __packed;
    178    1.1   hannken #define VIRTIO_NET_OK			0
    179    1.1   hannken #define VIRTIO_NET_ERR			1
    180    1.1   hannken 
    181    1.1   hannken struct virtio_net_ctrl_rx {
    182    1.1   hannken 	uint8_t	onoff;
    183    1.1   hannken } __packed;
    184    1.1   hannken 
    185    1.1   hannken struct virtio_net_ctrl_mac_tbl {
    186    1.1   hannken 	uint32_t nentries;
    187    1.1   hannken 	uint8_t macs[][ETHER_ADDR_LEN];
    188    1.1   hannken } __packed;
    189    1.1   hannken 
    190   1.74  yamaguch struct virtio_net_ctrl_mac_addr {
    191   1.74  yamaguch 	uint8_t mac[ETHER_ADDR_LEN];
    192   1.74  yamaguch } __packed;
    193   1.74  yamaguch 
    194    1.1   hannken struct virtio_net_ctrl_vlan {
    195    1.1   hannken 	uint16_t id;
    196    1.1   hannken } __packed;
    197    1.1   hannken 
    198   1.46  yamaguch struct virtio_net_ctrl_mq {
    199   1.46  yamaguch 	uint16_t virtqueue_pairs;
    200   1.46  yamaguch } __packed;
    201   1.46  yamaguch 
    202    1.1   hannken /*
    203    1.1   hannken  * if_vioifvar.h:
    204    1.1   hannken  */
    205   1.43  yamaguch 
    206   1.43  yamaguch /*
    207   1.43  yamaguch  * Locking notes:
    208   1.98  yamaguch  * + a field in vioif_netueue is protected by netq_lock (a spin mutex)
    209   1.43  yamaguch  *      - more than one lock cannot be held at onece
    210   1.98  yamaguch  * + a field in vioif_tx_context and vioif_rx_context is also protected
    211   1.98  yamaguch  *   by netq_lock.
    212   1.43  yamaguch  * + ctrlq_inuse is protected by ctrlq_wait_lock.
    213   1.43  yamaguch  *      - other fields in vioif_ctrlqueue are protected by ctrlq_inuse
    214   1.98  yamaguch  *      - netq_lock cannot be held along with ctrlq_wait_lock
    215   1.62  yamaguch  * + fields in vioif_softc except queues are protected by
    216   1.62  yamaguch  *   sc->sc_lock(an adaptive mutex)
    217   1.62  yamaguch  *      - the lock is held before acquisition of other locks
    218   1.43  yamaguch  */
    219   1.43  yamaguch 
    220   1.66   reinoud struct vioif_ctrl_cmdspec {
    221   1.66   reinoud 	bus_dmamap_t	dmamap;
    222   1.66   reinoud 	void		*buf;
    223   1.66   reinoud 	bus_size_t	bufsize;
    224   1.66   reinoud };
    225   1.66   reinoud 
    226   1.55  yamaguch struct vioif_work {
    227   1.55  yamaguch 	struct work	 cookie;
    228   1.55  yamaguch 	void		(*func)(void *);
    229   1.55  yamaguch 	void		*arg;
    230   1.55  yamaguch 	unsigned int	 added;
    231   1.55  yamaguch };
    232   1.55  yamaguch 
    233   1.95  yamaguch struct vioif_net_map {
    234   1.95  yamaguch 	struct virtio_net_hdr	*vnm_hdr;
    235   1.95  yamaguch 	bus_dmamap_t		 vnm_hdr_map;
    236   1.95  yamaguch 	struct mbuf		*vnm_mbuf;
    237   1.95  yamaguch 	bus_dmamap_t		 vnm_mbuf_map;
    238   1.95  yamaguch };
    239   1.95  yamaguch 
    240   1.98  yamaguch #define VIOIF_NETQ_RX		0
    241   1.98  yamaguch #define VIOIF_NETQ_TX		1
    242   1.98  yamaguch #define VIOIF_NETQ_IDX		2
    243   1.98  yamaguch #define VIOIF_NETQ_DIR(n)	((n) % VIOIF_NETQ_IDX)
    244   1.98  yamaguch #define VIOIF_NETQ_PAIRIDX(n)	((n) / VIOIF_NETQ_IDX)
    245   1.98  yamaguch #define VIOIF_NETQ_RXQID(n)	((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_RX)
    246   1.98  yamaguch #define VIOIF_NETQ_TXQID(n)	((n) * VIOIF_NETQ_IDX + VIOIF_NETQ_TX)
    247   1.98  yamaguch 
    248   1.98  yamaguch struct vioif_netqueue {
    249   1.98  yamaguch 	kmutex_t		 netq_lock;
    250   1.98  yamaguch 	struct virtqueue	*netq_vq;
    251   1.98  yamaguch 	bool			 netq_stopping;
    252   1.98  yamaguch 	bool			 netq_running_handle;
    253   1.98  yamaguch 	void			*netq_maps_kva;
    254   1.98  yamaguch 	struct vioif_net_map	*netq_maps;
    255   1.98  yamaguch 
    256   1.98  yamaguch 	void			*netq_softint;
    257   1.98  yamaguch 	struct vioif_work	 netq_work;
    258   1.98  yamaguch 	bool			 netq_workqueue;
    259   1.98  yamaguch 
    260   1.98  yamaguch 	char			 netq_evgroup[32];
    261   1.98  yamaguch 	struct evcnt		 netq_mbuf_load_failed;
    262   1.99  yamaguch 	struct evcnt		 netq_enqueue_failed;
    263   1.43  yamaguch 
    264   1.98  yamaguch 	void			*netq_ctx;
    265   1.43  yamaguch };
    266   1.43  yamaguch 
    267   1.98  yamaguch struct vioif_tx_context {
    268   1.98  yamaguch 	bool			 txc_link_active;
    269   1.98  yamaguch 	pcq_t			*txc_intrq;
    270   1.98  yamaguch 	void			*txc_deferred_transmit;
    271   1.43  yamaguch 
    272   1.98  yamaguch 	struct evcnt		 txc_defrag_failed;
    273   1.98  yamaguch };
    274   1.43  yamaguch 
    275   1.98  yamaguch struct vioif_rx_context {
    276   1.98  yamaguch 	struct evcnt		 rxc_mbuf_enobufs;
    277   1.43  yamaguch };
    278   1.43  yamaguch struct vioif_ctrlqueue {
    279   1.43  yamaguch 	struct virtqueue		*ctrlq_vq;
    280   1.43  yamaguch 	enum {
    281   1.43  yamaguch 		FREE, INUSE, DONE
    282   1.43  yamaguch 	}				ctrlq_inuse;
    283   1.43  yamaguch 	kcondvar_t			ctrlq_wait;
    284   1.43  yamaguch 	kmutex_t			ctrlq_wait_lock;
    285   1.44  yamaguch 	struct lwp			*ctrlq_owner;
    286   1.43  yamaguch 
    287   1.43  yamaguch 	struct virtio_net_ctrl_cmd	*ctrlq_cmd;
    288   1.43  yamaguch 	struct virtio_net_ctrl_status	*ctrlq_status;
    289   1.43  yamaguch 	struct virtio_net_ctrl_rx	*ctrlq_rx;
    290   1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_uc;
    291   1.43  yamaguch 	struct virtio_net_ctrl_mac_tbl	*ctrlq_mac_tbl_mc;
    292   1.74  yamaguch 	struct virtio_net_ctrl_mac_addr	*ctrlq_mac_addr;
    293   1.46  yamaguch 	struct virtio_net_ctrl_mq	*ctrlq_mq;
    294   1.43  yamaguch 
    295   1.43  yamaguch 	bus_dmamap_t			ctrlq_cmd_dmamap;
    296   1.43  yamaguch 	bus_dmamap_t			ctrlq_status_dmamap;
    297   1.43  yamaguch 	bus_dmamap_t			ctrlq_rx_dmamap;
    298   1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_uc_dmamap;
    299   1.43  yamaguch 	bus_dmamap_t			ctrlq_tbl_mc_dmamap;
    300   1.74  yamaguch 	bus_dmamap_t			ctrlq_mac_addr_dmamap;
    301   1.46  yamaguch 	bus_dmamap_t			ctrlq_mq_dmamap;
    302   1.63  yamaguch 
    303   1.63  yamaguch 	struct evcnt			ctrlq_cmd_load_failed;
    304   1.63  yamaguch 	struct evcnt			ctrlq_cmd_failed;
    305   1.43  yamaguch };
    306   1.43  yamaguch 
    307    1.1   hannken struct vioif_softc {
    308    1.1   hannken 	device_t		sc_dev;
    309   1.62  yamaguch 	kmutex_t		sc_lock;
    310   1.55  yamaguch 	struct sysctllog	*sc_sysctllog;
    311    1.1   hannken 
    312    1.1   hannken 	struct virtio_softc	*sc_virtio;
    313   1.46  yamaguch 	struct virtqueue	*sc_vqs;
    314   1.66   reinoud 	u_int			 sc_hdr_size;
    315   1.46  yamaguch 
    316   1.46  yamaguch 	int			sc_max_nvq_pairs;
    317   1.46  yamaguch 	int			sc_req_nvq_pairs;
    318   1.46  yamaguch 	int			sc_act_nvq_pairs;
    319    1.1   hannken 
    320    1.1   hannken 	uint8_t			sc_mac[ETHER_ADDR_LEN];
    321    1.1   hannken 	struct ethercom		sc_ethercom;
    322   1.82  knakahar 	int			sc_link_state;
    323    1.1   hannken 
    324   1.98  yamaguch 	struct vioif_netqueue	*sc_netqs;
    325   1.43  yamaguch 
    326   1.43  yamaguch 	bool			sc_has_ctrl;
    327   1.43  yamaguch 	struct vioif_ctrlqueue	sc_ctrlq;
    328   1.43  yamaguch 
    329  1.100  yamaguch 	bus_dma_segment_t	 sc_segs[1];
    330   1.43  yamaguch 	void			*sc_dmamem;
    331   1.43  yamaguch 	void			*sc_kmem;
    332    1.1   hannken 
    333  1.101  yamaguch 	void			*sc_cfg_softint;
    334   1.55  yamaguch 
    335   1.55  yamaguch 	struct workqueue	*sc_txrx_workqueue;
    336   1.55  yamaguch 	bool			 sc_txrx_workqueue_sysctl;
    337   1.55  yamaguch 	u_int			 sc_tx_intr_process_limit;
    338   1.55  yamaguch 	u_int			 sc_tx_process_limit;
    339   1.55  yamaguch 	u_int			 sc_rx_intr_process_limit;
    340   1.55  yamaguch 	u_int			 sc_rx_process_limit;
    341    1.1   hannken };
    342    1.1   hannken #define VIRTIO_NET_TX_MAXNSEGS		(16) /* XXX */
    343    1.1   hannken #define VIRTIO_NET_CTRL_MAC_MAXENTRIES	(64) /* XXX */
    344    1.1   hannken 
    345   1.55  yamaguch #define VIOIF_TX_INTR_PROCESS_LIMIT	256
    346   1.55  yamaguch #define VIOIF_TX_PROCESS_LIMIT		256
    347   1.55  yamaguch #define VIOIF_RX_INTR_PROCESS_LIMIT	0U
    348   1.55  yamaguch #define VIOIF_RX_PROCESS_LIMIT		256
    349   1.55  yamaguch 
    350   1.55  yamaguch #define VIOIF_WORKQUEUE_PRI		PRI_SOFTNET
    351   1.82  knakahar #define VIOIF_IS_LINK_ACTIVE(_sc)	((_sc)->sc_link_state == LINK_STATE_UP ? \
    352   1.82  knakahar 					    true : false)
    353   1.55  yamaguch 
    354    1.1   hannken /* cfattach interface functions */
    355    1.1   hannken static int	vioif_match(device_t, cfdata_t, void *);
    356    1.1   hannken static void	vioif_attach(device_t, device_t, void *);
    357   1.55  yamaguch static int	vioif_finalize_teardown(device_t);
    358    1.1   hannken 
    359    1.1   hannken /* ifnet interface functions */
    360    1.1   hannken static int	vioif_init(struct ifnet *);
    361    1.1   hannken static void	vioif_stop(struct ifnet *, int);
    362    1.1   hannken static void	vioif_start(struct ifnet *);
    363   1.46  yamaguch static int	vioif_transmit(struct ifnet *, struct mbuf *);
    364    1.1   hannken static int	vioif_ioctl(struct ifnet *, u_long, void *);
    365    1.1   hannken static void	vioif_watchdog(struct ifnet *);
    366  1.101  yamaguch static int	vioif_ifflags(struct vioif_softc *);
    367   1.75  yamaguch static int	vioif_ifflags_cb(struct ethercom *);
    368    1.1   hannken 
    369   1.98  yamaguch /* tx & rx */
    370  1.101  yamaguch static int	vioif_netqueue_init(struct vioif_softc *,
    371  1.101  yamaguch 		    struct virtio_softc *, size_t, u_int);
    372  1.101  yamaguch static void	vioif_netqueue_teardown(struct vioif_softc *,
    373  1.101  yamaguch 		    struct virtio_softc *, size_t);
    374   1.99  yamaguch static void	vioif_net_intr_enable(struct vioif_softc *,
    375   1.99  yamaguch 		    struct virtio_softc *);
    376   1.99  yamaguch static void	vioif_net_intr_disable(struct vioif_softc *,
    377   1.99  yamaguch 		    struct virtio_softc *);
    378  1.101  yamaguch static void	vioif_net_sched_handle(struct vioif_softc *,
    379  1.101  yamaguch 		    struct vioif_netqueue *);
    380   1.98  yamaguch 
    381    1.1   hannken /* rx */
    382   1.66   reinoud static void	vioif_populate_rx_mbufs_locked(struct vioif_softc *,
    383   1.98  yamaguch 		    struct vioif_netqueue *);
    384   1.54  yamaguch static int	vioif_rx_intr(void *);
    385   1.55  yamaguch static void	vioif_rx_handle(void *);
    386  1.101  yamaguch static void	vioif_rx_queue_clear(struct vioif_softc *,
    387  1.101  yamaguch 		    struct virtio_softc *, struct vioif_netqueue *);
    388    1.1   hannken 
    389    1.1   hannken /* tx */
    390  1.101  yamaguch static void	vioif_start_locked(struct ifnet *, struct vioif_netqueue *);
    391  1.101  yamaguch static void	vioif_transmit_locked(struct ifnet *, struct vioif_netqueue *);
    392  1.101  yamaguch static void	vioif_deferred_transmit(void *);
    393   1.54  yamaguch static int	vioif_tx_intr(void *);
    394   1.55  yamaguch static void	vioif_tx_handle(void *);
    395   1.93  yamaguch static void	vioif_tx_queue_clear(struct vioif_softc *, struct virtio_softc *,
    396   1.98  yamaguch 		    struct vioif_netqueue *);
    397    1.1   hannken 
    398  1.101  yamaguch /* controls */
    399  1.101  yamaguch static int	vioif_ctrl_intr(void *);
    400    1.1   hannken static int	vioif_ctrl_rx(struct vioif_softc *, int, bool);
    401    1.1   hannken static int	vioif_set_promisc(struct vioif_softc *, bool);
    402    1.1   hannken static int	vioif_set_allmulti(struct vioif_softc *, bool);
    403    1.1   hannken static int	vioif_set_rx_filter(struct vioif_softc *);
    404    1.1   hannken static int	vioif_rx_filter(struct vioif_softc *);
    405   1.74  yamaguch static int	vioif_set_mac_addr(struct vioif_softc *);
    406  1.101  yamaguch static int	vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
    407  1.101  yamaguch 
    408  1.101  yamaguch /* config interrupt */
    409   1.34     ozaki static int	vioif_config_change(struct virtio_softc *);
    410  1.101  yamaguch static void	vioif_cfg_softint(void *);
    411  1.101  yamaguch static void	vioif_update_link_status(struct vioif_softc *);
    412  1.101  yamaguch 
    413  1.101  yamaguch /* others */
    414  1.101  yamaguch static void	vioif_alloc_queues(struct vioif_softc *);
    415  1.101  yamaguch static void	vioif_free_queues(struct vioif_softc *);
    416  1.101  yamaguch static int	vioif_alloc_mems(struct vioif_softc *);
    417  1.101  yamaguch static struct workqueue*
    418  1.101  yamaguch 		vioif_workq_create(const char *, pri_t, int, int);
    419  1.101  yamaguch static void	vioif_workq_destroy(struct workqueue *);
    420  1.101  yamaguch static void	vioif_work_set(struct vioif_work *, void(*)(void *), void *);
    421  1.101  yamaguch static void	vioif_work_add(struct workqueue *, struct vioif_work *);
    422  1.101  yamaguch static void	vioif_work_wait(struct workqueue *, struct vioif_work *);
    423   1.55  yamaguch static int	vioif_setup_sysctl(struct vioif_softc *);
    424   1.63  yamaguch static void	vioif_setup_stats(struct vioif_softc *);
    425    1.1   hannken 
    426    1.1   hannken CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
    427    1.1   hannken 		  vioif_match, vioif_attach, NULL, NULL);
    428    1.1   hannken 
    429  1.101  yamaguch static void
    430  1.101  yamaguch vioif_intr_barrier(void)
    431  1.101  yamaguch {
    432  1.101  yamaguch 
    433  1.101  yamaguch 	/* wait for finish all interrupt handler */
    434  1.101  yamaguch 	xc_barrier(0);
    435  1.101  yamaguch }
    436  1.101  yamaguch 
    437  1.101  yamaguch static void
    438  1.101  yamaguch vioif_notify(struct virtio_softc *vsc, struct virtqueue *vq)
    439  1.101  yamaguch {
    440  1.101  yamaguch 
    441  1.101  yamaguch 	virtio_enqueue_commit(vsc, vq, -1, true);
    442  1.101  yamaguch }
    443  1.101  yamaguch 
    444    1.1   hannken static int
    445    1.1   hannken vioif_match(device_t parent, cfdata_t match, void *aux)
    446    1.1   hannken {
    447   1.32  jdolecek 	struct virtio_attach_args *va = aux;
    448    1.1   hannken 
    449   1.66   reinoud 	if (va->sc_childdevid == VIRTIO_DEVICE_ID_NETWORK)
    450    1.1   hannken 		return 1;
    451    1.1   hannken 
    452    1.1   hannken 	return 0;
    453    1.1   hannken }
    454    1.1   hannken 
    455  1.101  yamaguch static void
    456  1.101  yamaguch vioif_attach(device_t parent, device_t self, void *aux)
    457   1.58  yamaguch {
    458  1.101  yamaguch 	struct vioif_softc *sc = device_private(self);
    459  1.101  yamaguch 	struct virtio_softc *vsc = device_private(parent);
    460  1.101  yamaguch 	struct vioif_netqueue *txq0;
    461  1.101  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    462  1.101  yamaguch 	uint64_t features, req_features;
    463  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    464  1.101  yamaguch 	u_int softint_flags;
    465  1.101  yamaguch 	int r, i, req_flags;
    466  1.101  yamaguch 	char xnamebuf[MAXCOMLEN];
    467  1.101  yamaguch 	size_t netq_num;
    468  1.101  yamaguch 
    469  1.101  yamaguch 	if (virtio_child(vsc) != NULL) {
    470  1.101  yamaguch 		aprint_normal(": child already attached for %s; "
    471  1.101  yamaguch 		    "something wrong...\n", device_xname(parent));
    472  1.101  yamaguch 		return;
    473  1.101  yamaguch 	}
    474  1.101  yamaguch 
    475  1.101  yamaguch 	sc->sc_dev = self;
    476  1.101  yamaguch 	sc->sc_virtio = vsc;
    477  1.101  yamaguch 	sc->sc_link_state = LINK_STATE_UNKNOWN;
    478  1.101  yamaguch 
    479  1.101  yamaguch 	sc->sc_max_nvq_pairs = 1;
    480  1.101  yamaguch 	sc->sc_req_nvq_pairs = 1;
    481  1.101  yamaguch 	sc->sc_act_nvq_pairs = 1;
    482  1.101  yamaguch 	sc->sc_txrx_workqueue_sysctl = true;
    483  1.101  yamaguch 	sc->sc_tx_intr_process_limit = VIOIF_TX_INTR_PROCESS_LIMIT;
    484  1.101  yamaguch 	sc->sc_tx_process_limit = VIOIF_TX_PROCESS_LIMIT;
    485  1.101  yamaguch 	sc->sc_rx_intr_process_limit = VIOIF_RX_INTR_PROCESS_LIMIT;
    486  1.101  yamaguch 	sc->sc_rx_process_limit = VIOIF_RX_PROCESS_LIMIT;
    487  1.101  yamaguch 
    488  1.101  yamaguch 	mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_NONE);
    489  1.101  yamaguch 
    490  1.101  yamaguch 	snprintf(xnamebuf, sizeof(xnamebuf), "%s_txrx", device_xname(self));
    491  1.101  yamaguch 	sc->sc_txrx_workqueue = vioif_workq_create(xnamebuf, VIOIF_WORKQUEUE_PRI,
    492  1.101  yamaguch 	    IPL_NET, WQ_PERCPU | WQ_MPSAFE);
    493  1.101  yamaguch 	if (sc->sc_txrx_workqueue == NULL)
    494  1.101  yamaguch 		goto err;
    495   1.58  yamaguch 
    496  1.101  yamaguch 	req_flags = 0;
    497   1.58  yamaguch 
    498  1.101  yamaguch #ifdef VIOIF_MPSAFE
    499  1.101  yamaguch 	req_flags |= VIRTIO_F_INTR_MPSAFE;
    500  1.101  yamaguch #endif
    501  1.101  yamaguch 	req_flags |= VIRTIO_F_INTR_MSIX;
    502   1.58  yamaguch 
    503  1.101  yamaguch 	req_features =
    504  1.101  yamaguch 	    VIRTIO_NET_F_MAC | VIRTIO_NET_F_STATUS | VIRTIO_NET_F_CTRL_VQ |
    505  1.101  yamaguch 	    VIRTIO_NET_F_CTRL_RX | VIRTIO_F_NOTIFY_ON_EMPTY;
    506  1.101  yamaguch 	req_features |= VIRTIO_F_RING_EVENT_IDX;
    507  1.101  yamaguch 	req_features |= VIRTIO_NET_F_CTRL_MAC_ADDR;
    508  1.101  yamaguch #ifdef VIOIF_MULTIQ
    509  1.101  yamaguch 	req_features |= VIRTIO_NET_F_MQ;
    510  1.101  yamaguch #endif
    511  1.101  yamaguch 	virtio_child_attach_start(vsc, self, IPL_NET, NULL,
    512  1.101  yamaguch 	    vioif_config_change, virtio_vq_intrhand, req_flags,
    513  1.101  yamaguch 	    req_features, VIRTIO_NET_FLAG_BITS);
    514   1.58  yamaguch 
    515  1.101  yamaguch 	features = virtio_features(vsc);
    516  1.101  yamaguch 	if (features == 0)
    517  1.101  yamaguch 		goto err;
    518   1.58  yamaguch 
    519  1.101  yamaguch 	if (features & VIRTIO_NET_F_MAC) {
    520  1.101  yamaguch 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    521  1.101  yamaguch 			sc->sc_mac[i] = virtio_read_device_config_1(vsc,
    522  1.101  yamaguch 			    VIRTIO_NET_CONFIG_MAC + i);
    523  1.101  yamaguch 		}
    524  1.101  yamaguch 	} else {
    525  1.101  yamaguch 		/* code stolen from sys/net/if_tap.c */
    526  1.101  yamaguch 		struct timeval tv;
    527  1.101  yamaguch 		uint32_t ui;
    528  1.101  yamaguch 		getmicrouptime(&tv);
    529  1.101  yamaguch 		ui = (tv.tv_sec ^ tv.tv_usec) & 0xffffff;
    530  1.101  yamaguch 		memcpy(sc->sc_mac+3, (uint8_t *)&ui, 3);
    531  1.101  yamaguch 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
    532  1.101  yamaguch 			virtio_write_device_config_1(vsc,
    533  1.101  yamaguch 			    VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
    534  1.101  yamaguch 		}
    535   1.58  yamaguch 	}
    536   1.58  yamaguch 
    537  1.101  yamaguch 	/* 'Ethernet' with capital follows other ethernet driver attachment */
    538  1.101  yamaguch 	aprint_normal_dev(self, "Ethernet address %s\n",
    539  1.101  yamaguch 	    ether_sprintf(sc->sc_mac));
    540  1.101  yamaguch 
    541  1.101  yamaguch 	if (features & (VIRTIO_NET_F_MRG_RXBUF | VIRTIO_F_VERSION_1)) {
    542  1.101  yamaguch 		sc->sc_hdr_size = sizeof(struct virtio_net_hdr);
    543  1.101  yamaguch 	} else {
    544  1.101  yamaguch 		sc->sc_hdr_size = offsetof(struct virtio_net_hdr, num_buffers);
    545  1.101  yamaguch 	}
    546   1.58  yamaguch 
    547  1.101  yamaguch 	if ((features & VIRTIO_NET_F_CTRL_VQ) &&
    548  1.101  yamaguch 	    (features & VIRTIO_NET_F_CTRL_RX)) {
    549  1.101  yamaguch 		sc->sc_has_ctrl = true;
    550   1.58  yamaguch 
    551  1.101  yamaguch 		cv_init(&ctrlq->ctrlq_wait, "ctrl_vq");
    552  1.101  yamaguch 		mutex_init(&ctrlq->ctrlq_wait_lock, MUTEX_DEFAULT, IPL_NET);
    553  1.101  yamaguch 		ctrlq->ctrlq_inuse = FREE;
    554  1.101  yamaguch 	} else {
    555  1.101  yamaguch 		sc->sc_has_ctrl = false;
    556   1.58  yamaguch 	}
    557   1.58  yamaguch 
    558  1.101  yamaguch 	if (sc->sc_has_ctrl && (features & VIRTIO_NET_F_MQ)) {
    559  1.101  yamaguch 		sc->sc_max_nvq_pairs = virtio_read_device_config_2(vsc,
    560  1.101  yamaguch 		    VIRTIO_NET_CONFIG_MAX_VQ_PAIRS);
    561  1.101  yamaguch 
    562  1.101  yamaguch 		if (sc->sc_max_nvq_pairs > VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX)
    563  1.101  yamaguch 			goto err;
    564   1.58  yamaguch 
    565  1.101  yamaguch 		/* Limit the number of queue pairs to use */
    566  1.101  yamaguch 		sc->sc_req_nvq_pairs = MIN(sc->sc_max_nvq_pairs, ncpu);
    567  1.101  yamaguch 	}
    568   1.58  yamaguch 
    569  1.101  yamaguch 	vioif_alloc_queues(sc);
    570  1.101  yamaguch 	virtio_child_attach_set_vqs(vsc, sc->sc_vqs, sc->sc_req_nvq_pairs);
    571   1.58  yamaguch 
    572  1.101  yamaguch #ifdef VIOIF_MPSAFE
    573  1.101  yamaguch 	softint_flags = SOFTINT_NET | SOFTINT_MPSAFE;
    574  1.101  yamaguch #else
    575  1.101  yamaguch 	softint_flags = SOFTINT_NET;
    576  1.101  yamaguch #endif
    577   1.58  yamaguch 
    578  1.101  yamaguch 	/*
    579  1.101  yamaguch 	 * Initialize network queues
    580  1.101  yamaguch 	 */
    581  1.101  yamaguch 	netq_num = sc->sc_max_nvq_pairs * 2;
    582  1.101  yamaguch 	for (i = 0; i < netq_num; i++) {
    583  1.101  yamaguch 		r = vioif_netqueue_init(sc, vsc, i, softint_flags);
    584  1.101  yamaguch 		if (r != 0)
    585  1.101  yamaguch 			goto err;
    586  1.101  yamaguch 	}
    587   1.46  yamaguch 
    588  1.101  yamaguch 	if (sc->sc_has_ctrl) {
    589  1.101  yamaguch 		int ctrlq_idx = sc->sc_max_nvq_pairs * 2;
    590  1.101  yamaguch 		/*
    591  1.101  yamaguch 		 * Allocating a virtqueue for control channel
    592  1.101  yamaguch 		 */
    593  1.101  yamaguch 		sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[ctrlq_idx];
    594  1.101  yamaguch 		r = virtio_alloc_vq(vsc, ctrlq->ctrlq_vq, ctrlq_idx,
    595  1.101  yamaguch 		    NBPG, 1, "control");
    596  1.101  yamaguch 		if (r != 0) {
    597  1.101  yamaguch 			aprint_error_dev(self, "failed to allocate "
    598  1.101  yamaguch 			    "a virtqueue for control channel, error code %d\n",
    599  1.101  yamaguch 			    r);
    600   1.46  yamaguch 
    601  1.101  yamaguch 			sc->sc_has_ctrl = false;
    602  1.101  yamaguch 			cv_destroy(&ctrlq->ctrlq_wait);
    603  1.101  yamaguch 			mutex_destroy(&ctrlq->ctrlq_wait_lock);
    604  1.101  yamaguch 		} else {
    605  1.101  yamaguch 			ctrlq->ctrlq_vq->vq_intrhand = vioif_ctrl_intr;
    606  1.101  yamaguch 			ctrlq->ctrlq_vq->vq_intrhand_arg = (void *) ctrlq;
    607  1.101  yamaguch 		}
    608  1.101  yamaguch 	}
    609   1.46  yamaguch 
    610  1.101  yamaguch 	sc->sc_cfg_softint = softint_establish(softint_flags,
    611  1.101  yamaguch 	    vioif_cfg_softint, sc);
    612  1.101  yamaguch 	if (sc->sc_cfg_softint == NULL) {
    613  1.101  yamaguch 		aprint_error_dev(self, "cannot establish ctl softint\n");
    614  1.101  yamaguch 		goto err;
    615  1.101  yamaguch 	}
    616   1.46  yamaguch 
    617  1.101  yamaguch 	if (vioif_alloc_mems(sc) < 0)
    618  1.101  yamaguch 		goto err;
    619   1.46  yamaguch 
    620  1.101  yamaguch 	if (virtio_child_attach_finish(vsc) != 0)
    621  1.101  yamaguch 		goto err;
    622   1.46  yamaguch 
    623  1.101  yamaguch 	if (vioif_setup_sysctl(sc) != 0) {
    624  1.101  yamaguch 		aprint_error_dev(self, "unable to create sysctl node\n");
    625  1.101  yamaguch 		/* continue */
    626  1.101  yamaguch 	}
    627   1.98  yamaguch 
    628  1.101  yamaguch 	vioif_setup_stats(sc);
    629   1.98  yamaguch 
    630  1.101  yamaguch 	strlcpy(ifp->if_xname, device_xname(self), IFNAMSIZ);
    631  1.101  yamaguch 	ifp->if_softc = sc;
    632  1.101  yamaguch 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
    633  1.101  yamaguch #ifdef VIOIF_MPSAFE
    634  1.101  yamaguch 	ifp->if_extflags = IFEF_MPSAFE;
    635  1.101  yamaguch #endif
    636  1.101  yamaguch 	ifp->if_start = vioif_start;
    637  1.101  yamaguch 	if (sc->sc_req_nvq_pairs > 1)
    638  1.101  yamaguch 		ifp->if_transmit = vioif_transmit;
    639  1.101  yamaguch 	ifp->if_ioctl = vioif_ioctl;
    640  1.101  yamaguch 	ifp->if_init = vioif_init;
    641  1.101  yamaguch 	ifp->if_stop = vioif_stop;
    642  1.101  yamaguch 	ifp->if_capabilities = 0;
    643  1.101  yamaguch 	ifp->if_watchdog = vioif_watchdog;
    644  1.101  yamaguch 	txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
    645  1.101  yamaguch 	IFQ_SET_MAXLEN(&ifp->if_snd, MAX(txq0->netq_vq->vq_num, IFQ_MAXLEN));
    646  1.101  yamaguch 	IFQ_SET_READY(&ifp->if_snd);
    647   1.98  yamaguch 
    648  1.101  yamaguch 	sc->sc_ethercom.ec_capabilities |= ETHERCAP_VLAN_MTU;
    649   1.98  yamaguch 
    650  1.101  yamaguch 	if_attach(ifp);
    651  1.101  yamaguch 	if_deferred_start_init(ifp, NULL);
    652  1.101  yamaguch 	ether_ifattach(ifp, sc->sc_mac);
    653  1.101  yamaguch 	ether_set_ifflags_cb(&sc->sc_ethercom, vioif_ifflags_cb);
    654   1.98  yamaguch 
    655  1.101  yamaguch 	return;
    656   1.98  yamaguch 
    657   1.98  yamaguch err:
    658  1.101  yamaguch 	netq_num = sc->sc_max_nvq_pairs * 2;
    659  1.101  yamaguch 	for (i = 0; i < netq_num; i++) {
    660  1.101  yamaguch 		vioif_netqueue_teardown(sc, vsc, i);
    661   1.98  yamaguch 	}
    662   1.98  yamaguch 
    663  1.101  yamaguch 	if (sc->sc_has_ctrl) {
    664  1.101  yamaguch 		cv_destroy(&ctrlq->ctrlq_wait);
    665  1.101  yamaguch 		mutex_destroy(&ctrlq->ctrlq_wait_lock);
    666  1.101  yamaguch 		virtio_free_vq(vsc, ctrlq->ctrlq_vq);
    667  1.101  yamaguch 		ctrlq->ctrlq_vq = NULL;
    668   1.98  yamaguch 	}
    669   1.98  yamaguch 
    670  1.101  yamaguch 	vioif_free_queues(sc);
    671  1.101  yamaguch 	mutex_destroy(&sc->sc_lock);
    672  1.101  yamaguch 	virtio_child_attach_failed(vsc);
    673  1.101  yamaguch 	config_finalize_register(self, vioif_finalize_teardown);
    674   1.98  yamaguch 
    675  1.101  yamaguch 	return;
    676   1.98  yamaguch }
    677   1.98  yamaguch 
    678  1.101  yamaguch static int
    679  1.101  yamaguch vioif_finalize_teardown(device_t self)
    680   1.98  yamaguch {
    681  1.101  yamaguch 	struct vioif_softc *sc = device_private(self);
    682   1.98  yamaguch 
    683  1.101  yamaguch 	if (sc->sc_txrx_workqueue != NULL) {
    684  1.101  yamaguch 		vioif_workq_destroy(sc->sc_txrx_workqueue);
    685  1.101  yamaguch 		sc->sc_txrx_workqueue = NULL;
    686   1.46  yamaguch 	}
    687   1.98  yamaguch 
    688  1.101  yamaguch 	return 0;
    689   1.46  yamaguch }
    690   1.46  yamaguch 
    691    1.1   hannken /*
    692  1.101  yamaguch  * Interface functions for ifnet
    693    1.1   hannken  */
    694    1.1   hannken static int
    695  1.101  yamaguch vioif_init(struct ifnet *ifp)
    696    1.1   hannken {
    697  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    698    1.1   hannken 	struct virtio_softc *vsc = sc->sc_virtio;
    699   1.98  yamaguch 	struct vioif_netqueue *netq;
    700   1.98  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    701  1.101  yamaguch 	int r, i;
    702  1.101  yamaguch 
    703  1.101  yamaguch 	vioif_stop(ifp, 0);
    704  1.101  yamaguch 
    705  1.101  yamaguch 	r = virtio_reinit_start(vsc);
    706  1.101  yamaguch 	if (r != 0) {
    707  1.101  yamaguch 		log(LOG_ERR, "%s: reset failed\n", ifp->if_xname);
    708  1.101  yamaguch 		return EIO;
    709  1.101  yamaguch 	}
    710    1.1   hannken 
    711  1.101  yamaguch 	virtio_negotiate_features(vsc, virtio_features(vsc));
    712   1.98  yamaguch 
    713  1.101  yamaguch 	for (i = 0; i < sc->sc_req_nvq_pairs; i++) {
    714  1.101  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
    715   1.46  yamaguch 
    716  1.101  yamaguch 		mutex_enter(&netq->netq_lock);
    717  1.101  yamaguch 		vioif_populate_rx_mbufs_locked(sc, netq);
    718  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    719   1.46  yamaguch 	}
    720   1.98  yamaguch 
    721  1.101  yamaguch 	virtio_reinit_end(vsc);
    722  1.101  yamaguch 
    723  1.101  yamaguch 	if (sc->sc_has_ctrl)
    724  1.101  yamaguch 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
    725   1.98  yamaguch 
    726  1.101  yamaguch 	r = vioif_ctrl_mq_vq_pairs_set(sc, sc->sc_req_nvq_pairs);
    727  1.101  yamaguch 	if (r == 0)
    728  1.101  yamaguch 		sc->sc_act_nvq_pairs = sc->sc_req_nvq_pairs;
    729  1.101  yamaguch 	else
    730  1.101  yamaguch 		sc->sc_act_nvq_pairs = 1;
    731  1.101  yamaguch 
    732  1.101  yamaguch 	SET(ifp->if_flags, IFF_RUNNING);
    733  1.101  yamaguch 	CLR(ifp->if_flags, IFF_OACTIVE);
    734   1.42  yamaguch 
    735  1.101  yamaguch 	vioif_net_intr_enable(sc, vsc);
    736   1.42  yamaguch 
    737  1.101  yamaguch 	vioif_update_link_status(sc);
    738  1.101  yamaguch 	r = vioif_rx_filter(sc);
    739   1.98  yamaguch 
    740  1.101  yamaguch 	return r;
    741  1.101  yamaguch }
    742   1.98  yamaguch 
    743  1.101  yamaguch static void
    744  1.101  yamaguch vioif_stop(struct ifnet *ifp, int disable)
    745  1.101  yamaguch {
    746  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    747  1.101  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
    748  1.101  yamaguch 	struct vioif_netqueue *netq;
    749  1.101  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
    750  1.101  yamaguch 	size_t i, act_qnum;
    751    1.1   hannken 
    752  1.101  yamaguch 	act_qnum = sc->sc_act_nvq_pairs * 2;
    753   1.46  yamaguch 
    754  1.101  yamaguch 	CLR(ifp->if_flags, IFF_RUNNING);
    755  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
    756  1.101  yamaguch 		netq = &sc->sc_netqs[i];
    757   1.95  yamaguch 
    758  1.101  yamaguch 		mutex_enter(&netq->netq_lock);
    759  1.101  yamaguch 		netq->netq_stopping = true;
    760  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    761   1.98  yamaguch 	}
    762   1.46  yamaguch 
    763  1.101  yamaguch 	/* disable interrupts */
    764  1.101  yamaguch 	vioif_net_intr_disable(sc, vsc);
    765  1.101  yamaguch 	if (sc->sc_has_ctrl)
    766  1.101  yamaguch 		virtio_stop_vq_intr(vsc, ctrlq->ctrlq_vq);
    767  1.101  yamaguch 
    768  1.101  yamaguch 	/*
    769  1.101  yamaguch 	 * only way to stop interrupt, I/O and DMA is resetting...
    770  1.101  yamaguch 	 *
    771  1.101  yamaguch 	 * NOTE: Devices based on VirtIO draft specification can not
    772  1.101  yamaguch 	 * stop interrupt completely even if virtio_stop_vq_intr() is called.
    773  1.101  yamaguch 	 */
    774  1.101  yamaguch 	virtio_reset(vsc);
    775  1.101  yamaguch 
    776  1.101  yamaguch 	vioif_intr_barrier();
    777   1.98  yamaguch 
    778  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
    779  1.101  yamaguch 		netq = &sc->sc_netqs[i];
    780  1.101  yamaguch 		vioif_work_wait(sc->sc_txrx_workqueue, &netq->netq_work);
    781  1.101  yamaguch 	}
    782   1.42  yamaguch 
    783  1.101  yamaguch 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
    784  1.101  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_RXQID(i)];
    785  1.101  yamaguch 		vioif_rx_queue_clear(sc, vsc, netq);
    786   1.98  yamaguch 
    787  1.101  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
    788  1.101  yamaguch 		vioif_tx_queue_clear(sc, vsc, netq);
    789   1.98  yamaguch 	}
    790   1.98  yamaguch 
    791  1.101  yamaguch 	/* all packet processing is stopped */
    792  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
    793  1.101  yamaguch 		netq = &sc->sc_netqs[i];
    794    1.1   hannken 
    795  1.101  yamaguch 		mutex_enter(&netq->netq_lock);
    796  1.101  yamaguch 		netq->netq_stopping = false;
    797  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    798  1.101  yamaguch 	}
    799  1.101  yamaguch }
    800   1.95  yamaguch 
    801  1.101  yamaguch static void
    802  1.101  yamaguch vioif_start(struct ifnet *ifp)
    803  1.101  yamaguch {
    804  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    805  1.101  yamaguch 	struct vioif_netqueue *txq0 = &sc->sc_netqs[VIOIF_NETQ_TXQID(0)];
    806   1.58  yamaguch 
    807  1.101  yamaguch #ifdef VIOIF_MPSAFE
    808  1.101  yamaguch 	KASSERT(if_is_mpsafe(ifp));
    809  1.101  yamaguch #endif
    810   1.58  yamaguch 
    811  1.101  yamaguch 	mutex_enter(&txq0->netq_lock);
    812  1.101  yamaguch 	vioif_start_locked(ifp, txq0);
    813  1.101  yamaguch 	mutex_exit(&txq0->netq_lock);
    814  1.101  yamaguch }
    815    1.1   hannken 
    816  1.101  yamaguch static inline int
    817  1.101  yamaguch vioif_select_txqueue(struct ifnet *ifp, struct mbuf *m)
    818  1.101  yamaguch {
    819  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    820  1.101  yamaguch 	u_int cpuid = cpu_index(curcpu());
    821   1.58  yamaguch 
    822  1.101  yamaguch 	return VIOIF_NETQ_TXQID(cpuid % sc->sc_act_nvq_pairs);
    823  1.101  yamaguch }
    824    1.1   hannken 
    825  1.101  yamaguch static int
    826  1.101  yamaguch vioif_transmit(struct ifnet *ifp, struct mbuf *m)
    827  1.101  yamaguch {
    828  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    829  1.101  yamaguch 	struct vioif_netqueue *netq;
    830  1.101  yamaguch 	struct vioif_tx_context *txc;
    831  1.101  yamaguch 	int qid;
    832    1.1   hannken 
    833  1.101  yamaguch 	qid = vioif_select_txqueue(ifp, m);
    834  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
    835  1.101  yamaguch 	txc = netq->netq_ctx;
    836   1.46  yamaguch 
    837  1.101  yamaguch 	if (__predict_false(!pcq_put(txc->txc_intrq, m))) {
    838  1.101  yamaguch 		m_freem(m);
    839  1.101  yamaguch 		return ENOBUFS;
    840  1.101  yamaguch 	}
    841    1.1   hannken 
    842  1.101  yamaguch 	net_stat_ref_t nsr = IF_STAT_GETREF(ifp);
    843  1.101  yamaguch 	if_statadd_ref(nsr, if_obytes, m->m_pkthdr.len);
    844  1.101  yamaguch 	if (m->m_flags & M_MCAST)
    845  1.101  yamaguch 		if_statinc_ref(nsr, if_omcasts);
    846  1.101  yamaguch 	IF_STAT_PUTREF(ifp);
    847   1.74  yamaguch 
    848  1.101  yamaguch 	if (mutex_tryenter(&netq->netq_lock)) {
    849  1.101  yamaguch 		vioif_transmit_locked(ifp, netq);
    850  1.101  yamaguch 		mutex_exit(&netq->netq_lock);
    851    1.1   hannken 	}
    852    1.1   hannken 
    853    1.1   hannken 	return 0;
    854  1.101  yamaguch }
    855    1.1   hannken 
    856  1.101  yamaguch void
    857  1.101  yamaguch vioif_watchdog(struct ifnet *ifp)
    858  1.101  yamaguch {
    859  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    860  1.101  yamaguch 	struct vioif_netqueue *netq;
    861  1.101  yamaguch 	int i;
    862  1.101  yamaguch 
    863  1.101  yamaguch 	if (ifp->if_flags & IFF_RUNNING) {
    864  1.101  yamaguch 		for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
    865  1.101  yamaguch 			netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
    866   1.46  yamaguch 
    867  1.101  yamaguch 			mutex_enter(&netq->netq_lock);
    868  1.101  yamaguch 			if (!netq->netq_running_handle) {
    869  1.101  yamaguch 				netq->netq_running_handle = true;
    870  1.101  yamaguch 				vioif_net_sched_handle(sc, netq);
    871  1.101  yamaguch 			}
    872  1.101  yamaguch 			mutex_exit(&netq->netq_lock);
    873   1.46  yamaguch 		}
    874    1.1   hannken 	}
    875  1.101  yamaguch }
    876  1.101  yamaguch 
    877  1.101  yamaguch static int
    878  1.101  yamaguch vioif_ioctl(struct ifnet *ifp, u_long cmd, void *data)
    879  1.101  yamaguch {
    880  1.101  yamaguch 	int s, r;
    881  1.101  yamaguch 
    882  1.101  yamaguch 	s = splnet();
    883  1.101  yamaguch 
    884  1.101  yamaguch 	r = ether_ioctl(ifp, cmd, data);
    885  1.101  yamaguch 	if (r == ENETRESET && (cmd == SIOCADDMULTI || cmd == SIOCDELMULTI)) {
    886  1.101  yamaguch 		if (ifp->if_flags & IFF_RUNNING) {
    887  1.101  yamaguch 			r = vioif_rx_filter(ifp->if_softc);
    888  1.101  yamaguch 		} else {
    889  1.101  yamaguch 			r = 0;
    890  1.101  yamaguch 		}
    891    1.1   hannken 	}
    892  1.101  yamaguch 
    893  1.101  yamaguch 	splx(s);
    894  1.101  yamaguch 
    895  1.101  yamaguch 	return r;
    896    1.1   hannken }
    897    1.1   hannken 
    898  1.101  yamaguch static int
    899  1.101  yamaguch vioif_ifflags(struct vioif_softc *sc)
    900    1.1   hannken {
    901    1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
    902  1.101  yamaguch 	bool onoff;
    903  1.101  yamaguch 	int r;
    904    1.1   hannken 
    905  1.101  yamaguch 	if (!sc->sc_has_ctrl) {
    906  1.101  yamaguch 		/* no ctrl vq; always promisc and allmulti */
    907  1.101  yamaguch 		ifp->if_flags |= (IFF_PROMISC | IFF_ALLMULTI);
    908  1.101  yamaguch 		return 0;
    909    1.1   hannken 	}
    910    1.1   hannken 
    911  1.101  yamaguch 	onoff = ifp->if_flags & IFF_ALLMULTI ? true : false;
    912  1.101  yamaguch 	r = vioif_set_allmulti(sc, onoff);
    913  1.101  yamaguch 	if (r != 0) {
    914  1.101  yamaguch 		log(LOG_WARNING,
    915  1.101  yamaguch 		    "%s: couldn't %sable ALLMULTI\n",
    916  1.101  yamaguch 		    ifp->if_xname, onoff ? "en" : "dis");
    917  1.101  yamaguch 		if (onoff == false) {
    918  1.101  yamaguch 			ifp->if_flags |= IFF_ALLMULTI;
    919  1.101  yamaguch 		}
    920  1.101  yamaguch 	}
    921    1.1   hannken 
    922  1.101  yamaguch 	onoff = ifp->if_flags & IFF_PROMISC ? true : false;
    923  1.101  yamaguch 	r = vioif_set_promisc(sc, onoff);
    924  1.101  yamaguch 	if (r != 0) {
    925  1.101  yamaguch 		log(LOG_WARNING,
    926  1.101  yamaguch 		    "%s: couldn't %sable PROMISC\n",
    927  1.101  yamaguch 		    ifp->if_xname, onoff ? "en" : "dis");
    928  1.101  yamaguch 		if (onoff == false) {
    929  1.101  yamaguch 			ifp->if_flags |= IFF_PROMISC;
    930  1.101  yamaguch 		}
    931  1.101  yamaguch 	}
    932   1.55  yamaguch 
    933  1.101  yamaguch 	return 0;
    934  1.101  yamaguch }
    935   1.62  yamaguch 
    936  1.101  yamaguch static int
    937  1.101  yamaguch vioif_ifflags_cb(struct ethercom *ec)
    938  1.101  yamaguch {
    939  1.101  yamaguch 	struct ifnet *ifp = &ec->ec_if;
    940  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
    941   1.46  yamaguch 
    942  1.101  yamaguch 	return vioif_ifflags(sc);
    943  1.101  yamaguch }
    944    1.1   hannken 
    945  1.101  yamaguch static int
    946  1.101  yamaguch vioif_setup_sysctl(struct vioif_softc *sc)
    947  1.101  yamaguch {
    948  1.101  yamaguch 	const char *devname;
    949  1.101  yamaguch 	struct sysctllog **log;
    950  1.101  yamaguch 	const struct sysctlnode *rnode, *rxnode, *txnode;
    951  1.101  yamaguch 	int error;
    952   1.32  jdolecek 
    953  1.101  yamaguch 	log = &sc->sc_sysctllog;
    954  1.101  yamaguch 	devname = device_xname(sc->sc_dev);
    955   1.32  jdolecek 
    956  1.101  yamaguch 	error = sysctl_createv(log, 0, NULL, &rnode,
    957  1.101  yamaguch 	    0, CTLTYPE_NODE, devname,
    958  1.101  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings"),
    959  1.101  yamaguch 	    NULL, 0, NULL, 0, CTL_HW, CTL_CREATE, CTL_EOL);
    960  1.101  yamaguch 	if (error)
    961  1.101  yamaguch 		goto out;
    962    1.7     ozaki 
    963  1.101  yamaguch 	error = sysctl_createv(log, 0, &rnode, NULL,
    964  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_BOOL, "txrx_workqueue",
    965  1.101  yamaguch 	    SYSCTL_DESCR("Use workqueue for packet processing"),
    966  1.101  yamaguch 	    NULL, 0, &sc->sc_txrx_workqueue_sysctl, 0, CTL_CREATE, CTL_EOL);
    967  1.101  yamaguch 	if (error)
    968  1.101  yamaguch 		goto out;
    969   1.32  jdolecek 
    970  1.101  yamaguch 	error = sysctl_createv(log, 0, &rnode, &rxnode,
    971  1.101  yamaguch 	    0, CTLTYPE_NODE, "rx",
    972  1.101  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Rx"),
    973  1.101  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
    974  1.101  yamaguch 	if (error)
    975  1.101  yamaguch 		goto out;
    976    1.1   hannken 
    977  1.101  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
    978  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
    979  1.101  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for interrupt processing"),
    980  1.101  yamaguch 	    NULL, 0, &sc->sc_rx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
    981  1.101  yamaguch 	if (error)
    982  1.101  yamaguch 		goto out;
    983   1.66   reinoud 
    984  1.101  yamaguch 	error = sysctl_createv(log, 0, &rxnode, NULL,
    985  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
    986  1.101  yamaguch 	    SYSCTL_DESCR("max number of Rx packets to process for deferred processing"),
    987  1.101  yamaguch 	    NULL, 0, &sc->sc_rx_process_limit, 0, CTL_CREATE, CTL_EOL);
    988  1.101  yamaguch 	if (error)
    989  1.101  yamaguch 		goto out;
    990   1.46  yamaguch 
    991  1.101  yamaguch 	error = sysctl_createv(log, 0, &rnode, &txnode,
    992  1.101  yamaguch 	    0, CTLTYPE_NODE, "tx",
    993  1.101  yamaguch 	    SYSCTL_DESCR("virtio-net information and settings for Tx"),
    994  1.101  yamaguch 	    NULL, 0, NULL, 0, CTL_CREATE, CTL_EOL);
    995  1.101  yamaguch 	if (error)
    996  1.101  yamaguch 		goto out;
    997   1.46  yamaguch 
    998  1.101  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
    999  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "intr_process_limit",
   1000  1.101  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for interrupt processing"),
   1001  1.101  yamaguch 	    NULL, 0, &sc->sc_tx_intr_process_limit, 0, CTL_CREATE, CTL_EOL);
   1002  1.101  yamaguch 	if (error)
   1003  1.101  yamaguch 		goto out;
   1004   1.46  yamaguch 
   1005  1.101  yamaguch 	error = sysctl_createv(log, 0, &txnode, NULL,
   1006  1.101  yamaguch 	    CTLFLAG_READWRITE, CTLTYPE_INT, "process_limit",
   1007  1.101  yamaguch 	    SYSCTL_DESCR("max number of Tx packets to process for deferred processing"),
   1008  1.101  yamaguch 	    NULL, 0, &sc->sc_tx_process_limit, 0, CTL_CREATE, CTL_EOL);
   1009   1.46  yamaguch 
   1010  1.101  yamaguch out:
   1011  1.101  yamaguch 	if (error)
   1012  1.101  yamaguch 		sysctl_teardown(log);
   1013   1.46  yamaguch 
   1014  1.101  yamaguch 	return error;
   1015  1.101  yamaguch }
   1016   1.46  yamaguch 
   1017  1.101  yamaguch static void
   1018  1.101  yamaguch vioif_setup_stats(struct vioif_softc *sc)
   1019  1.101  yamaguch {
   1020  1.101  yamaguch 	struct vioif_netqueue *netq;
   1021  1.101  yamaguch 	struct vioif_tx_context *txc;
   1022  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1023  1.101  yamaguch 	size_t i, netq_num;
   1024    1.7     ozaki 
   1025   1.98  yamaguch 	netq_num = sc->sc_max_nvq_pairs * 2;
   1026   1.98  yamaguch 	for (i = 0; i < netq_num; i++) {
   1027  1.101  yamaguch 		netq = &sc->sc_netqs[i];
   1028  1.101  yamaguch 		evcnt_attach_dynamic(&netq->netq_mbuf_load_failed, EVCNT_TYPE_MISC,
   1029  1.101  yamaguch 		    NULL, netq->netq_evgroup, "failed to load mbuf to DMA");
   1030  1.101  yamaguch 		evcnt_attach_dynamic(&netq->netq_enqueue_failed,
   1031  1.101  yamaguch 		    EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
   1032  1.101  yamaguch 		    "virtqueue enqueue failed failed");
   1033   1.17     ozaki 
   1034  1.101  yamaguch 		switch (VIOIF_NETQ_DIR(i)) {
   1035  1.101  yamaguch 		case VIOIF_NETQ_RX:
   1036  1.101  yamaguch 			rxc = netq->netq_ctx;
   1037  1.101  yamaguch 			evcnt_attach_dynamic(&rxc->rxc_mbuf_enobufs,
   1038  1.101  yamaguch 			    EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
   1039  1.101  yamaguch 			    "no receive buffer");
   1040  1.101  yamaguch 			break;
   1041  1.101  yamaguch 		case VIOIF_NETQ_TX:
   1042  1.101  yamaguch 			txc = netq->netq_ctx;
   1043  1.101  yamaguch 			evcnt_attach_dynamic(&txc->txc_defrag_failed,
   1044  1.101  yamaguch 			    EVCNT_TYPE_MISC, NULL, netq->netq_evgroup,
   1045  1.101  yamaguch 			    "m_defrag() failed");
   1046  1.101  yamaguch 			break;
   1047    1.1   hannken 		}
   1048    1.1   hannken 	}
   1049   1.34     ozaki 
   1050  1.101  yamaguch 	evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_load_failed, EVCNT_TYPE_MISC,
   1051  1.101  yamaguch 	    NULL, device_xname(sc->sc_dev), "control command dmamap load failed");
   1052  1.101  yamaguch 	evcnt_attach_dynamic(&sc->sc_ctrlq.ctrlq_cmd_failed, EVCNT_TYPE_MISC,
   1053  1.101  yamaguch 	    NULL, device_xname(sc->sc_dev), "control command failed");
   1054  1.101  yamaguch }
   1055    1.1   hannken 
   1056  1.101  yamaguch /*
   1057  1.101  yamaguch  * allocate memory
   1058  1.101  yamaguch  */
   1059  1.101  yamaguch static int
   1060  1.101  yamaguch vioif_dmamap_create(struct vioif_softc *sc, bus_dmamap_t *map,
   1061  1.101  yamaguch     bus_size_t size, int nsegs, const char *usage)
   1062  1.101  yamaguch {
   1063  1.101  yamaguch 	int r;
   1064    1.1   hannken 
   1065  1.101  yamaguch 	r = bus_dmamap_create(virtio_dmat(sc->sc_virtio), size,
   1066  1.101  yamaguch 	    nsegs, size, 0, BUS_DMA_NOWAIT | BUS_DMA_ALLOCNOW, map);
   1067   1.32  jdolecek 
   1068  1.101  yamaguch 	if (r != 0) {
   1069  1.101  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap creation failed, "
   1070  1.101  yamaguch 		    "error code %d\n", usage, r);
   1071   1.55  yamaguch 	}
   1072   1.55  yamaguch 
   1073  1.101  yamaguch 	return r;
   1074  1.101  yamaguch }
   1075   1.63  yamaguch 
   1076  1.101  yamaguch static void
   1077  1.101  yamaguch vioif_dmamap_destroy(struct vioif_softc *sc, bus_dmamap_t *map)
   1078  1.101  yamaguch {
   1079    1.1   hannken 
   1080  1.101  yamaguch 	if (*map) {
   1081  1.101  yamaguch 		bus_dmamap_destroy(virtio_dmat(sc->sc_virtio), *map);
   1082  1.101  yamaguch 		*map = NULL;
   1083  1.101  yamaguch 	}
   1084  1.101  yamaguch }
   1085   1.11     ozaki 
   1086  1.101  yamaguch static int
   1087  1.101  yamaguch vioif_dmamap_create_load(struct vioif_softc *sc, bus_dmamap_t *map,
   1088  1.101  yamaguch     void *buf, bus_size_t size, int nsegs, int rw, const char *usage)
   1089  1.101  yamaguch {
   1090  1.101  yamaguch 	int r;
   1091    1.1   hannken 
   1092  1.101  yamaguch 	r = vioif_dmamap_create(sc, map, size, nsegs, usage);
   1093  1.101  yamaguch 	if (r != 0)
   1094  1.101  yamaguch 		return 1;
   1095    1.1   hannken 
   1096  1.101  yamaguch 	r = bus_dmamap_load(virtio_dmat(sc->sc_virtio), *map, buf,
   1097  1.101  yamaguch 	    size, NULL, rw | BUS_DMA_NOWAIT);
   1098  1.101  yamaguch 	if (r != 0) {
   1099  1.101  yamaguch 		vioif_dmamap_destroy(sc, map);
   1100  1.101  yamaguch 		aprint_error_dev(sc->sc_dev, "%s dmamap load failed. "
   1101  1.101  yamaguch 		    "error code %d\n", usage, r);
   1102   1.43  yamaguch 	}
   1103    1.7     ozaki 
   1104  1.101  yamaguch 	return r;
   1105    1.1   hannken }
   1106    1.1   hannken 
   1107  1.101  yamaguch static void *
   1108  1.101  yamaguch vioif_assign_mem(intptr_t *p, size_t size)
   1109   1.55  yamaguch {
   1110  1.101  yamaguch 	intptr_t rv;
   1111   1.55  yamaguch 
   1112  1.101  yamaguch 	rv = *p;
   1113  1.101  yamaguch 	*p += size;
   1114   1.55  yamaguch 
   1115  1.101  yamaguch 	return (void *)rv;
   1116   1.55  yamaguch }
   1117   1.55  yamaguch 
   1118    1.1   hannken /*
   1119  1.101  yamaguch  * dma memory is used for:
   1120  1.101  yamaguch  *   netq_maps_kva:	 metadata array for received frames (READ) and
   1121  1.101  yamaguch  *			 sent frames (WRITE)
   1122  1.101  yamaguch  *   ctrlq_cmd:		 command to be sent via ctrl vq (WRITE)
   1123  1.101  yamaguch  *   ctrlq_status:	 return value for a command via ctrl vq (READ)
   1124  1.101  yamaguch  *   ctrlq_rx:		 parameter for a VIRTIO_NET_CTRL_RX class command
   1125  1.101  yamaguch  *			 (WRITE)
   1126  1.101  yamaguch  *   ctrlq_mac_tbl_uc:	 unicast MAC address filter for a VIRTIO_NET_CTRL_MAC
   1127  1.101  yamaguch  *			 class command (WRITE)
   1128  1.101  yamaguch  *   ctrlq_mac_tbl_mc:	 multicast MAC address filter for a VIRTIO_NET_CTRL_MAC
   1129  1.101  yamaguch  *			 class command (WRITE)
   1130  1.101  yamaguch  * ctrlq_* structures are allocated only one each; they are protected by
   1131  1.101  yamaguch  * ctrlq_inuse variable and ctrlq_wait condvar.
   1132    1.1   hannken  */
   1133    1.1   hannken static int
   1134  1.101  yamaguch vioif_alloc_mems(struct vioif_softc *sc)
   1135    1.1   hannken {
   1136   1.33     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   1137   1.98  yamaguch 	struct vioif_netqueue *netq;
   1138   1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   1139  1.101  yamaguch 	struct vioif_net_map *maps;
   1140  1.101  yamaguch 	unsigned int vq_num;
   1141  1.101  yamaguch 	int r, rsegs;
   1142  1.101  yamaguch 	bus_size_t dmamemsize;
   1143  1.101  yamaguch 	size_t qid, i, netq_num, kmemsize;
   1144  1.101  yamaguch 	void *vaddr;
   1145  1.101  yamaguch 	intptr_t p;
   1146  1.101  yamaguch 
   1147  1.101  yamaguch 	netq_num = sc->sc_max_nvq_pairs * 2;
   1148    1.1   hannken 
   1149  1.101  yamaguch 	/* allocate DMA memory */
   1150  1.101  yamaguch 	dmamemsize = 0;
   1151    1.7     ozaki 
   1152  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1153  1.101  yamaguch 		maps = sc->sc_netqs[qid].netq_maps;
   1154  1.101  yamaguch 		vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
   1155  1.101  yamaguch 		dmamemsize += sizeof(*maps[0].vnm_hdr) * vq_num;
   1156   1.71  yamaguch 	}
   1157   1.71  yamaguch 
   1158  1.101  yamaguch 	if (sc->sc_has_ctrl) {
   1159  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_cmd);
   1160  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_status);
   1161  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_rx);
   1162  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
   1163  1.101  yamaguch 		    + ETHER_ADDR_LEN;
   1164  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mac_tbl)
   1165  1.101  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES;
   1166  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mac_addr);
   1167  1.101  yamaguch 		dmamemsize += sizeof(struct virtio_net_ctrl_mq);
   1168  1.101  yamaguch 	}
   1169   1.46  yamaguch 
   1170  1.101  yamaguch 	r = bus_dmamem_alloc(virtio_dmat(vsc), dmamemsize, 0, 0,
   1171  1.101  yamaguch 	    &sc->sc_segs[0], 1, &rsegs, BUS_DMA_NOWAIT);
   1172  1.101  yamaguch 	if (r != 0) {
   1173  1.101  yamaguch 		aprint_error_dev(sc->sc_dev,
   1174  1.101  yamaguch 		    "DMA memory allocation failed, size %zu, "
   1175  1.101  yamaguch 		    "error code %d\n", dmamemsize, r);
   1176  1.101  yamaguch 		goto err_none;
   1177  1.101  yamaguch 	}
   1178  1.101  yamaguch 	r = bus_dmamem_map(virtio_dmat(vsc), &sc->sc_segs[0], 1,
   1179  1.101  yamaguch 	    dmamemsize, &vaddr, BUS_DMA_NOWAIT);
   1180  1.101  yamaguch 	if (r != 0) {
   1181  1.101  yamaguch 		aprint_error_dev(sc->sc_dev,
   1182  1.101  yamaguch 		    "DMA memory map failed, error code %d\n", r);
   1183  1.101  yamaguch 		goto err_dmamem_alloc;
   1184   1.46  yamaguch 	}
   1185   1.46  yamaguch 
   1186  1.101  yamaguch 	/* assign DMA memory */
   1187  1.101  yamaguch 	memset(vaddr, 0, dmamemsize);
   1188  1.101  yamaguch 	sc->sc_dmamem = vaddr;
   1189  1.101  yamaguch 	p = (intptr_t) vaddr;
   1190   1.47  yamaguch 
   1191  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1192  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1193  1.101  yamaguch 		maps = netq->netq_maps;
   1194  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1195   1.46  yamaguch 
   1196  1.101  yamaguch 		netq->netq_maps_kva = vioif_assign_mem(&p,
   1197  1.101  yamaguch 		    sizeof(*maps[0].vnm_hdr) * vq_num);
   1198  1.101  yamaguch 	}
   1199   1.46  yamaguch 
   1200  1.101  yamaguch 	if (sc->sc_has_ctrl) {
   1201  1.101  yamaguch 		ctrlq->ctrlq_cmd = vioif_assign_mem(&p,
   1202  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_cmd));
   1203  1.101  yamaguch 		ctrlq->ctrlq_status = vioif_assign_mem(&p,
   1204  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_status));
   1205  1.101  yamaguch 		ctrlq->ctrlq_rx = vioif_assign_mem(&p,
   1206  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_rx));
   1207  1.101  yamaguch 		ctrlq->ctrlq_mac_tbl_uc = vioif_assign_mem(&p,
   1208  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc)
   1209  1.101  yamaguch 		    + ETHER_ADDR_LEN);
   1210  1.101  yamaguch 		ctrlq->ctrlq_mac_tbl_mc = vioif_assign_mem(&p,
   1211  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
   1212  1.101  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES);
   1213  1.101  yamaguch 		ctrlq->ctrlq_mac_addr = vioif_assign_mem(&p,
   1214  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_addr));
   1215  1.101  yamaguch 		ctrlq->ctrlq_mq = vioif_assign_mem(&p, sizeof(*ctrlq->ctrlq_mq));
   1216   1.76  yamaguch 	}
   1217   1.33     ozaki 
   1218  1.101  yamaguch 	/* allocate kmem */
   1219  1.101  yamaguch 	kmemsize = 0;
   1220   1.85  yamaguch 
   1221  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1222  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1223  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1224   1.55  yamaguch 
   1225  1.101  yamaguch 		kmemsize += sizeof(netq->netq_maps[0]) * vq_num;
   1226   1.85  yamaguch 	}
   1227   1.85  yamaguch 
   1228  1.101  yamaguch 	vaddr = kmem_zalloc(kmemsize, KM_SLEEP);
   1229  1.101  yamaguch 	sc->sc_kmem = vaddr;
   1230   1.98  yamaguch 
   1231  1.101  yamaguch 	/* assign allocated kmem */
   1232  1.101  yamaguch 	p = (intptr_t) vaddr;
   1233   1.46  yamaguch 
   1234  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1235  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1236  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1237   1.85  yamaguch 
   1238  1.101  yamaguch 		netq->netq_maps = vioif_assign_mem(&p,
   1239  1.101  yamaguch 		    sizeof(netq->netq_maps[0]) * vq_num);
   1240   1.85  yamaguch 	}
   1241    1.1   hannken 
   1242  1.101  yamaguch 	/* prepare dmamaps */
   1243  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1244  1.101  yamaguch 		static const struct {
   1245  1.101  yamaguch 			const char	*msg_hdr;
   1246  1.101  yamaguch 			const char	*msg_payload;
   1247  1.101  yamaguch 			int		 dma_flag;
   1248  1.101  yamaguch 			bus_size_t	 dma_size;
   1249  1.101  yamaguch 			int		 dma_nsegs;
   1250  1.101  yamaguch 		} dmaparams[VIOIF_NETQ_IDX] = {
   1251  1.101  yamaguch 			[VIOIF_NETQ_RX] = {
   1252  1.101  yamaguch 				.msg_hdr	= "rx header",
   1253  1.101  yamaguch 				.msg_payload	= "rx payload",
   1254  1.101  yamaguch 				.dma_flag	= BUS_DMA_READ,
   1255  1.101  yamaguch 				.dma_size	= MCLBYTES - ETHER_ALIGN,
   1256  1.101  yamaguch 				.dma_nsegs	= 1,
   1257  1.101  yamaguch 			},
   1258  1.101  yamaguch 			[VIOIF_NETQ_TX] = {
   1259  1.101  yamaguch 				.msg_hdr	= "tx header",
   1260  1.101  yamaguch 				.msg_payload	= "tx payload",
   1261  1.101  yamaguch 				.dma_flag	= BUS_DMA_WRITE,
   1262  1.101  yamaguch 				.dma_size	= ETHER_MAX_LEN,
   1263  1.101  yamaguch 				.dma_nsegs	= VIRTIO_NET_TX_MAXNSEGS,
   1264  1.101  yamaguch 			}
   1265  1.101  yamaguch 		};
   1266    1.1   hannken 
   1267  1.101  yamaguch 		struct virtio_net_hdr *hdrs;
   1268  1.101  yamaguch 		int dir;
   1269   1.46  yamaguch 
   1270  1.101  yamaguch 		dir = VIOIF_NETQ_DIR(qid);
   1271  1.101  yamaguch 		netq = &sc->sc_netqs[qid];
   1272  1.101  yamaguch 		vq_num = netq->netq_vq->vq_num;
   1273  1.101  yamaguch 		maps = netq->netq_maps;
   1274  1.101  yamaguch 		hdrs = netq->netq_maps_kva;
   1275    1.7     ozaki 
   1276  1.101  yamaguch 		for (i = 0; i < vq_num; i++) {
   1277  1.101  yamaguch 			maps[i].vnm_hdr = &hdrs[i];
   1278  1.101  yamaguch 
   1279  1.101  yamaguch 			r = vioif_dmamap_create_load(sc, &maps[i].vnm_hdr_map,
   1280  1.101  yamaguch 			    maps[i].vnm_hdr, sc->sc_hdr_size, 1,
   1281  1.101  yamaguch 			    dmaparams[dir].dma_flag, dmaparams[dir].msg_hdr);
   1282  1.101  yamaguch 			if (r != 0)
   1283  1.101  yamaguch 				goto err_reqs;
   1284   1.98  yamaguch 
   1285  1.101  yamaguch 			r = vioif_dmamap_create(sc, &maps[i].vnm_mbuf_map,
   1286  1.101  yamaguch 			    dmaparams[dir].dma_size, dmaparams[dir].dma_nsegs,
   1287  1.101  yamaguch 			    dmaparams[dir].msg_payload);
   1288  1.101  yamaguch 			if (r != 0)
   1289  1.101  yamaguch 				goto err_reqs;
   1290  1.101  yamaguch 		}
   1291  1.101  yamaguch 	}
   1292    1.7     ozaki 
   1293  1.101  yamaguch 	if (sc->sc_has_ctrl) {
   1294  1.101  yamaguch 		/* control vq class & command */
   1295  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_cmd_dmamap,
   1296  1.101  yamaguch 		    ctrlq->ctrlq_cmd, sizeof(*ctrlq->ctrlq_cmd), 1,
   1297  1.101  yamaguch 		    BUS_DMA_WRITE, "control command");
   1298  1.101  yamaguch 		if (r != 0)
   1299  1.101  yamaguch 			goto err_reqs;
   1300    1.1   hannken 
   1301  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_status_dmamap,
   1302  1.101  yamaguch 		    ctrlq->ctrlq_status, sizeof(*ctrlq->ctrlq_status), 1,
   1303  1.101  yamaguch 		    BUS_DMA_READ, "control status");
   1304  1.101  yamaguch 		if (r != 0)
   1305  1.101  yamaguch 			goto err_reqs;
   1306    1.1   hannken 
   1307  1.101  yamaguch 		/* control vq rx mode command parameter */
   1308  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_rx_dmamap,
   1309  1.101  yamaguch 		    ctrlq->ctrlq_rx, sizeof(*ctrlq->ctrlq_rx), 1,
   1310  1.101  yamaguch 		    BUS_DMA_WRITE, "rx mode control command");
   1311  1.101  yamaguch 		if (r != 0)
   1312  1.101  yamaguch 			goto err_reqs;
   1313   1.46  yamaguch 
   1314  1.101  yamaguch 		/* multiqueue set command */
   1315  1.101  yamaguch 		r = vioif_dmamap_create_load(sc, &ctrlq->ctrlq_mq_dmamap,
   1316  1.101  yamaguch 		    ctrlq->ctrlq_mq, sizeof(*ctrlq->ctrlq_mq), 1,
   1317  1.101  yamaguch 		    BUS_DMA_WRITE, "multiqueue set command");
   1318  1.101  yamaguch 		if (r != 0)
   1319  1.101  yamaguch 			goto err_reqs;
   1320   1.36  jdolecek 
   1321  1.101  yamaguch 		/* control vq MAC filter table for unicast */
   1322  1.101  yamaguch 		/* do not load now since its length is variable */
   1323  1.101  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_uc_dmamap,
   1324  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_uc)
   1325  1.101  yamaguch 		    + ETHER_ADDR_LEN, 1,
   1326  1.101  yamaguch 		    "unicast MAC address filter command");
   1327  1.101  yamaguch 		if (r != 0)
   1328  1.101  yamaguch 			goto err_reqs;
   1329   1.95  yamaguch 
   1330  1.101  yamaguch 		/* control vq MAC filter table for multicast */
   1331  1.101  yamaguch 		r = vioif_dmamap_create(sc, &ctrlq->ctrlq_tbl_mc_dmamap,
   1332  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_tbl_mc)
   1333  1.101  yamaguch 		    + ETHER_ADDR_LEN * VIRTIO_NET_CTRL_MAC_MAXENTRIES, 1,
   1334  1.101  yamaguch 		    "multicast MAC address filter command");
   1335  1.101  yamaguch 		if (r != 0)
   1336  1.101  yamaguch 			goto err_reqs;
   1337   1.36  jdolecek 
   1338  1.101  yamaguch 		/* control vq MAC address set command */
   1339  1.101  yamaguch 		r = vioif_dmamap_create_load(sc,
   1340  1.101  yamaguch 		    &ctrlq->ctrlq_mac_addr_dmamap,
   1341  1.101  yamaguch 		    ctrlq->ctrlq_mac_addr,
   1342  1.101  yamaguch 		    sizeof(*ctrlq->ctrlq_mac_addr), 1,
   1343  1.101  yamaguch 		    BUS_DMA_WRITE, "mac addr set command");
   1344  1.101  yamaguch 		if (r != 0)
   1345  1.101  yamaguch 			goto err_reqs;
   1346  1.101  yamaguch 	}
   1347   1.36  jdolecek 
   1348  1.101  yamaguch 	return 0;
   1349   1.36  jdolecek 
   1350  1.101  yamaguch err_reqs:
   1351  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_mc_dmamap);
   1352  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_tbl_uc_dmamap);
   1353  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_rx_dmamap);
   1354  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_status_dmamap);
   1355  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_cmd_dmamap);
   1356  1.101  yamaguch 	vioif_dmamap_destroy(sc, &ctrlq->ctrlq_mac_addr_dmamap);
   1357  1.101  yamaguch 	for (qid = 0; qid < netq_num; qid++) {
   1358  1.101  yamaguch 		vq_num = sc->sc_netqs[qid].netq_vq->vq_num;
   1359  1.101  yamaguch 		maps = sc->sc_netqs[qid].netq_maps;
   1360   1.99  yamaguch 
   1361  1.101  yamaguch 		for (i = 0; i < vq_num; i++) {
   1362  1.101  yamaguch 			vioif_dmamap_destroy(sc, &maps[i].vnm_mbuf_map);
   1363  1.101  yamaguch 			vioif_dmamap_destroy(sc, &maps[i].vnm_hdr_map);
   1364    1.1   hannken 		}
   1365    1.1   hannken 	}
   1366  1.101  yamaguch 	if (sc->sc_kmem) {
   1367  1.101  yamaguch 		kmem_free(sc->sc_kmem, kmemsize);
   1368  1.101  yamaguch 		sc->sc_kmem = NULL;
   1369    1.1   hannken 	}
   1370  1.101  yamaguch 	bus_dmamem_unmap(virtio_dmat(vsc), sc->sc_dmamem, dmamemsize);
   1371  1.101  yamaguch err_dmamem_alloc:
   1372  1.101  yamaguch 	bus_dmamem_free(virtio_dmat(vsc), &sc->sc_segs[0], 1);
   1373  1.101  yamaguch err_none:
   1374  1.101  yamaguch 	return -1;
   1375   1.46  yamaguch }
   1376   1.46  yamaguch 
   1377   1.46  yamaguch static void
   1378  1.101  yamaguch vioif_alloc_queues(struct vioif_softc *sc)
   1379   1.46  yamaguch {
   1380  1.101  yamaguch 	int nvq_pairs = sc->sc_max_nvq_pairs;
   1381  1.101  yamaguch 	size_t nvqs, netq_num;
   1382  1.101  yamaguch 
   1383  1.101  yamaguch 	KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
   1384   1.46  yamaguch 
   1385  1.101  yamaguch 	nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
   1386  1.101  yamaguch 	if (sc->sc_has_ctrl)
   1387  1.101  yamaguch 		nvqs++;
   1388   1.46  yamaguch 
   1389  1.101  yamaguch 	sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_SLEEP);
   1390  1.101  yamaguch 	sc->sc_netqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * netq_num,
   1391  1.101  yamaguch 	    KM_SLEEP);
   1392   1.46  yamaguch }
   1393   1.46  yamaguch 
   1394   1.46  yamaguch static void
   1395  1.101  yamaguch vioif_free_queues(struct vioif_softc *sc)
   1396   1.46  yamaguch {
   1397  1.101  yamaguch 	size_t nvqs, netq_num;
   1398   1.46  yamaguch 
   1399  1.101  yamaguch 	nvqs = netq_num = sc->sc_max_nvq_pairs * 2;
   1400  1.101  yamaguch 	if (sc->sc_ctrlq.ctrlq_vq)
   1401  1.101  yamaguch 		nvqs++;
   1402   1.46  yamaguch 
   1403  1.101  yamaguch 	kmem_free(sc->sc_netqs, sizeof(sc->sc_netqs[0]) * netq_num);
   1404  1.101  yamaguch 	kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
   1405  1.101  yamaguch 	sc->sc_netqs = NULL;
   1406  1.101  yamaguch 	sc->sc_vqs = NULL;
   1407   1.46  yamaguch }
   1408   1.46  yamaguch 
   1409  1.101  yamaguch /*
   1410  1.101  yamaguch  * Network queues
   1411  1.101  yamaguch  */
   1412  1.101  yamaguch static int
   1413  1.101  yamaguch vioif_netqueue_init(struct vioif_softc *sc, struct virtio_softc *vsc,
   1414  1.101  yamaguch     size_t qid, u_int softint_flags)
   1415   1.46  yamaguch {
   1416  1.101  yamaguch 	static const struct {
   1417  1.101  yamaguch 		const char	*dirname;
   1418  1.101  yamaguch 		int		 segsize;
   1419  1.101  yamaguch 		int		 nsegs;
   1420  1.101  yamaguch 		int 		(*intrhand)(void *);
   1421  1.101  yamaguch 		void		(*sihand)(void *);
   1422  1.101  yamaguch 	} params[VIOIF_NETQ_IDX] = {
   1423  1.101  yamaguch 		[VIOIF_NETQ_RX] = {
   1424  1.101  yamaguch 			.dirname	= "rx",
   1425  1.101  yamaguch 			.segsize	= MCLBYTES,
   1426  1.101  yamaguch 			.nsegs		= 2,
   1427  1.101  yamaguch 			.intrhand	= vioif_rx_intr,
   1428  1.101  yamaguch 			.sihand		= vioif_rx_handle,
   1429  1.101  yamaguch 		},
   1430  1.101  yamaguch 		[VIOIF_NETQ_TX] = {
   1431  1.101  yamaguch 			.dirname	= "tx",
   1432  1.101  yamaguch 			.segsize	= ETHER_MAX_LEN - ETHER_HDR_LEN,
   1433  1.101  yamaguch 			.nsegs		= 2,
   1434  1.101  yamaguch 			.intrhand	= vioif_tx_intr,
   1435  1.101  yamaguch 			.sihand		= vioif_tx_handle,
   1436  1.101  yamaguch 		}
   1437  1.101  yamaguch 	};
   1438   1.46  yamaguch 
   1439  1.101  yamaguch 	struct virtqueue *vq;
   1440  1.101  yamaguch 	struct vioif_netqueue *netq;
   1441  1.101  yamaguch 	struct vioif_tx_context *txc;
   1442  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1443  1.101  yamaguch 	char qname[32];
   1444  1.101  yamaguch 	int r, dir;
   1445   1.46  yamaguch 
   1446  1.101  yamaguch 	txc = NULL;
   1447  1.101  yamaguch 	rxc = NULL;
   1448  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
   1449  1.101  yamaguch 	vq = &sc->sc_vqs[qid];
   1450  1.101  yamaguch 	dir = VIOIF_NETQ_DIR(qid);
   1451   1.46  yamaguch 
   1452  1.101  yamaguch 	netq->netq_vq = &sc->sc_vqs[qid];
   1453  1.101  yamaguch 	netq->netq_stopping = false;
   1454  1.101  yamaguch 	netq->netq_running_handle = false;
   1455   1.46  yamaguch 
   1456  1.101  yamaguch 	snprintf(qname, sizeof(qname), "%s%zu",
   1457  1.101  yamaguch 	    params[dir].dirname, VIOIF_NETQ_PAIRIDX(qid));
   1458  1.101  yamaguch 	snprintf(netq->netq_evgroup, sizeof(netq->netq_evgroup),
   1459  1.101  yamaguch 	    "%s-%s", device_xname(sc->sc_dev), qname);
   1460   1.46  yamaguch 
   1461  1.101  yamaguch 	mutex_init(&netq->netq_lock, MUTEX_DEFAULT, IPL_NET);
   1462  1.101  yamaguch 	r = virtio_alloc_vq(vsc, vq, qid,
   1463  1.101  yamaguch 	    params[dir].segsize + sc->sc_hdr_size,
   1464  1.101  yamaguch 	    params[dir].nsegs, qname);
   1465  1.101  yamaguch 	if (r != 0)
   1466  1.101  yamaguch 		goto err;
   1467  1.101  yamaguch 	netq->netq_vq = vq;
   1468   1.46  yamaguch 
   1469  1.101  yamaguch 	netq->netq_vq->vq_intrhand = params[dir].intrhand;
   1470  1.101  yamaguch 	netq->netq_vq->vq_intrhand_arg = netq;
   1471  1.101  yamaguch 	netq->netq_softint = softint_establish(softint_flags,
   1472  1.101  yamaguch 	    params[dir].sihand, netq);
   1473  1.101  yamaguch 	if (netq->netq_softint == NULL) {
   1474  1.101  yamaguch 		aprint_error_dev(sc->sc_dev,
   1475  1.101  yamaguch 		    "couldn't establish %s softint\n",
   1476  1.101  yamaguch 		    params[dir].dirname);
   1477  1.101  yamaguch 		goto err;
   1478   1.46  yamaguch 	}
   1479  1.101  yamaguch 	vioif_work_set(&netq->netq_work, params[dir].sihand, netq);
   1480   1.46  yamaguch 
   1481  1.101  yamaguch 	switch (dir) {
   1482  1.101  yamaguch 	case VIOIF_NETQ_RX:
   1483  1.101  yamaguch 		rxc = kmem_zalloc(sizeof(*rxc), KM_SLEEP);
   1484  1.101  yamaguch 		netq->netq_ctx = rxc;
   1485  1.101  yamaguch 		/* nothing to do */
   1486  1.101  yamaguch 		break;
   1487  1.101  yamaguch 	case VIOIF_NETQ_TX:
   1488  1.101  yamaguch 		txc = kmem_zalloc(sizeof(*txc), KM_SLEEP);
   1489  1.101  yamaguch 		netq->netq_ctx = (void *)txc;
   1490  1.101  yamaguch 		txc->txc_deferred_transmit = softint_establish(softint_flags,
   1491  1.101  yamaguch 		    vioif_deferred_transmit, netq);
   1492  1.101  yamaguch 		if (txc->txc_deferred_transmit == NULL) {
   1493  1.101  yamaguch 			aprint_error_dev(sc->sc_dev,
   1494  1.101  yamaguch 			    "couldn't establish softint for "
   1495  1.101  yamaguch 			    "tx deferred transmit\n");
   1496  1.101  yamaguch 			goto err;
   1497  1.101  yamaguch 		}
   1498  1.101  yamaguch 		txc->txc_link_active = VIOIF_IS_LINK_ACTIVE(sc);
   1499  1.101  yamaguch 		txc->txc_intrq = pcq_create(vq->vq_num, KM_SLEEP);
   1500  1.101  yamaguch 		break;
   1501   1.46  yamaguch 	}
   1502   1.46  yamaguch 
   1503   1.46  yamaguch 	return 0;
   1504   1.46  yamaguch 
   1505  1.101  yamaguch err:
   1506  1.101  yamaguch 	netq->netq_ctx = NULL;
   1507    1.7     ozaki 
   1508  1.101  yamaguch 	if (rxc != NULL) {
   1509  1.101  yamaguch 		kmem_free(rxc, sizeof(*rxc));
   1510  1.101  yamaguch 	}
   1511    1.1   hannken 
   1512  1.101  yamaguch 	if (txc != NULL) {
   1513  1.101  yamaguch 		if (txc->txc_deferred_transmit != NULL)
   1514  1.101  yamaguch 			softint_disestablish(txc->txc_deferred_transmit);
   1515  1.101  yamaguch 		if (txc->txc_intrq != NULL)
   1516  1.101  yamaguch 			pcq_destroy(txc->txc_intrq);
   1517  1.101  yamaguch 		kmem_free(txc, sizeof(txc));
   1518  1.101  yamaguch 	}
   1519    1.1   hannken 
   1520  1.101  yamaguch 	vioif_work_set(&netq->netq_work, NULL, NULL);
   1521  1.101  yamaguch 	if (netq->netq_softint != NULL) {
   1522  1.101  yamaguch 		softint_disestablish(netq->netq_softint);
   1523  1.101  yamaguch 		netq->netq_softint = NULL;
   1524    1.1   hannken 	}
   1525  1.101  yamaguch 	netq->netq_vq->vq_intrhand = NULL;
   1526  1.101  yamaguch 	netq->netq_vq->vq_intrhand_arg = NULL;
   1527    1.1   hannken 
   1528  1.101  yamaguch 	virtio_free_vq(vsc, vq);
   1529  1.101  yamaguch 	mutex_destroy(&netq->netq_lock);
   1530  1.101  yamaguch 	netq->netq_vq = NULL;
   1531    1.1   hannken 
   1532  1.101  yamaguch 	return -1;
   1533    1.1   hannken }
   1534    1.1   hannken 
   1535  1.101  yamaguch static void
   1536  1.101  yamaguch vioif_netqueue_teardown(struct vioif_softc *sc, struct virtio_softc *vsc,
   1537  1.101  yamaguch     size_t qid)
   1538    1.1   hannken {
   1539   1.98  yamaguch 	struct vioif_netqueue *netq;
   1540  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1541  1.101  yamaguch 	struct vioif_tx_context *txc;
   1542  1.101  yamaguch 	int dir;
   1543  1.101  yamaguch 
   1544  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
   1545  1.101  yamaguch 
   1546  1.101  yamaguch 	if (netq->netq_vq == NULL)
   1547  1.101  yamaguch 		return;
   1548  1.101  yamaguch 
   1549  1.101  yamaguch 	netq = &sc->sc_netqs[qid];
   1550  1.101  yamaguch 	dir = VIOIF_NETQ_DIR(qid);
   1551  1.101  yamaguch 	switch (dir) {
   1552  1.101  yamaguch 	case VIOIF_NETQ_RX:
   1553  1.101  yamaguch 		rxc = netq->netq_ctx;
   1554  1.101  yamaguch 		netq->netq_ctx = NULL;
   1555  1.101  yamaguch 		kmem_free(rxc, sizeof(*rxc));
   1556  1.101  yamaguch 		break;
   1557  1.101  yamaguch 	case VIOIF_NETQ_TX:
   1558  1.101  yamaguch 		txc = netq->netq_ctx;
   1559  1.101  yamaguch 		netq->netq_ctx = NULL;
   1560  1.101  yamaguch 		softint_disestablish(txc->txc_deferred_transmit);
   1561  1.101  yamaguch 		pcq_destroy(txc->txc_intrq);
   1562  1.101  yamaguch 		kmem_free(txc, sizeof(*txc));
   1563  1.101  yamaguch 		break;
   1564  1.101  yamaguch 	}
   1565    1.1   hannken 
   1566  1.101  yamaguch 	softint_disestablish(netq->netq_softint);
   1567  1.101  yamaguch 	virtio_free_vq(vsc, netq->netq_vq);
   1568  1.101  yamaguch 	mutex_destroy(&netq->netq_lock);
   1569  1.101  yamaguch 	netq->netq_vq = NULL;
   1570    1.1   hannken }
   1571    1.1   hannken 
   1572   1.98  yamaguch static void
   1573   1.98  yamaguch vioif_net_sched_handle(struct vioif_softc *sc, struct vioif_netqueue *netq)
   1574   1.98  yamaguch {
   1575   1.98  yamaguch 
   1576   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1577   1.98  yamaguch 	KASSERT(!netq->netq_stopping);
   1578   1.98  yamaguch 
   1579   1.98  yamaguch 	if (netq->netq_workqueue) {
   1580   1.98  yamaguch 		vioif_work_add(sc->sc_txrx_workqueue, &netq->netq_work);
   1581   1.98  yamaguch 	} else {
   1582   1.98  yamaguch 		softint_schedule(netq->netq_softint);
   1583   1.98  yamaguch 	}
   1584   1.98  yamaguch }
   1585   1.98  yamaguch 
   1586   1.99  yamaguch static int
   1587   1.99  yamaguch vioif_net_load_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map,
   1588   1.99  yamaguch    struct mbuf *m, int dma_flags)
   1589   1.99  yamaguch {
   1590   1.99  yamaguch 	int r;
   1591   1.99  yamaguch 
   1592   1.99  yamaguch 	KASSERT(map->vnm_mbuf == NULL);
   1593   1.99  yamaguch 
   1594   1.99  yamaguch 	r = bus_dmamap_load_mbuf(virtio_dmat(vsc),
   1595   1.99  yamaguch 	    map->vnm_mbuf_map, m, dma_flags | BUS_DMA_NOWAIT);
   1596   1.99  yamaguch 	if (r == 0) {
   1597   1.99  yamaguch 		map->vnm_mbuf = m;
   1598   1.99  yamaguch 	}
   1599   1.99  yamaguch 
   1600   1.99  yamaguch 	return r;
   1601   1.99  yamaguch }
   1602   1.99  yamaguch 
   1603   1.99  yamaguch static void
   1604   1.99  yamaguch vioif_net_unload_mbuf(struct virtio_softc *vsc, struct vioif_net_map *map)
   1605   1.99  yamaguch {
   1606   1.99  yamaguch 
   1607   1.99  yamaguch 	KASSERT(map->vnm_mbuf != NULL);
   1608   1.99  yamaguch 	bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
   1609   1.99  yamaguch 	map->vnm_mbuf = NULL;
   1610   1.99  yamaguch }
   1611   1.99  yamaguch 
   1612   1.99  yamaguch static int
   1613   1.99  yamaguch vioif_net_enqueue(struct virtio_softc *vsc, struct virtqueue *vq,
   1614   1.99  yamaguch     int slot, struct vioif_net_map *map, int dma_ops, bool is_write)
   1615   1.99  yamaguch {
   1616   1.99  yamaguch 	int r;
   1617   1.99  yamaguch 
   1618   1.99  yamaguch 	KASSERT(map->vnm_mbuf != NULL);
   1619   1.99  yamaguch 
   1620   1.99  yamaguch 	/* This should actually never fail */
   1621   1.99  yamaguch 	r = virtio_enqueue_reserve(vsc, vq, slot,
   1622   1.99  yamaguch 	    map->vnm_mbuf_map->dm_nsegs + 1);
   1623   1.99  yamaguch 	if (r != 0) {
   1624   1.99  yamaguch 		/* slot already freed by virtio_enqueue_reserve */
   1625   1.99  yamaguch 		return r;
   1626   1.99  yamaguch 	}
   1627   1.99  yamaguch 
   1628   1.99  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
   1629   1.99  yamaguch 	    0, map->vnm_mbuf_map->dm_mapsize, dma_ops);
   1630   1.99  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
   1631   1.99  yamaguch 	    0, map->vnm_hdr_map->dm_mapsize, dma_ops);
   1632   1.99  yamaguch 
   1633   1.99  yamaguch 	virtio_enqueue(vsc, vq, slot, map->vnm_hdr_map, is_write);
   1634   1.99  yamaguch 	virtio_enqueue(vsc, vq, slot, map->vnm_mbuf_map, is_write);
   1635   1.99  yamaguch 	virtio_enqueue_commit(vsc, vq, slot, false);
   1636   1.99  yamaguch 
   1637   1.99  yamaguch 	return 0;
   1638   1.99  yamaguch }
   1639   1.99  yamaguch 
   1640   1.99  yamaguch static int
   1641   1.99  yamaguch vioif_net_enqueue_tx(struct virtio_softc *vsc, struct virtqueue *vq,
   1642   1.99  yamaguch     int slot, struct vioif_net_map *map)
   1643   1.99  yamaguch {
   1644   1.99  yamaguch 
   1645   1.99  yamaguch 	return vioif_net_enqueue(vsc, vq, slot, map,
   1646   1.99  yamaguch 	    BUS_DMASYNC_PREWRITE, true);
   1647   1.99  yamaguch }
   1648   1.99  yamaguch 
   1649   1.99  yamaguch static int
   1650   1.99  yamaguch vioif_net_enqueue_rx(struct virtio_softc *vsc, struct virtqueue *vq,
   1651   1.99  yamaguch     int slot, struct vioif_net_map *map)
   1652   1.99  yamaguch {
   1653   1.99  yamaguch 
   1654   1.99  yamaguch 	return vioif_net_enqueue(vsc, vq, slot, map,
   1655   1.99  yamaguch 	    BUS_DMASYNC_PREREAD, false);
   1656   1.99  yamaguch }
   1657   1.99  yamaguch 
   1658  1.101  yamaguch static struct mbuf *
   1659  1.101  yamaguch vioif_net_dequeue_commit(struct virtio_softc *vsc, struct virtqueue *vq,
   1660  1.101  yamaguch    int slot, struct vioif_net_map *map, int dma_flags)
   1661  1.101  yamaguch {
   1662  1.101  yamaguch 	struct mbuf *m;
   1663  1.101  yamaguch 
   1664  1.101  yamaguch 	m = map->vnm_mbuf;
   1665  1.101  yamaguch 	KASSERT(m != NULL);
   1666  1.101  yamaguch 	map->vnm_mbuf = NULL;
   1667  1.101  yamaguch 
   1668  1.101  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_hdr_map,
   1669  1.101  yamaguch 	    0, map->vnm_hdr_map->dm_mapsize, dma_flags);
   1670  1.101  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), map->vnm_mbuf_map,
   1671  1.101  yamaguch 	    0, map->vnm_mbuf_map->dm_mapsize, dma_flags);
   1672  1.101  yamaguch 
   1673  1.101  yamaguch 	bus_dmamap_unload(virtio_dmat(vsc), map->vnm_mbuf_map);
   1674  1.101  yamaguch 	virtio_dequeue_commit(vsc, vq, slot);
   1675  1.101  yamaguch 
   1676  1.101  yamaguch 	return m;
   1677  1.101  yamaguch }
   1678  1.101  yamaguch 
   1679  1.101  yamaguch static void
   1680  1.101  yamaguch vioif_net_intr_enable(struct vioif_softc *sc, struct virtio_softc *vsc)
   1681  1.101  yamaguch {
   1682  1.101  yamaguch 	struct vioif_netqueue *netq;
   1683  1.101  yamaguch 	size_t i, act_qnum;
   1684  1.101  yamaguch 	int enqueued;
   1685  1.101  yamaguch 
   1686  1.101  yamaguch 	act_qnum = sc->sc_act_nvq_pairs * 2;
   1687  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
   1688  1.101  yamaguch 		netq = &sc->sc_netqs[i];
   1689  1.101  yamaguch 
   1690  1.101  yamaguch 		KASSERT(!netq->netq_stopping);
   1691  1.101  yamaguch 		KASSERT(!netq->netq_running_handle);
   1692  1.101  yamaguch 
   1693  1.101  yamaguch 		enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
   1694  1.101  yamaguch 		if (enqueued != 0) {
   1695  1.101  yamaguch 			virtio_stop_vq_intr(vsc, netq->netq_vq);
   1696  1.101  yamaguch 
   1697  1.101  yamaguch 			mutex_enter(&netq->netq_lock);
   1698  1.101  yamaguch 			netq->netq_running_handle = true;
   1699  1.101  yamaguch 			vioif_net_sched_handle(sc, netq);
   1700  1.101  yamaguch 			mutex_exit(&netq->netq_lock);
   1701  1.101  yamaguch 		}
   1702  1.101  yamaguch 	}
   1703  1.101  yamaguch }
   1704  1.101  yamaguch 
   1705  1.101  yamaguch static void
   1706  1.101  yamaguch vioif_net_intr_disable(struct vioif_softc *sc, struct virtio_softc *vsc)
   1707  1.101  yamaguch {
   1708  1.101  yamaguch 	struct vioif_netqueue *netq;
   1709  1.101  yamaguch 	size_t i, act_qnum;
   1710  1.101  yamaguch 
   1711  1.101  yamaguch 	act_qnum = sc->sc_act_nvq_pairs * 2;
   1712  1.101  yamaguch 	for (i = 0; i < act_qnum; i++) {
   1713  1.101  yamaguch 		netq = &sc->sc_netqs[i];
   1714  1.101  yamaguch 
   1715  1.101  yamaguch 		virtio_stop_vq_intr(vsc, netq->netq_vq);
   1716  1.101  yamaguch 	}
   1717  1.101  yamaguch }
   1718  1.101  yamaguch 
   1719  1.101  yamaguch /*
   1720  1.101  yamaguch  * Receive implementation
   1721  1.101  yamaguch  */
   1722  1.101  yamaguch /* enqueue mbufs to receive slots */
   1723  1.101  yamaguch static void
   1724  1.101  yamaguch vioif_populate_rx_mbufs_locked(struct vioif_softc *sc, struct vioif_netqueue *netq)
   1725  1.101  yamaguch {
   1726  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1727  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1728  1.101  yamaguch 	struct vioif_rx_context *rxc;
   1729  1.101  yamaguch 	struct vioif_net_map *map;
   1730  1.101  yamaguch 	struct mbuf *m;
   1731  1.101  yamaguch 	int i, r, ndone = 0;
   1732  1.101  yamaguch 
   1733  1.101  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1734  1.101  yamaguch 
   1735  1.101  yamaguch 	rxc = netq->netq_ctx;
   1736  1.101  yamaguch 
   1737  1.101  yamaguch 	for (i = 0; i < vq->vq_num; i++) {
   1738  1.101  yamaguch 		int slot;
   1739  1.101  yamaguch 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1740  1.101  yamaguch 		if (r == EAGAIN)
   1741  1.101  yamaguch 			break;
   1742  1.101  yamaguch 		if (__predict_false(r != 0))
   1743  1.101  yamaguch 			panic("enqueue_prep for rx buffers");
   1744  1.101  yamaguch 
   1745  1.101  yamaguch 		MGETHDR(m, M_DONTWAIT, MT_DATA);
   1746  1.101  yamaguch 		if (m == NULL) {
   1747  1.101  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   1748  1.101  yamaguch 			rxc->rxc_mbuf_enobufs.ev_count++;
   1749  1.101  yamaguch 			break;
   1750  1.101  yamaguch 		}
   1751  1.101  yamaguch 		MCLGET(m, M_DONTWAIT);
   1752  1.101  yamaguch 		if ((m->m_flags & M_EXT) == 0) {
   1753  1.101  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   1754  1.101  yamaguch 			m_freem(m);
   1755  1.101  yamaguch 			rxc->rxc_mbuf_enobufs.ev_count++;
   1756  1.101  yamaguch 			break;
   1757  1.101  yamaguch 		}
   1758  1.101  yamaguch 
   1759  1.101  yamaguch 		m->m_len = m->m_pkthdr.len = MCLBYTES;
   1760  1.101  yamaguch 		m_adj(m, ETHER_ALIGN);
   1761  1.101  yamaguch 
   1762  1.101  yamaguch 		map = &netq->netq_maps[slot];
   1763  1.101  yamaguch 		r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_READ);
   1764  1.101  yamaguch 		if (r != 0) {
   1765  1.101  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   1766  1.101  yamaguch 			m_freem(m);
   1767  1.101  yamaguch 			netq->netq_mbuf_load_failed.ev_count++;
   1768  1.101  yamaguch 			break;
   1769  1.101  yamaguch 		}
   1770  1.101  yamaguch 
   1771  1.101  yamaguch 		r = vioif_net_enqueue_rx(vsc, vq, slot, map);
   1772  1.101  yamaguch 		if (r != 0) {
   1773  1.101  yamaguch 			vioif_net_unload_mbuf(vsc, map);
   1774  1.101  yamaguch 			netq->netq_enqueue_failed.ev_count++;
   1775  1.101  yamaguch 			m_freem(m);
   1776  1.101  yamaguch 			/* slot already freed by vioif_net_enqueue_rx */
   1777  1.101  yamaguch 			break;
   1778  1.101  yamaguch 		}
   1779  1.101  yamaguch 
   1780  1.101  yamaguch 		ndone++;
   1781  1.101  yamaguch 	}
   1782  1.101  yamaguch 
   1783  1.101  yamaguch 	if (ndone > 0)
   1784  1.101  yamaguch 		vioif_notify(vsc, vq);
   1785  1.101  yamaguch }
   1786  1.101  yamaguch 
   1787  1.101  yamaguch /* dequeue received packets */
   1788  1.101  yamaguch static bool
   1789  1.101  yamaguch vioif_rx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   1790  1.101  yamaguch     struct vioif_netqueue *netq, u_int limit, size_t *ndeqp)
   1791  1.101  yamaguch {
   1792  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1793  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   1794  1.101  yamaguch 	struct vioif_net_map *map;
   1795  1.101  yamaguch 	struct mbuf *m;
   1796  1.101  yamaguch 	int slot, len;
   1797  1.101  yamaguch 	bool more;
   1798  1.101  yamaguch 	size_t ndeq;
   1799  1.101  yamaguch 
   1800  1.101  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1801  1.101  yamaguch 
   1802  1.101  yamaguch 	more = false;
   1803  1.101  yamaguch 	ndeq = 0;
   1804  1.101  yamaguch 
   1805  1.101  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   1806  1.101  yamaguch 		goto done;
   1807  1.101  yamaguch 
   1808  1.101  yamaguch 	for (;;ndeq++) {
   1809  1.101  yamaguch 		if (ndeq >= limit) {
   1810  1.101  yamaguch 			more = true;
   1811  1.101  yamaguch 			break;
   1812  1.101  yamaguch 		}
   1813  1.101  yamaguch 
   1814  1.101  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   1815  1.101  yamaguch 			break;
   1816  1.101  yamaguch 
   1817  1.101  yamaguch 		map = &netq->netq_maps[slot];
   1818  1.101  yamaguch 		KASSERT(map->vnm_mbuf != NULL);
   1819  1.101  yamaguch 		m = vioif_net_dequeue_commit(vsc, vq, slot,
   1820  1.101  yamaguch 		    map, BUS_DMASYNC_POSTREAD);
   1821  1.101  yamaguch 		KASSERT(m != NULL);
   1822  1.101  yamaguch 
   1823  1.101  yamaguch 		m->m_len = m->m_pkthdr.len = len - sc->sc_hdr_size;
   1824  1.101  yamaguch 		m_set_rcvif(m, ifp);
   1825  1.101  yamaguch 		if_percpuq_enqueue(ifp->if_percpuq, m);
   1826  1.101  yamaguch 	}
   1827  1.101  yamaguch 
   1828  1.101  yamaguch done:
   1829  1.101  yamaguch 	if (ndeqp != NULL)
   1830  1.101  yamaguch 		*ndeqp = ndeq;
   1831  1.101  yamaguch 
   1832  1.101  yamaguch 	return more;
   1833  1.101  yamaguch }
   1834  1.101  yamaguch 
   1835  1.101  yamaguch static void
   1836  1.101  yamaguch vioif_rx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
   1837  1.101  yamaguch     struct vioif_netqueue *netq)
   1838  1.101  yamaguch {
   1839  1.101  yamaguch 	struct vioif_net_map *map;
   1840  1.101  yamaguch 	struct mbuf *m;
   1841  1.101  yamaguch 	unsigned int i, vq_num;
   1842  1.101  yamaguch 	bool more;
   1843  1.101  yamaguch 
   1844  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   1845  1.101  yamaguch 
   1846  1.101  yamaguch 	vq_num = netq->netq_vq->vq_num;
   1847  1.101  yamaguch 	for (;;) {
   1848  1.101  yamaguch 		more = vioif_rx_deq_locked(sc, vsc, netq, vq_num, NULL);
   1849  1.101  yamaguch 		if (more == false)
   1850  1.101  yamaguch 			break;
   1851  1.101  yamaguch 	}
   1852  1.101  yamaguch 
   1853  1.101  yamaguch 	for (i = 0; i < vq_num; i++) {
   1854  1.101  yamaguch 		map = &netq->netq_maps[i];
   1855  1.101  yamaguch 
   1856  1.101  yamaguch 		m = map->vnm_mbuf;
   1857  1.101  yamaguch 		if (m == NULL)
   1858  1.101  yamaguch 			continue;
   1859  1.101  yamaguch 
   1860  1.101  yamaguch 		vioif_net_unload_mbuf(vsc, map);
   1861  1.101  yamaguch 		m_freem(m);
   1862  1.101  yamaguch 	}
   1863  1.101  yamaguch 	mutex_exit(&netq->netq_lock);
   1864  1.101  yamaguch }
   1865  1.101  yamaguch 
   1866  1.101  yamaguch static void
   1867  1.101  yamaguch vioif_rx_handle_locked(void *xnetq, u_int limit)
   1868  1.101  yamaguch {
   1869  1.101  yamaguch 	struct vioif_netqueue *netq = xnetq;
   1870  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1871  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1872  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1873  1.101  yamaguch 	bool more;
   1874  1.101  yamaguch 	int enqueued;
   1875  1.101  yamaguch 	size_t ndeq;
   1876  1.101  yamaguch 
   1877  1.101  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1878  1.101  yamaguch 	KASSERT(!netq->netq_stopping);
   1879  1.101  yamaguch 
   1880  1.101  yamaguch 	more = vioif_rx_deq_locked(sc, vsc, netq, limit, &ndeq);
   1881  1.101  yamaguch 	if (ndeq > 0)
   1882  1.101  yamaguch 		vioif_populate_rx_mbufs_locked(sc, netq);
   1883  1.101  yamaguch 
   1884  1.101  yamaguch 	if (more) {
   1885  1.101  yamaguch 		vioif_net_sched_handle(sc, netq);
   1886  1.101  yamaguch 		return;
   1887  1.101  yamaguch 	}
   1888  1.101  yamaguch 
   1889  1.101  yamaguch 	enqueued = virtio_start_vq_intr(vsc, netq->netq_vq);
   1890  1.101  yamaguch 	if (enqueued != 0) {
   1891  1.101  yamaguch 		virtio_stop_vq_intr(vsc, netq->netq_vq);
   1892  1.101  yamaguch 		vioif_net_sched_handle(sc, netq);
   1893  1.101  yamaguch 		return;
   1894  1.101  yamaguch 	}
   1895  1.101  yamaguch 
   1896  1.101  yamaguch 	netq->netq_running_handle = false;
   1897  1.101  yamaguch }
   1898  1.101  yamaguch 
   1899  1.101  yamaguch static int
   1900  1.101  yamaguch vioif_rx_intr(void *arg)
   1901   1.99  yamaguch {
   1902  1.101  yamaguch 	struct vioif_netqueue *netq = arg;
   1903  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1904  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1905  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1906  1.101  yamaguch 	u_int limit;
   1907   1.99  yamaguch 
   1908  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   1909   1.99  yamaguch 
   1910  1.101  yamaguch 	/* handler is already running in softint/workqueue */
   1911  1.101  yamaguch 	if (netq->netq_running_handle)
   1912  1.101  yamaguch 		goto done;
   1913   1.99  yamaguch 
   1914  1.101  yamaguch 	netq->netq_running_handle = true;
   1915   1.99  yamaguch 
   1916  1.101  yamaguch 	limit = sc->sc_rx_intr_process_limit;
   1917  1.101  yamaguch 	virtio_stop_vq_intr(vsc, vq);
   1918  1.101  yamaguch 	vioif_rx_handle_locked(netq, limit);
   1919   1.99  yamaguch 
   1920  1.101  yamaguch done:
   1921  1.101  yamaguch 	mutex_exit(&netq->netq_lock);
   1922  1.101  yamaguch 	return 1;
   1923   1.99  yamaguch }
   1924   1.99  yamaguch 
   1925   1.99  yamaguch static void
   1926  1.101  yamaguch vioif_rx_handle(void *xnetq)
   1927   1.99  yamaguch {
   1928  1.101  yamaguch 	struct vioif_netqueue *netq = xnetq;
   1929  1.101  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1930  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   1931  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   1932  1.101  yamaguch 	u_int limit;
   1933   1.99  yamaguch 
   1934  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   1935   1.99  yamaguch 
   1936  1.101  yamaguch 	KASSERT(netq->netq_running_handle);
   1937   1.99  yamaguch 
   1938  1.101  yamaguch 	if (netq->netq_stopping) {
   1939  1.101  yamaguch 		netq->netq_running_handle = false;
   1940  1.101  yamaguch 		goto done;
   1941   1.99  yamaguch 	}
   1942   1.99  yamaguch 
   1943  1.101  yamaguch 	limit = sc->sc_rx_process_limit;
   1944  1.101  yamaguch 	vioif_rx_handle_locked(netq, limit);
   1945   1.99  yamaguch 
   1946  1.101  yamaguch done:
   1947  1.101  yamaguch 	mutex_exit(&netq->netq_lock);
   1948   1.99  yamaguch }
   1949   1.99  yamaguch 
   1950    1.1   hannken /*
   1951  1.101  yamaguch  * Transmition implementation
   1952    1.1   hannken  */
   1953  1.101  yamaguch /* enqueue mbufs to send */
   1954    1.1   hannken static void
   1955  1.101  yamaguch vioif_send_common_locked(struct ifnet *ifp, struct vioif_netqueue *netq,
   1956  1.101  yamaguch     bool is_transmit)
   1957   1.12     ozaki {
   1958  1.101  yamaguch 	struct vioif_softc *sc = ifp->if_softc;
   1959  1.101  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   1960   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   1961  1.101  yamaguch 	struct vioif_tx_context *txc;
   1962   1.95  yamaguch 	struct vioif_net_map *map;
   1963   1.89  yamaguch 	struct mbuf *m;
   1964  1.101  yamaguch 	int queued = 0;
   1965    1.1   hannken 
   1966   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   1967   1.98  yamaguch 
   1968  1.101  yamaguch 	if (netq->netq_stopping ||
   1969  1.101  yamaguch 	    !ISSET(ifp->if_flags, IFF_RUNNING))
   1970  1.101  yamaguch 		return;
   1971  1.101  yamaguch 
   1972  1.101  yamaguch 	txc = netq->netq_ctx;
   1973  1.101  yamaguch 
   1974  1.101  yamaguch 	if (!txc->txc_link_active)
   1975  1.101  yamaguch 		return;
   1976  1.101  yamaguch 
   1977  1.101  yamaguch 	if (!is_transmit &&
   1978  1.101  yamaguch 	    ISSET(ifp->if_flags, IFF_OACTIVE))
   1979  1.101  yamaguch 		return;
   1980    1.7     ozaki 
   1981  1.101  yamaguch 	for (;;) {
   1982  1.101  yamaguch 		int slot, r;
   1983    1.1   hannken 		r = virtio_enqueue_prep(vsc, vq, &slot);
   1984    1.1   hannken 		if (r == EAGAIN)
   1985    1.1   hannken 			break;
   1986   1.97  yamaguch 		if (__predict_false(r != 0))
   1987  1.101  yamaguch 			panic("enqueue_prep for tx buffers");
   1988  1.101  yamaguch 
   1989  1.101  yamaguch 		if (is_transmit)
   1990  1.101  yamaguch 			m = pcq_get(txc->txc_intrq);
   1991  1.101  yamaguch 		else
   1992  1.101  yamaguch 			IFQ_DEQUEUE(&ifp->if_snd, m);
   1993   1.89  yamaguch 
   1994   1.89  yamaguch 		if (m == NULL) {
   1995   1.91  yamaguch 			virtio_enqueue_abort(vsc, vq, slot);
   1996   1.91  yamaguch 			break;
   1997   1.91  yamaguch 		}
   1998   1.89  yamaguch 
   1999  1.101  yamaguch 		map = &netq->netq_maps[slot];
   2000  1.101  yamaguch 		KASSERT(map->vnm_mbuf == NULL);
   2001   1.89  yamaguch 
   2002  1.101  yamaguch 		r = vioif_net_load_mbuf(vsc, map, m, BUS_DMA_WRITE);
   2003   1.91  yamaguch 		if (r != 0) {
   2004  1.101  yamaguch 			/* maybe just too fragmented */
   2005  1.101  yamaguch 			struct mbuf *newm;
   2006  1.101  yamaguch 
   2007  1.101  yamaguch 			newm = m_defrag(m, M_NOWAIT);
   2008  1.101  yamaguch 			if (newm != NULL) {
   2009  1.101  yamaguch 				m = newm;
   2010  1.101  yamaguch 				r = vioif_net_load_mbuf(vsc, map, m,
   2011  1.101  yamaguch 				    BUS_DMA_WRITE);
   2012  1.101  yamaguch 			} else {
   2013  1.101  yamaguch 				txc->txc_defrag_failed.ev_count++;
   2014  1.101  yamaguch 				r = -1;
   2015  1.101  yamaguch 			}
   2016  1.101  yamaguch 
   2017  1.101  yamaguch 			if (r != 0) {
   2018  1.101  yamaguch 				netq->netq_mbuf_load_failed.ev_count++;
   2019  1.101  yamaguch 				m_freem(m);
   2020  1.101  yamaguch 				if_statinc(ifp, if_oerrors);
   2021  1.101  yamaguch 				virtio_enqueue_abort(vsc, vq, slot);
   2022  1.101  yamaguch 				continue;
   2023  1.101  yamaguch 			}
   2024    1.1   hannken 		}
   2025   1.89  yamaguch 
   2026  1.101  yamaguch 		memset(map->vnm_hdr, 0, sc->sc_hdr_size);
   2027  1.101  yamaguch 
   2028  1.101  yamaguch 		r = vioif_net_enqueue_tx(vsc, vq, slot, map);
   2029    1.1   hannken 		if (r != 0) {
   2030  1.101  yamaguch 			netq->netq_enqueue_failed.ev_count++;
   2031   1.99  yamaguch 			vioif_net_unload_mbuf(vsc, map);
   2032   1.89  yamaguch 			m_freem(m);
   2033  1.101  yamaguch 			/* slot already freed by vioif_net_enqueue_tx */
   2034  1.101  yamaguch 
   2035  1.101  yamaguch 			if_statinc(ifp, if_oerrors);
   2036  1.101  yamaguch 			continue;
   2037    1.1   hannken 		}
   2038   1.95  yamaguch 
   2039  1.101  yamaguch 		queued++;
   2040  1.101  yamaguch 		bpf_mtap(ifp, m, BPF_D_OUT);
   2041    1.1   hannken 	}
   2042   1.99  yamaguch 
   2043  1.101  yamaguch 	if (queued > 0) {
   2044   1.99  yamaguch 		vioif_notify(vsc, vq);
   2045  1.101  yamaguch 		ifp->if_timer = 5;
   2046   1.93  yamaguch 	}
   2047    1.7     ozaki }
   2048    1.7     ozaki 
   2049  1.101  yamaguch /* dequeue sent mbufs */
   2050   1.55  yamaguch static bool
   2051  1.101  yamaguch vioif_tx_deq_locked(struct vioif_softc *sc, struct virtio_softc *vsc,
   2052  1.101  yamaguch     struct vioif_netqueue *netq, u_int limit)
   2053    1.7     ozaki {
   2054   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2055    1.1   hannken 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2056   1.95  yamaguch 	struct vioif_net_map *map;
   2057    1.1   hannken 	struct mbuf *m;
   2058    1.1   hannken 	int slot, len;
   2059  1.101  yamaguch 	bool more = false;
   2060    1.1   hannken 
   2061   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   2062    1.7     ozaki 
   2063   1.54  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   2064  1.101  yamaguch 		return false;
   2065   1.55  yamaguch 
   2066  1.101  yamaguch 	for (;;) {
   2067  1.101  yamaguch 		if (limit-- == 0) {
   2068   1.55  yamaguch 			more = true;
   2069   1.55  yamaguch 			break;
   2070   1.55  yamaguch 		}
   2071   1.55  yamaguch 
   2072   1.55  yamaguch 		if (virtio_dequeue(vsc, vq, &slot, &len) != 0)
   2073   1.55  yamaguch 			break;
   2074   1.55  yamaguch 
   2075   1.98  yamaguch 		map = &netq->netq_maps[slot];
   2076   1.95  yamaguch 		KASSERT(map->vnm_mbuf != NULL);
   2077   1.99  yamaguch 		m = vioif_net_dequeue_commit(vsc, vq, slot,
   2078  1.101  yamaguch 		    map, BUS_DMASYNC_POSTWRITE);
   2079   1.99  yamaguch 		KASSERT(m != NULL);
   2080   1.95  yamaguch 
   2081  1.101  yamaguch 		if_statinc(ifp, if_opackets);
   2082  1.101  yamaguch 		m_freem(m);
   2083   1.70     skrll 	}
   2084   1.87  yamaguch 
   2085  1.101  yamaguch 	return more;
   2086   1.66   reinoud }
   2087   1.66   reinoud 
   2088  1.101  yamaguch static void
   2089  1.101  yamaguch vioif_tx_queue_clear(struct vioif_softc *sc, struct virtio_softc *vsc,
   2090  1.101  yamaguch     struct vioif_netqueue *netq)
   2091    1.1   hannken {
   2092  1.101  yamaguch 	struct vioif_net_map *map;
   2093  1.101  yamaguch 	struct mbuf *m;
   2094  1.101  yamaguch 	unsigned int i, vq_num;
   2095  1.101  yamaguch 	bool more;
   2096   1.55  yamaguch 
   2097   1.98  yamaguch 	mutex_enter(&netq->netq_lock);
   2098    1.7     ozaki 
   2099  1.101  yamaguch 	vq_num = netq->netq_vq->vq_num;
   2100  1.101  yamaguch 	for (;;) {
   2101  1.101  yamaguch 		more = vioif_tx_deq_locked(sc, vsc, netq, vq_num);
   2102  1.101  yamaguch 		if (more == false)
   2103  1.101  yamaguch 			break;
   2104  1.101  yamaguch 	}
   2105   1.55  yamaguch 
   2106  1.101  yamaguch 	for (i = 0; i < vq_num; i++) {
   2107  1.101  yamaguch 		map = &netq->netq_maps[i];
   2108   1.55  yamaguch 
   2109  1.101  yamaguch 		m = map->vnm_mbuf;
   2110  1.101  yamaguch 		if (m == NULL)
   2111  1.101  yamaguch 			continue;
   2112   1.55  yamaguch 
   2113  1.101  yamaguch 		vioif_net_unload_mbuf(vsc, map);
   2114  1.101  yamaguch 		m_freem(m);
   2115  1.101  yamaguch 	}
   2116   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2117   1.55  yamaguch }
   2118   1.55  yamaguch 
   2119   1.55  yamaguch static void
   2120  1.101  yamaguch vioif_start_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
   2121   1.55  yamaguch {
   2122   1.55  yamaguch 
   2123  1.101  yamaguch 	/*
   2124  1.101  yamaguch 	 * ifp->if_obytes and ifp->if_omcasts are added in if_transmit()@if.c.
   2125  1.101  yamaguch 	 */
   2126  1.101  yamaguch 	vioif_send_common_locked(ifp, netq, false);
   2127  1.101  yamaguch 
   2128  1.101  yamaguch }
   2129   1.85  yamaguch 
   2130  1.101  yamaguch static void
   2131  1.101  yamaguch vioif_transmit_locked(struct ifnet *ifp, struct vioif_netqueue *netq)
   2132  1.101  yamaguch {
   2133   1.55  yamaguch 
   2134  1.101  yamaguch 	vioif_send_common_locked(ifp, netq, true);
   2135  1.101  yamaguch }
   2136    1.1   hannken 
   2137  1.101  yamaguch static void
   2138  1.101  yamaguch vioif_deferred_transmit(void *arg)
   2139  1.101  yamaguch {
   2140  1.101  yamaguch 	struct vioif_netqueue *netq = arg;
   2141  1.101  yamaguch 	struct virtio_softc *vsc = netq->netq_vq->vq_owner;
   2142  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2143  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2144    1.1   hannken 
   2145  1.101  yamaguch 	mutex_enter(&netq->netq_lock);
   2146  1.101  yamaguch 	vioif_send_common_locked(ifp, netq, true);
   2147   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2148    1.1   hannken }
   2149    1.1   hannken 
   2150   1.66   reinoud static void
   2151   1.98  yamaguch vioif_tx_handle_locked(struct vioif_netqueue *netq, u_int limit)
   2152   1.66   reinoud {
   2153   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2154   1.98  yamaguch 	struct vioif_tx_context *txc = netq->netq_ctx;
   2155   1.66   reinoud 	struct virtio_softc *vsc = vq->vq_owner;
   2156   1.66   reinoud 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2157   1.66   reinoud 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2158   1.66   reinoud 	bool more;
   2159   1.87  yamaguch 	int enqueued;
   2160   1.66   reinoud 
   2161   1.98  yamaguch 	KASSERT(mutex_owned(&netq->netq_lock));
   2162   1.98  yamaguch 	KASSERT(!netq->netq_stopping);
   2163   1.66   reinoud 
   2164   1.98  yamaguch 	more = vioif_tx_deq_locked(sc, vsc, netq, limit);
   2165   1.66   reinoud 	if (more) {
   2166   1.98  yamaguch 		vioif_net_sched_handle(sc, netq);
   2167   1.66   reinoud 		return;
   2168   1.66   reinoud 	}
   2169   1.66   reinoud 
   2170   1.87  yamaguch 	enqueued = (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX) ?
   2171   1.87  yamaguch 	    virtio_postpone_intr_smart(vsc, vq):
   2172   1.87  yamaguch 	    virtio_start_vq_intr(vsc, vq);
   2173   1.87  yamaguch 	if (enqueued != 0) {
   2174   1.87  yamaguch 		virtio_stop_vq_intr(vsc, vq);
   2175   1.98  yamaguch 		vioif_net_sched_handle(sc, netq);
   2176   1.66   reinoud 		return;
   2177   1.66   reinoud 	}
   2178   1.66   reinoud 
   2179   1.98  yamaguch 	netq->netq_running_handle = false;
   2180   1.84  yamaguch 
   2181   1.66   reinoud 	/* for ALTQ */
   2182   1.98  yamaguch 	if (netq == &sc->sc_netqs[VIOIF_NETQ_TXQID(0)]) {
   2183   1.66   reinoud 		if_schedule_deferred_start(ifp);
   2184   1.66   reinoud 		ifp->if_flags &= ~IFF_OACTIVE;
   2185   1.66   reinoud 	}
   2186   1.98  yamaguch 	softint_schedule(txc->txc_deferred_transmit);
   2187   1.66   reinoud }
   2188   1.66   reinoud 
   2189    1.1   hannken static int
   2190   1.54  yamaguch vioif_tx_intr(void *arg)
   2191    1.1   hannken {
   2192   1.98  yamaguch 	struct vioif_netqueue *netq = arg;
   2193   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2194    1.1   hannken 	struct virtio_softc *vsc = vq->vq_owner;
   2195   1.32  jdolecek 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2196   1.55  yamaguch 	u_int limit;
   2197   1.55  yamaguch 
   2198   1.98  yamaguch 	mutex_enter(&netq->netq_lock);
   2199    1.7     ozaki 
   2200   1.84  yamaguch 	/* tx handler is already running in softint/workqueue */
   2201   1.98  yamaguch 	if (netq->netq_running_handle)
   2202   1.84  yamaguch 		goto done;
   2203    1.7     ozaki 
   2204   1.98  yamaguch 	if (netq->netq_stopping)
   2205   1.84  yamaguch 		goto done;
   2206   1.55  yamaguch 
   2207   1.98  yamaguch 	netq->netq_running_handle = true;
   2208   1.55  yamaguch 
   2209   1.84  yamaguch 	virtio_stop_vq_intr(vsc, vq);
   2210   1.98  yamaguch 	netq->netq_workqueue = sc->sc_txrx_workqueue_sysctl;
   2211   1.98  yamaguch 	limit = sc->sc_tx_intr_process_limit;
   2212   1.98  yamaguch 	vioif_tx_handle_locked(netq, limit);
   2213    1.7     ozaki 
   2214   1.84  yamaguch done:
   2215   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2216   1.55  yamaguch 	return 1;
   2217   1.55  yamaguch }
   2218   1.55  yamaguch 
   2219   1.55  yamaguch static void
   2220   1.98  yamaguch vioif_tx_handle(void *xnetq)
   2221   1.55  yamaguch {
   2222   1.98  yamaguch 	struct vioif_netqueue *netq = xnetq;
   2223   1.98  yamaguch 	struct virtqueue *vq = netq->netq_vq;
   2224   1.55  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   2225   1.55  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2226   1.55  yamaguch 	u_int limit;
   2227   1.55  yamaguch 
   2228   1.98  yamaguch 	mutex_enter(&netq->netq_lock);
   2229   1.85  yamaguch 
   2230   1.98  yamaguch 	KASSERT(netq->netq_running_handle);
   2231   1.85  yamaguch 
   2232   1.98  yamaguch 	if (netq->netq_stopping) {
   2233   1.98  yamaguch 		netq->netq_running_handle = false;
   2234   1.85  yamaguch 		goto done;
   2235   1.85  yamaguch 	}
   2236   1.85  yamaguch 
   2237   1.55  yamaguch 	limit = sc->sc_tx_process_limit;
   2238   1.98  yamaguch 	vioif_tx_handle_locked(netq, limit);
   2239   1.55  yamaguch 
   2240   1.85  yamaguch done:
   2241   1.98  yamaguch 	mutex_exit(&netq->netq_lock);
   2242    1.7     ozaki }
   2243    1.7     ozaki 
   2244    1.1   hannken /*
   2245    1.1   hannken  * Control vq
   2246    1.1   hannken  */
   2247    1.1   hannken /* issue a VIRTIO_NET_CTRL_RX class command and wait for completion */
   2248   1.44  yamaguch static void
   2249   1.44  yamaguch vioif_ctrl_acquire(struct vioif_softc *sc)
   2250    1.1   hannken {
   2251   1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2252    1.1   hannken 
   2253   1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2254   1.43  yamaguch 	while (ctrlq->ctrlq_inuse != FREE)
   2255   1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   2256   1.43  yamaguch 	ctrlq->ctrlq_inuse = INUSE;
   2257   1.44  yamaguch 	ctrlq->ctrlq_owner = curlwp;
   2258   1.44  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2259   1.44  yamaguch }
   2260   1.44  yamaguch 
   2261   1.44  yamaguch static void
   2262   1.44  yamaguch vioif_ctrl_release(struct vioif_softc *sc)
   2263   1.44  yamaguch {
   2264   1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2265   1.44  yamaguch 
   2266   1.44  yamaguch 	KASSERT(ctrlq->ctrlq_inuse != FREE);
   2267   1.44  yamaguch 	KASSERT(ctrlq->ctrlq_owner == curlwp);
   2268   1.44  yamaguch 
   2269   1.44  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2270   1.44  yamaguch 	ctrlq->ctrlq_inuse = FREE;
   2271   1.46  yamaguch 	ctrlq->ctrlq_owner = NULL;
   2272   1.44  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   2273   1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2274   1.44  yamaguch }
   2275   1.44  yamaguch 
   2276   1.44  yamaguch static int
   2277   1.44  yamaguch vioif_ctrl_load_cmdspec(struct vioif_softc *sc,
   2278   1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2279   1.44  yamaguch {
   2280   1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2281   1.44  yamaguch 	int i, r, loaded;
   2282   1.44  yamaguch 
   2283   1.44  yamaguch 	loaded = 0;
   2284   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2285   1.44  yamaguch 		r = bus_dmamap_load(virtio_dmat(vsc),
   2286   1.44  yamaguch 		    specs[i].dmamap, specs[i].buf, specs[i].bufsize,
   2287   1.48   msaitoh 		    NULL, BUS_DMA_WRITE | BUS_DMA_NOWAIT);
   2288   1.44  yamaguch 		if (r) {
   2289   1.63  yamaguch 			sc->sc_ctrlq.ctrlq_cmd_load_failed.ev_count++;
   2290   1.44  yamaguch 			goto err;
   2291   1.44  yamaguch 		}
   2292   1.44  yamaguch 		loaded++;
   2293   1.44  yamaguch 
   2294   1.44  yamaguch 	}
   2295   1.44  yamaguch 
   2296   1.44  yamaguch 	return r;
   2297   1.44  yamaguch 
   2298   1.44  yamaguch err:
   2299   1.44  yamaguch 	for (i = 0; i < loaded; i++) {
   2300   1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   2301   1.44  yamaguch 	}
   2302   1.44  yamaguch 
   2303   1.44  yamaguch 	return r;
   2304   1.44  yamaguch }
   2305   1.44  yamaguch 
   2306   1.44  yamaguch static void
   2307   1.44  yamaguch vioif_ctrl_unload_cmdspec(struct vioif_softc *sc,
   2308   1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2309   1.44  yamaguch {
   2310   1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2311   1.44  yamaguch 	int i;
   2312   1.44  yamaguch 
   2313   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2314   1.44  yamaguch 		bus_dmamap_unload(virtio_dmat(vsc), specs[i].dmamap);
   2315   1.44  yamaguch 	}
   2316   1.44  yamaguch }
   2317   1.44  yamaguch 
   2318   1.44  yamaguch static int
   2319   1.44  yamaguch vioif_ctrl_send_command(struct vioif_softc *sc, uint8_t class, uint8_t cmd,
   2320   1.44  yamaguch     struct vioif_ctrl_cmdspec *specs, int nspecs)
   2321   1.44  yamaguch {
   2322   1.44  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2323   1.44  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2324   1.44  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2325   1.44  yamaguch 	int i, r, slot;
   2326   1.43  yamaguch 
   2327   1.44  yamaguch 	ctrlq->ctrlq_cmd->class = class;
   2328   1.43  yamaguch 	ctrlq->ctrlq_cmd->command = cmd;
   2329    1.1   hannken 
   2330   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap,
   2331   1.50  christos 	    0, sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_PREWRITE);
   2332   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2333   1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap,
   2334   1.50  christos 		    0, specs[i].bufsize, BUS_DMASYNC_PREWRITE);
   2335   1.44  yamaguch 	}
   2336   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap,
   2337   1.50  christos 	    0, sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_PREREAD);
   2338    1.1   hannken 
   2339   1.66   reinoud 	/* we need to explicitly (re)start vq intr when using RING EVENT IDX */
   2340   1.66   reinoud 	if (virtio_features(vsc) & VIRTIO_F_RING_EVENT_IDX)
   2341   1.66   reinoud 		virtio_start_vq_intr(vsc, ctrlq->ctrlq_vq);
   2342   1.66   reinoud 
   2343    1.1   hannken 	r = virtio_enqueue_prep(vsc, vq, &slot);
   2344    1.1   hannken 	if (r != 0)
   2345    1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2346   1.44  yamaguch 	r = virtio_enqueue_reserve(vsc, vq, slot, nspecs + 2);
   2347    1.1   hannken 	if (r != 0)
   2348    1.1   hannken 		panic("%s: control vq busy!?", device_xname(sc->sc_dev));
   2349   1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_cmd_dmamap, true);
   2350   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2351   1.44  yamaguch 		virtio_enqueue(vsc, vq, slot, specs[i].dmamap, true);
   2352   1.44  yamaguch 	}
   2353   1.43  yamaguch 	virtio_enqueue(vsc, vq, slot, ctrlq->ctrlq_status_dmamap, false);
   2354    1.1   hannken 	virtio_enqueue_commit(vsc, vq, slot, true);
   2355    1.1   hannken 
   2356    1.1   hannken 	/* wait for done */
   2357   1.43  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2358   1.43  yamaguch 	while (ctrlq->ctrlq_inuse != DONE)
   2359   1.43  yamaguch 		cv_wait(&ctrlq->ctrlq_wait, &ctrlq->ctrlq_wait_lock);
   2360   1.43  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2361    1.1   hannken 	/* already dequeueued */
   2362    1.1   hannken 
   2363   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_cmd_dmamap, 0,
   2364   1.50  christos 	    sizeof(struct virtio_net_ctrl_cmd), BUS_DMASYNC_POSTWRITE);
   2365   1.44  yamaguch 	for (i = 0; i < nspecs; i++) {
   2366   1.44  yamaguch 		bus_dmamap_sync(virtio_dmat(vsc), specs[i].dmamap, 0,
   2367   1.50  christos 		    specs[i].bufsize, BUS_DMASYNC_POSTWRITE);
   2368   1.44  yamaguch 	}
   2369   1.43  yamaguch 	bus_dmamap_sync(virtio_dmat(vsc), ctrlq->ctrlq_status_dmamap, 0,
   2370   1.50  christos 	    sizeof(struct virtio_net_ctrl_status), BUS_DMASYNC_POSTREAD);
   2371    1.1   hannken 
   2372   1.43  yamaguch 	if (ctrlq->ctrlq_status->ack == VIRTIO_NET_OK)
   2373    1.1   hannken 		r = 0;
   2374    1.1   hannken 	else {
   2375   1.63  yamaguch 		device_printf(sc->sc_dev, "failed setting rx mode\n");
   2376   1.63  yamaguch 		sc->sc_ctrlq.ctrlq_cmd_failed.ev_count++;
   2377    1.1   hannken 		r = EIO;
   2378    1.1   hannken 	}
   2379    1.1   hannken 
   2380   1.44  yamaguch 	return r;
   2381   1.44  yamaguch }
   2382   1.44  yamaguch 
   2383  1.101  yamaguch /* ctrl vq interrupt; wake up the command issuer */
   2384  1.101  yamaguch static int
   2385  1.101  yamaguch vioif_ctrl_intr(void *arg)
   2386  1.101  yamaguch {
   2387  1.101  yamaguch 	struct vioif_ctrlqueue *ctrlq = arg;
   2388  1.101  yamaguch 	struct virtqueue *vq = ctrlq->ctrlq_vq;
   2389  1.101  yamaguch 	struct virtio_softc *vsc = vq->vq_owner;
   2390  1.101  yamaguch 	int r, slot;
   2391  1.101  yamaguch 
   2392  1.101  yamaguch 	if (virtio_vq_is_enqueued(vsc, vq) == false)
   2393  1.101  yamaguch 		return 0;
   2394  1.101  yamaguch 
   2395  1.101  yamaguch 	r = virtio_dequeue(vsc, vq, &slot, NULL);
   2396  1.101  yamaguch 	if (r == ENOENT)
   2397  1.101  yamaguch 		return 0;
   2398  1.101  yamaguch 	virtio_dequeue_commit(vsc, vq, slot);
   2399  1.101  yamaguch 
   2400  1.101  yamaguch 	mutex_enter(&ctrlq->ctrlq_wait_lock);
   2401  1.101  yamaguch 	ctrlq->ctrlq_inuse = DONE;
   2402  1.101  yamaguch 	cv_signal(&ctrlq->ctrlq_wait);
   2403  1.101  yamaguch 	mutex_exit(&ctrlq->ctrlq_wait_lock);
   2404  1.101  yamaguch 
   2405  1.101  yamaguch 	return 1;
   2406  1.101  yamaguch }
   2407  1.101  yamaguch 
   2408   1.44  yamaguch static int
   2409   1.44  yamaguch vioif_ctrl_rx(struct vioif_softc *sc, int cmd, bool onoff)
   2410   1.44  yamaguch {
   2411   1.44  yamaguch 	struct virtio_net_ctrl_rx *rx = sc->sc_ctrlq.ctrlq_rx;
   2412   1.44  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2413   1.44  yamaguch 	int r;
   2414   1.44  yamaguch 
   2415   1.44  yamaguch 	if (!sc->sc_has_ctrl)
   2416   1.44  yamaguch 		return ENOTSUP;
   2417   1.44  yamaguch 
   2418   1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2419   1.44  yamaguch 
   2420   1.44  yamaguch 	rx->onoff = onoff;
   2421   1.44  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_rx_dmamap;
   2422   1.44  yamaguch 	specs[0].buf = rx;
   2423   1.44  yamaguch 	specs[0].bufsize = sizeof(*rx);
   2424   1.44  yamaguch 
   2425   1.44  yamaguch 	r = vioif_ctrl_send_command(sc, VIRTIO_NET_CTRL_RX, cmd,
   2426   1.44  yamaguch 	    specs, __arraycount(specs));
   2427    1.3  christos 
   2428   1.44  yamaguch 	vioif_ctrl_release(sc);
   2429    1.1   hannken 	return r;
   2430    1.1   hannken }
   2431    1.1   hannken 
   2432    1.1   hannken static int
   2433    1.1   hannken vioif_set_promisc(struct vioif_softc *sc, bool onoff)
   2434    1.1   hannken {
   2435   1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_PROMISC, onoff);
   2436    1.1   hannken }
   2437    1.1   hannken 
   2438    1.1   hannken static int
   2439    1.1   hannken vioif_set_allmulti(struct vioif_softc *sc, bool onoff)
   2440    1.1   hannken {
   2441   1.50  christos 	return vioif_ctrl_rx(sc, VIRTIO_NET_CTRL_RX_ALLMULTI, onoff);
   2442    1.1   hannken }
   2443    1.1   hannken 
   2444    1.1   hannken static int
   2445  1.101  yamaguch vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *sc, int nvq_pairs)
   2446    1.1   hannken {
   2447  1.101  yamaguch 	struct virtio_net_ctrl_mq *mq = sc->sc_ctrlq.ctrlq_mq;
   2448  1.101  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2449   1.44  yamaguch 	int r;
   2450   1.44  yamaguch 
   2451   1.32  jdolecek 	if (!sc->sc_has_ctrl)
   2452    1.1   hannken 		return ENOTSUP;
   2453    1.1   hannken 
   2454  1.101  yamaguch 	if (nvq_pairs <= 1)
   2455  1.101  yamaguch 		return EINVAL;
   2456  1.101  yamaguch 
   2457   1.44  yamaguch 	vioif_ctrl_acquire(sc);
   2458    1.1   hannken 
   2459  1.101  yamaguch 	mq->virtqueue_pairs = virtio_rw16(sc->sc_virtio, nvq_pairs);
   2460  1.101  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_mq_dmamap;
   2461  1.101  yamaguch 	specs[0].buf = mq;
   2462  1.101  yamaguch 	specs[0].bufsize = sizeof(*mq);
   2463    1.1   hannken 
   2464   1.44  yamaguch 	r = vioif_ctrl_send_command(sc,
   2465  1.101  yamaguch 	    VIRTIO_NET_CTRL_MQ, VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET,
   2466  1.101  yamaguch 	    specs, __arraycount(specs));
   2467    1.1   hannken 
   2468   1.44  yamaguch 	vioif_ctrl_release(sc);
   2469    1.3  christos 
   2470    1.1   hannken 	return r;
   2471    1.1   hannken }
   2472    1.1   hannken 
   2473   1.46  yamaguch static int
   2474   1.74  yamaguch vioif_set_mac_addr(struct vioif_softc *sc)
   2475   1.74  yamaguch {
   2476   1.74  yamaguch 	struct virtio_net_ctrl_mac_addr *ma =
   2477   1.74  yamaguch 	    sc->sc_ctrlq.ctrlq_mac_addr;
   2478   1.74  yamaguch 	struct vioif_ctrl_cmdspec specs[1];
   2479   1.74  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2480   1.74  yamaguch 	int nspecs = __arraycount(specs);
   2481   1.78  yamaguch 	uint64_t features;
   2482   1.74  yamaguch 	int r;
   2483   1.78  yamaguch 	size_t i;
   2484   1.74  yamaguch 
   2485   1.74  yamaguch 	if (!sc->sc_has_ctrl)
   2486   1.74  yamaguch 		return ENOTSUP;
   2487   1.74  yamaguch 
   2488   1.78  yamaguch 	if (memcmp(CLLADDR(ifp->if_sadl), sc->sc_mac,
   2489   1.78  yamaguch 	    ETHER_ADDR_LEN) == 0) {
   2490   1.78  yamaguch 		return 0;
   2491   1.78  yamaguch 	}
   2492   1.74  yamaguch 
   2493   1.78  yamaguch 	memcpy(sc->sc_mac, CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
   2494   1.74  yamaguch 
   2495   1.78  yamaguch 	features = virtio_features(sc->sc_virtio);
   2496   1.78  yamaguch 	if (features & VIRTIO_NET_F_CTRL_MAC_ADDR) {
   2497   1.78  yamaguch 		vioif_ctrl_acquire(sc);
   2498   1.78  yamaguch 
   2499   1.78  yamaguch 		memcpy(ma->mac, sc->sc_mac, ETHER_ADDR_LEN);
   2500   1.78  yamaguch 		specs[0].dmamap = sc->sc_ctrlq.ctrlq_mac_addr_dmamap;
   2501   1.78  yamaguch 		specs[0].buf = ma;
   2502   1.78  yamaguch 		specs[0].bufsize = sizeof(*ma);
   2503   1.78  yamaguch 
   2504   1.78  yamaguch 		r = vioif_ctrl_send_command(sc,
   2505   1.78  yamaguch 		    VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_ADDR_SET,
   2506   1.78  yamaguch 		    specs, nspecs);
   2507   1.74  yamaguch 
   2508   1.78  yamaguch 		vioif_ctrl_release(sc);
   2509   1.78  yamaguch 	} else {
   2510   1.78  yamaguch 		for (i = 0; i < __arraycount(sc->sc_mac); i++) {
   2511   1.78  yamaguch 			virtio_write_device_config_1(sc->sc_virtio,
   2512   1.78  yamaguch 			    VIRTIO_NET_CONFIG_MAC + i, sc->sc_mac[i]);
   2513   1.78  yamaguch 		}
   2514   1.78  yamaguch 		r = 0;
   2515   1.78  yamaguch 	}
   2516   1.74  yamaguch 
   2517   1.74  yamaguch 	return r;
   2518   1.74  yamaguch }
   2519   1.74  yamaguch 
   2520   1.74  yamaguch static int
   2521  1.101  yamaguch vioif_set_rx_filter(struct vioif_softc *sc)
   2522    1.1   hannken {
   2523  1.101  yamaguch 	/* filter already set in ctrlq->ctrlq_mac_tbl */
   2524  1.101  yamaguch 	struct virtio_softc *vsc = sc->sc_virtio;
   2525  1.101  yamaguch 	struct virtio_net_ctrl_mac_tbl *mac_tbl_uc, *mac_tbl_mc;
   2526  1.101  yamaguch 	struct vioif_ctrl_cmdspec specs[2];
   2527  1.101  yamaguch 	int nspecs = __arraycount(specs);
   2528  1.101  yamaguch 	int r;
   2529    1.1   hannken 
   2530  1.101  yamaguch 	mac_tbl_uc = sc->sc_ctrlq.ctrlq_mac_tbl_uc;
   2531  1.101  yamaguch 	mac_tbl_mc = sc->sc_ctrlq.ctrlq_mac_tbl_mc;
   2532   1.54  yamaguch 
   2533  1.101  yamaguch 	if (!sc->sc_has_ctrl)
   2534  1.101  yamaguch 		return ENOTSUP;
   2535    1.1   hannken 
   2536  1.101  yamaguch 	vioif_ctrl_acquire(sc);
   2537    1.1   hannken 
   2538  1.101  yamaguch 	specs[0].dmamap = sc->sc_ctrlq.ctrlq_tbl_uc_dmamap;
   2539  1.101  yamaguch 	specs[0].buf = mac_tbl_uc;
   2540  1.101  yamaguch 	specs[0].bufsize = sizeof(*mac_tbl_uc)
   2541  1.101  yamaguch 	    + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_uc->nentries));
   2542    1.1   hannken 
   2543  1.101  yamaguch 	specs[1].dmamap = sc->sc_ctrlq.ctrlq_tbl_mc_dmamap;
   2544  1.101  yamaguch 	specs[1].buf = mac_tbl_mc;
   2545  1.101  yamaguch 	specs[1].bufsize = sizeof(*mac_tbl_mc)
   2546  1.101  yamaguch 	    + (ETHER_ADDR_LEN * virtio_rw32(vsc, mac_tbl_mc->nentries));
   2547   1.75  yamaguch 
   2548  1.101  yamaguch 	r = vioif_ctrl_load_cmdspec(sc, specs, nspecs);
   2549  1.101  yamaguch 	if (r != 0)
   2550  1.101  yamaguch 		goto out;
   2551   1.75  yamaguch 
   2552  1.101  yamaguch 	r = vioif_ctrl_send_command(sc,
   2553  1.101  yamaguch 	    VIRTIO_NET_CTRL_MAC, VIRTIO_NET_CTRL_MAC_TABLE_SET,
   2554  1.101  yamaguch 	    specs, nspecs);
   2555   1.75  yamaguch 
   2556  1.101  yamaguch 	vioif_ctrl_unload_cmdspec(sc, specs, nspecs);
   2557   1.75  yamaguch 
   2558  1.101  yamaguch out:
   2559  1.101  yamaguch 	vioif_ctrl_release(sc);
   2560   1.75  yamaguch 
   2561  1.101  yamaguch 	return r;
   2562   1.75  yamaguch }
   2563   1.75  yamaguch 
   2564    1.1   hannken /*
   2565    1.1   hannken  * If multicast filter small enough (<=MAXENTRIES) set rx filter
   2566    1.1   hannken  * If large multicast filter exist use ALLMULTI
   2567    1.1   hannken  * If setting rx filter fails fall back to ALLMULTI
   2568    1.1   hannken  */
   2569    1.1   hannken static int
   2570    1.1   hannken vioif_rx_filter(struct vioif_softc *sc)
   2571    1.1   hannken {
   2572   1.66   reinoud 	struct virtio_softc *vsc = sc->sc_virtio;
   2573   1.48   msaitoh 	struct ethercom *ec = &sc->sc_ethercom;
   2574   1.48   msaitoh 	struct ifnet *ifp = &ec->ec_if;
   2575    1.1   hannken 	struct ether_multi *enm;
   2576    1.1   hannken 	struct ether_multistep step;
   2577   1.43  yamaguch 	struct vioif_ctrlqueue *ctrlq = &sc->sc_ctrlq;
   2578    1.1   hannken 	int nentries;
   2579   1.75  yamaguch 	bool allmulti = 0;
   2580    1.1   hannken 	int r;
   2581    1.1   hannken 
   2582   1.75  yamaguch 	if (!sc->sc_has_ctrl) {
   2583   1.75  yamaguch 		goto set_ifflags;
   2584    1.1   hannken 	}
   2585    1.1   hannken 
   2586   1.74  yamaguch 	memcpy(ctrlq->ctrlq_mac_tbl_uc->macs[0],
   2587   1.74  yamaguch 	    CLLADDR(ifp->if_sadl), ETHER_ADDR_LEN);
   2588   1.74  yamaguch 
   2589   1.75  yamaguch 	nentries = 0;
   2590   1.75  yamaguch 	allmulti = false;
   2591   1.75  yamaguch 
   2592   1.48   msaitoh 	ETHER_LOCK(ec);
   2593   1.75  yamaguch 	for (ETHER_FIRST_MULTI(step, ec, enm); enm != NULL;
   2594   1.75  yamaguch 	    ETHER_NEXT_MULTI(step, enm)) {
   2595    1.1   hannken 		if (nentries >= VIRTIO_NET_CTRL_MAC_MAXENTRIES) {
   2596   1.75  yamaguch 			allmulti = true;
   2597   1.75  yamaguch 			break;
   2598    1.1   hannken 		}
   2599   1.50  christos 		if (memcmp(enm->enm_addrlo, enm->enm_addrhi, ETHER_ADDR_LEN)) {
   2600   1.75  yamaguch 			allmulti = true;
   2601   1.75  yamaguch 			break;
   2602    1.1   hannken 		}
   2603   1.75  yamaguch 
   2604   1.43  yamaguch 		memcpy(ctrlq->ctrlq_mac_tbl_mc->macs[nentries],
   2605   1.50  christos 		    enm->enm_addrlo, ETHER_ADDR_LEN);
   2606   1.75  yamaguch 		nentries++;
   2607    1.1   hannken 	}
   2608   1.48   msaitoh 	ETHER_UNLOCK(ec);
   2609   1.30     ozaki 
   2610   1.74  yamaguch 	r = vioif_set_mac_addr(sc);
   2611   1.74  yamaguch 	if (r != 0) {
   2612   1.74  yamaguch 		log(LOG_WARNING, "%s: couldn't set MAC address\n",
   2613   1.74  yamaguch 		    ifp->if_xname);
   2614   1.74  yamaguch 	}
   2615   1.74  yamaguch 
   2616   1.75  yamaguch 	if (!allmulti) {
   2617   1.74  yamaguch 		ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 1);
   2618   1.66   reinoud 		ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, nentries);
   2619    1.1   hannken 		r = vioif_set_rx_filter(sc);
   2620    1.1   hannken 		if (r != 0) {
   2621   1.75  yamaguch 			allmulti = true; /* fallback */
   2622    1.1   hannken 		}
   2623   1.75  yamaguch 	}
   2624   1.75  yamaguch 
   2625   1.75  yamaguch 	if (allmulti) {
   2626   1.66   reinoud 		ctrlq->ctrlq_mac_tbl_uc->nentries = virtio_rw32(vsc, 0);
   2627   1.66   reinoud 		ctrlq->ctrlq_mac_tbl_mc->nentries = virtio_rw32(vsc, 0);
   2628    1.1   hannken 		r = vioif_set_rx_filter(sc);
   2629    1.1   hannken 		if (r != 0) {
   2630   1.75  yamaguch 			log(LOG_DEBUG, "%s: couldn't clear RX filter\n",
   2631   1.75  yamaguch 			    ifp->if_xname);
   2632   1.75  yamaguch 			/* what to do on failure? */
   2633    1.1   hannken 		}
   2634   1.75  yamaguch 
   2635   1.75  yamaguch 		ifp->if_flags |= IFF_ALLMULTI;
   2636    1.1   hannken 	}
   2637   1.75  yamaguch 
   2638   1.75  yamaguch set_ifflags:
   2639   1.75  yamaguch 	r = vioif_ifflags(sc);
   2640    1.1   hannken 
   2641    1.1   hannken 	return r;
   2642    1.1   hannken }
   2643    1.1   hannken 
   2644  1.101  yamaguch /*
   2645  1.101  yamaguch  * VM configuration changes
   2646  1.101  yamaguch  */
   2647  1.101  yamaguch static int
   2648  1.101  yamaguch vioif_config_change(struct virtio_softc *vsc)
   2649  1.101  yamaguch {
   2650  1.101  yamaguch 	struct vioif_softc *sc = device_private(virtio_child(vsc));
   2651  1.101  yamaguch 
   2652  1.101  yamaguch 	softint_schedule(sc->sc_cfg_softint);
   2653  1.101  yamaguch 	return 0;
   2654  1.101  yamaguch }
   2655  1.101  yamaguch 
   2656  1.101  yamaguch static void
   2657  1.101  yamaguch vioif_cfg_softint(void *arg)
   2658  1.101  yamaguch {
   2659  1.101  yamaguch 	struct vioif_softc *sc = arg;
   2660  1.101  yamaguch 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2661  1.101  yamaguch 
   2662  1.101  yamaguch 	vioif_update_link_status(sc);
   2663  1.101  yamaguch 	vioif_start(ifp);
   2664  1.101  yamaguch }
   2665  1.101  yamaguch 
   2666   1.82  knakahar static int
   2667   1.82  knakahar vioif_get_link_status(struct vioif_softc *sc)
   2668   1.34     ozaki {
   2669   1.34     ozaki 	struct virtio_softc *vsc = sc->sc_virtio;
   2670   1.34     ozaki 	uint16_t status;
   2671   1.34     ozaki 
   2672   1.34     ozaki 	if (virtio_features(vsc) & VIRTIO_NET_F_STATUS)
   2673   1.34     ozaki 		status = virtio_read_device_config_2(vsc,
   2674   1.34     ozaki 		    VIRTIO_NET_CONFIG_STATUS);
   2675   1.34     ozaki 	else
   2676   1.34     ozaki 		status = VIRTIO_NET_S_LINK_UP;
   2677   1.34     ozaki 
   2678   1.82  knakahar 	if ((status & VIRTIO_NET_S_LINK_UP) != 0)
   2679   1.82  knakahar 		return LINK_STATE_UP;
   2680   1.82  knakahar 
   2681   1.82  knakahar 	return LINK_STATE_DOWN;
   2682   1.34     ozaki }
   2683   1.34     ozaki 
   2684   1.34     ozaki static void
   2685   1.34     ozaki vioif_update_link_status(struct vioif_softc *sc)
   2686   1.34     ozaki {
   2687   1.34     ozaki 	struct ifnet *ifp = &sc->sc_ethercom.ec_if;
   2688   1.98  yamaguch 	struct vioif_netqueue *netq;
   2689   1.98  yamaguch 	struct vioif_tx_context *txc;
   2690   1.82  knakahar 	bool active;
   2691   1.46  yamaguch 	int link, i;
   2692   1.34     ozaki 
   2693   1.62  yamaguch 	mutex_enter(&sc->sc_lock);
   2694   1.62  yamaguch 
   2695   1.82  knakahar 	link = vioif_get_link_status(sc);
   2696   1.34     ozaki 
   2697   1.82  knakahar 	if (link == sc->sc_link_state)
   2698   1.82  knakahar 		goto done;
   2699   1.34     ozaki 
   2700   1.82  knakahar 	sc->sc_link_state = link;
   2701   1.82  knakahar 
   2702   1.82  knakahar 	active = VIOIF_IS_LINK_ACTIVE(sc);
   2703   1.82  knakahar 	for (i = 0; i < sc->sc_act_nvq_pairs; i++) {
   2704   1.98  yamaguch 		netq = &sc->sc_netqs[VIOIF_NETQ_TXQID(i)];
   2705   1.34     ozaki 
   2706   1.98  yamaguch 		mutex_enter(&netq->netq_lock);
   2707   1.98  yamaguch 		txc = netq->netq_ctx;
   2708   1.98  yamaguch 		txc->txc_link_active = active;
   2709   1.98  yamaguch 		mutex_exit(&netq->netq_lock);
   2710   1.34     ozaki 	}
   2711   1.34     ozaki 
   2712   1.82  knakahar 	if_link_state_change(ifp, sc->sc_link_state);
   2713   1.62  yamaguch 
   2714   1.82  knakahar done:
   2715   1.62  yamaguch 	mutex_exit(&sc->sc_lock);
   2716   1.34     ozaki }
   2717   1.34     ozaki 
   2718  1.101  yamaguch static void
   2719  1.101  yamaguch vioif_workq_work(struct work *wk, void *context)
   2720   1.34     ozaki {
   2721  1.101  yamaguch 	struct vioif_work *work;
   2722   1.34     ozaki 
   2723  1.101  yamaguch 	work = container_of(wk, struct vioif_work, cookie);
   2724   1.34     ozaki 
   2725  1.101  yamaguch 	atomic_store_relaxed(&work->added, 0);
   2726  1.101  yamaguch 	work->func(work->arg);
   2727   1.34     ozaki }
   2728   1.34     ozaki 
   2729   1.55  yamaguch static struct workqueue *
   2730   1.55  yamaguch vioif_workq_create(const char *name, pri_t prio, int ipl, int flags)
   2731   1.55  yamaguch {
   2732   1.55  yamaguch 	struct workqueue *wq;
   2733   1.55  yamaguch 	int error;
   2734   1.55  yamaguch 
   2735   1.55  yamaguch 	error = workqueue_create(&wq, name, vioif_workq_work, NULL,
   2736   1.55  yamaguch 	    prio, ipl, flags);
   2737   1.55  yamaguch 
   2738   1.55  yamaguch 	if (error)
   2739   1.55  yamaguch 		return NULL;
   2740   1.55  yamaguch 
   2741   1.55  yamaguch 	return wq;
   2742   1.55  yamaguch }
   2743   1.55  yamaguch 
   2744   1.55  yamaguch static void
   2745   1.55  yamaguch vioif_workq_destroy(struct workqueue *wq)
   2746   1.55  yamaguch {
   2747   1.55  yamaguch 
   2748   1.55  yamaguch 	workqueue_destroy(wq);
   2749   1.55  yamaguch }
   2750   1.55  yamaguch 
   2751   1.55  yamaguch static void
   2752   1.55  yamaguch vioif_work_set(struct vioif_work *work, void (*func)(void *), void *arg)
   2753   1.55  yamaguch {
   2754   1.55  yamaguch 
   2755   1.55  yamaguch 	memset(work, 0, sizeof(*work));
   2756   1.55  yamaguch 	work->func = func;
   2757   1.55  yamaguch 	work->arg = arg;
   2758   1.55  yamaguch }
   2759   1.55  yamaguch 
   2760   1.55  yamaguch static void
   2761   1.55  yamaguch vioif_work_add(struct workqueue *wq, struct vioif_work *work)
   2762   1.55  yamaguch {
   2763   1.55  yamaguch 
   2764   1.55  yamaguch 	if (atomic_load_relaxed(&work->added) != 0)
   2765   1.55  yamaguch 		return;
   2766   1.55  yamaguch 
   2767   1.55  yamaguch 	atomic_store_relaxed(&work->added, 1);
   2768   1.55  yamaguch 	kpreempt_disable();
   2769   1.55  yamaguch 	workqueue_enqueue(wq, &work->cookie, NULL);
   2770   1.55  yamaguch 	kpreempt_enable();
   2771   1.55  yamaguch }
   2772   1.55  yamaguch 
   2773   1.55  yamaguch static void
   2774   1.55  yamaguch vioif_work_wait(struct workqueue *wq, struct vioif_work *work)
   2775   1.55  yamaguch {
   2776   1.55  yamaguch 
   2777   1.55  yamaguch 	workqueue_wait(wq, &work->cookie);
   2778   1.55  yamaguch }
   2779   1.55  yamaguch 
   2780   1.26  pgoyette MODULE(MODULE_CLASS_DRIVER, if_vioif, "virtio");
   2781   1.48   msaitoh 
   2782   1.26  pgoyette #ifdef _MODULE
   2783   1.26  pgoyette #include "ioconf.c"
   2784   1.26  pgoyette #endif
   2785   1.48   msaitoh 
   2786   1.48   msaitoh static int
   2787   1.26  pgoyette if_vioif_modcmd(modcmd_t cmd, void *opaque)
   2788   1.26  pgoyette {
   2789   1.26  pgoyette 	int error = 0;
   2790   1.48   msaitoh 
   2791   1.26  pgoyette #ifdef _MODULE
   2792   1.26  pgoyette 	switch (cmd) {
   2793   1.26  pgoyette 	case MODULE_CMD_INIT:
   2794   1.48   msaitoh 		error = config_init_component(cfdriver_ioconf_if_vioif,
   2795   1.48   msaitoh 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2796   1.26  pgoyette 		break;
   2797   1.26  pgoyette 	case MODULE_CMD_FINI:
   2798   1.26  pgoyette 		error = config_fini_component(cfdriver_ioconf_if_vioif,
   2799   1.26  pgoyette 		    cfattach_ioconf_if_vioif, cfdata_ioconf_if_vioif);
   2800   1.26  pgoyette 		break;
   2801   1.26  pgoyette 	default:
   2802   1.26  pgoyette 		error = ENOTTY;
   2803   1.48   msaitoh 		break;
   2804   1.26  pgoyette 	}
   2805   1.26  pgoyette #endif
   2806   1.48   msaitoh 
   2807   1.26  pgoyette 	return error;
   2808   1.26  pgoyette }
   2809